FailedProxy/Varaverk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

massive update. Master conf split, now modular with a load sceriprt to drive all configs to scripts. with unraid scpecific safeguard tests , and improved standardized ux. including dynamic host detect, who am i who else it there. EVERY SINGLE SCRIPT UPDATED. DEBATING THAT THIS IS ACUALLY V2

Browse Source
This commit is contained in:
FailedProxy
2026-05-03 17:16:49 -04:00
parent 2691a35e80
commit ec7de648dc
72 changed files with 25640 additions and 14629 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.vscode/settings.json
Vendored
+10
View File
@@ -0,0 +1,10 @@
{
"files.exclude": {
"**/.cache/**": true,
"**/.next/**": true,
"**/build/**": true,
"**/coverage/**": true,
"**/dist/**": true,
"**/node_modules/**": true
}
}
Docker_Essentials/README-Docker_Essentials.md
+1138 -276
View File
File diff suppressed because it is too large Load Diff
Docker_Essentials/docker_daily_restart.sh
+241 -58
View File
@@ -1,24 +1,64 @@
#!/bin/bash
# -----------------------------------------------------------------------------------------------
# --------------------------------- Docker Daily Restart ---------------------------------------
# -----------------------------------------------------------------------------------------------
# Restarts or starts specified Docker containers with retry logic.
# Containers are configured in Master.conf under DAILY_RESTART_CONTAINERS.
# Uses global RETRY_COUNT and SLEEP from Master.conf for retry behaviour.
# Sends notifications on completion or failure via common.sh notify().
# Supports --dry-run to preview what would be restarted without taking action.
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ================================= Docker Daily Restart =======================================
# ==============================================================================================
# Restarts or starts all containers in HOST*_DAILY_RESTART_CONTAINERS.
# Called by daily_sync_maintenance.sh via DAILY_MAINTENANCE_SCRIPTS every night at 1am.
# Can also be run manually for ad hoc restarts.
#
# ── WHY DAILY RESTARTS ────────────────────────────────────────────────────────────────────────
# Some containers degrade over time without a restart:
# Dispatcharr — Live TV scheduler accumulates state and slows down
# NginxProxyManager — connection table grows, occasional stale proxy entries
# Authelia — session cache benefits from periodic clearing
# Daily restart is intentional maintenance, not just housekeeping.
#
# ── BEHAVIOUR ─────────────────────────────────────────────────────────────────────────────────
# Running containers → docker restart (graceful stop + start)
# Stopped containers → left stopped — was down intentionally, do not bring back up
# Missing containers → logged and skipped — not treated as fatal
# Each action uses RETRY_COUNT + SLEEP from master.conf for retry logic.
#
# The "was running → restart, was stopped → leave stopped" rule is consistent
# across the entire ecosystem — container state is always respected.
#
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
# Dependency ordering — containers restart in dependency-safe order using
# HOST*_WATCHDOG_DEPENDENCIES from master_host*.conf. If Authelia depends on
# Mariadb + Redis, those restart first with CONTAINER_DELAY before Authelia starts.
#
# Restart verification — after each restart, container state is checked after a short
# settle period. If the container fails to stay running it is marked as failed and
# a notification is sent rather than silently passing.
#
# Timeout protection — all docker commands are wrapped in a 30 second timeout.
# A hung Docker daemon cannot cause this script to hang indefinitely.
# Timed-out commands are retried per RETRY_COUNT before marking as failed.
#
# ── CONFIGURATION (master_host*.conf) ─────────────────────────────────────────────────────────
# HOST*_DAILY_RESTART_CONTAINERS — list of containers to restart daily
# Set by detect_hosts() alias → DAILY_RESTART_CONTAINERS used by this script
#
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
# RETRY_COUNT — retry attempts before giving up on a container
# SLEEP — seconds between retry attempts
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
# docker_daily_restart.sh — normal restart
# docker_daily_restart.sh --dry-run — preview without restarting
# docker_daily_restart.sh --log — verbose output
# docker_daily_restart.sh --status — show config and exit
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../Master.conf"
source "$SCRIPT_DIR/../common.sh"
source "$SCRIPT_DIR/../load_config.sh"
parse_args "$@"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Setup ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_GEAR Setup ━━━"
@@ -26,7 +66,6 @@ if [[ "$EUID" -ne 0 ]]; then
error "Must be run as root"
exit 1
fi
success "Running as root"
acquire_lock
@@ -36,30 +75,41 @@ if ! command -v docker &>/dev/null; then
notify "Docker daily restart failed — Docker not found on $(hostname)" "Docker Daily Restart" "warning"
exit 1
fi
success "Docker found"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Status ━━━
# -----------------------------------------------------------------------------------------------
# detect_hosts() sets MY_ID and aliases HOST*_DAILY_RESTART_CONTAINERS → DAILY_RESTART_CONTAINERS
detect_hosts
if [[ ${#DAILY_RESTART_CONTAINERS[@]} -eq 0 ]]; then
warn "DAILY_RESTART_CONTAINERS is empty for $MY_ID — nothing to restart"
warn "Check HOST${MY_ID#HOST}_DAILY_RESTART_CONTAINERS in master_host*.conf"
exit 0
fi
# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
echo ""
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_CONTAINERS Containers: ${DAILY_RESTART_CONTAINERS[*]}"
echo "$ICON_RETRY Retries: $RETRY_COUNT"
echo "$ICON_TIME Sleep: ${SLEEP}s between retries"
echo "$ICON_NOTIFY Notifications: unRAID=${NOTIFY_UNRAID:-false} Discord=$([[ -n "${DISCORD_WEBHOOK:-}" ]] && echo enabled || echo disabled)"
echo "$ICON_GEAR Dry Run: $DRY_RUN"
echo "$ICON_RETRY Retries: $RETRY_COUNT"
echo "$ICON_TIME Sleep: ${SLEEP}s between retries"
echo "$ICON_NOTIFY Notify: unRAID=${NOTIFY_UNRAID:-false} Discord=$([[ -n "${MY_DISCORD_WEBHOOK:-}" ]] && echo enabled || echo disabled)"
echo "$ICON_GEAR Dry Run: $DRY_RUN"
echo "━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
fi
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be restarted"
# -----------------------------------------------------------------------------------------------
# FUNCTIONS
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ── FUNCTIONS ─────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Retries a docker command up to RETRY_COUNT times with SLEEP seconds between attempts.
# Usage: retry_docker docker restart ContainerName
retry_docker() {
local attempt=1
@@ -80,9 +130,132 @@ retry_docker() {
return 1
}
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_CONTAINERS Daily Restart ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ── FUNCTIONS ─────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Wraps docker commands with a 30 second timeout.
# Prevents a hung Docker daemon from causing the script to hang indefinitely.
# Usage: docker_cmd docker restart ContainerName
DOCKER_TIMEOUT=30
docker_cmd() {
timeout "$DOCKER_TIMEOUT" "$@"
local exit_code=$?
if [[ "$exit_code" -eq 124 ]]; then
error "Docker command timed out after ${DOCKER_TIMEOUT}s: $*"
return 1
fi
return "$exit_code"
}
# Verifies a container is still running after restart.
# Gives the container a short settle period before checking.
# Returns 0 if running, 1 if crashed or stopped.
RESTART_VERIFY_WAIT=5 # seconds to wait before checking state post-restart
verify_running() {
local container="$1"
sleep "$RESTART_VERIFY_WAIT"
local state
state=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
if [[ "$state" != "true" ]]; then
error "$container failed to stay running after restart — may have crashed"
return 1
fi
return 0
}
# Builds a dependency-safe restart order from DAILY_RESTART_CONTAINERS.
# Containers that are dependencies of others restart first.
# Returns ordered list in ORDERED_RESTART array.
build_restart_order() {
ORDERED_RESTART=()
local remaining=("${DAILY_RESTART_CONTAINERS[@]}")
local placed=()
# First pass — add dependency containers that appear in our list
for container in "${remaining[@]}"; do
[[ -z "$container" ]] && continue
local is_dependency=false
# Check if this container is a dependency of any other in our list
for dep_string in "${WATCHDOG_DEPENDENCIES[@]:-}"; do
if [[ "$dep_string" == *"$container"* ]]; then
is_dependency=true
break
fi
done
# Also check associative array format
for dependent in "${!WATCHDOG_DEPENDENCIES[@]:-}"; do
if [[ "${WATCHDOG_DEPENDENCIES[$dependent]}" == *"$container"* ]]; then
is_dependency=true
break
fi
done
if [[ "$is_dependency" == true ]]; then
# Check not already placed
local already=false
for p in "${placed[@]:-}"; do [[ "$p" == "$container" ]] && already=true && break; done
if [[ "$already" == false ]]; then
ORDERED_RESTART+=("$container")
placed+=("$container")
fi
fi
done
# Second pass — add remaining containers (dependents and independents)
for container in "${remaining[@]}"; do
[[ -z "$container" ]] && continue
local already=false
for p in "${placed[@]:-}"; do [[ "$p" == "$container" ]] && already=true && break; done
if [[ "$already" == false ]]; then
ORDERED_RESTART+=("$container")
placed+=("$container")
fi
done
log "Restart order: ${ORDERED_RESTART[*]}"
}
# Checks if a container is a dependent of the previously restarted container.
# If so, waits CONTAINER_DELAY before restarting to allow dependency to settle.
# Usage: check_dependency_delay "$container" "$last_restarted"
check_dependency_delay() {
local container="$1"
local last="$2"
[[ -z "$last" ]] && return
local deps="${WATCHDOG_DEPENDENCIES[$container]:-}"
if [[ -n "$deps" ]] && [[ "$deps" == *"$last"* ]]; then
info "Waiting ${CONTAINER_DELAY}s — $container depends on $last..."
sleep "$CONTAINER_DELAY"
fi
}
# Retries a docker command up to RETRY_COUNT times with SLEEP seconds between attempts.
# Uses docker_cmd wrapper for timeout protection on each attempt.
# Usage: retry_docker docker restart ContainerName
retry_docker() {
local attempt=1
while [[ "$attempt" -le "$RETRY_COUNT" ]]; do
info "$ICON_RETRY Attempt $attempt of $RETRY_COUNT: $*"
if docker_cmd "$@"; then
success "Succeeded on attempt $attempt"
return 0
else
warn "Attempt $attempt failed"
(( attempt++ ))
[[ "$attempt" -le "$RETRY_COUNT" ]] && sleep "$SLEEP"
fi
done
error "Command failed after $RETRY_COUNT attempts: $*"
return 1
}
# ==============================================================================================
# ━━━ Daily Restart ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_CONTAINERS Daily Restart — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
echo "$ICON_CONTAINERS Containers: ${DAILY_RESTART_CONTAINERS[*]}"
@@ -92,30 +265,49 @@ echo ""
START=$(date +%s)
FAILED=()
RESTARTED=()
STARTED=()
SKIPPED=()
for container in "${DAILY_RESTART_CONTAINERS[@]}"; do
# Build dependency-safe restart order
build_restart_order
echo "$ICON_GEAR Restart order: ${ORDERED_RESTART[*]}"
echo ""
LAST_RESTARTED=""
for container in "${ORDERED_RESTART[@]}"; do
[[ -z "$container" ]] && continue
echo "━━━ $ICON_CONTAINERS $container ━━━"
if ! docker inspect "$container" &>/dev/null; then
error "$container does not exist — skipping"
FAILED+=("$container")
if ! timeout "$DOCKER_TIMEOUT" docker inspect "$container" &>/dev/null; then
warn "$container does not exist — skipping"
echo ""
continue
fi
STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
case "$STATUS" in
true)
echo "$ICON_RUNNING $container is running — restarting..."
# Wait if this container depends on the last one restarted
check_dependency_delay "$container" "$LAST_RESTARTED"
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would restart $container"
RESTARTED+=("$container")
else
if retry_docker docker restart "$container"; then
echo "$ICON_STARTED $container restarted"
RESTARTED+=("$container")
# Verify container stayed running after restart
if verify_running "$container"; then
echo "$ICON_STARTED $container restarted and running ✅"
RESTARTED+=("$container")
LAST_RESTARTED="$container"
else
error "$container restarted but crashed immediately"
notify "$container crashed after restart on $(hostname)" "Docker Daily Restart" "warning"
FAILED+=("$container")
fi
else
error "Failed to restart $container after $RETRY_COUNT attempts"
notify "$container failed to restart on $(hostname)" "Docker Daily Restart" "warning"
@@ -124,20 +316,10 @@ for container in "${DAILY_RESTART_CONTAINERS[@]}"; do
fi
;;
false)
echo "$ICON_NOT_RUNNING $container is stopped — starting..."
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would start $container"
else
if retry_docker docker start "$container"; then
echo "$ICON_STARTED $container started"
STARTED+=("$container")
else
error "Failed to start $container after $RETRY_COUNT attempts"
notify "$container failed to start on $(hostname)" "Docker Daily Restart" "warning"
FAILED+=("$container")
fi
fi
# Container was stopped — leave it stopped
# Intentionally stopped containers are not restarted
echo "$ICON_NOT_RUNNING $container is stopped — skipping (respecting stopped state)"
SKIPPED+=("$container")
;;
*)
error "Unknown status for $container: $STATUS"
@@ -150,20 +332,21 @@ done
END=$(date +%s)
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Summary ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
echo "━━━━━ $ICON_SUMMARY DAILY RESTART SUMMARY ━━━━━"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
[[ ${#RESTARTED[@]} -gt 0 ]] && echo "$ICON_STARTED Restarted: ${RESTARTED[*]}"
[[ ${#STARTED[@]} -gt 0 ]] && echo "$ICON_STARTED Started: ${STARTED[*]}"
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}"
[[ ${#RESTARTED[@]} -gt 0 ]] && echo "$ICON_STARTED Restarted: ${RESTARTED[*]}"
[[ ${#SKIPPED[@]} -gt 0 ]] && echo "$ICON_NOT_RUNNING Skipped: ${SKIPPED[*]} (were stopped)"
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}"
if [[ "$DRY_RUN" == true ]]; then
echo "$ICON_WARN Status: DRY RUN — no changes made"
elif [[ ${#FAILED[@]} -eq 0 ]]; then
echo "$ICON_DONE Status: $ICON_SUCCESS ALL DONE"
notify "Daily restart complete — ${#RESTARTED[@]} restarted, ${#STARTED[@]} started on $(hostname)" "Docker Daily Restart" "normal"
notify "Daily restart complete — ${#RESTARTED[@]} restarted, ${#SKIPPED[@]} skipped (stopped) on $(hostname)" "Docker Daily Restart" "normal"
else
echo "$ICON_ERROR Status: $ICON_ERROR ${#FAILED[@]} container(s) failed"
notify "Daily restart completed with errors on $(hostname) — failed: ${FAILED[*]}" "Docker Daily Restart" "warning"
Docker_Essentials/docker_network_connect.sh
+211 -1340
View File
File diff suppressed because it is too large Load Diff
Docker_Essentials/docker_watchdog.sh
+455 -195
View File
@@ -1,56 +1,93 @@
#!/bin/bash
# -----------------------------------------------------------------------------------------------
# --------------------------------- Docker Watchdog --------------------------------------------
# -----------------------------------------------------------------------------------------------
# Two-tier self-healing container monitoring system — runs continuously as a background process.
# Started by array_start.sh at array start — runs until array stops or SIGTERM received.
# ==============================================================================================
# ================================= Docker Watchdog ============================================
# ==============================================================================================
# Two-tier self-healing container monitoring system.
# Runs continuously as a background process — started by array_start.sh at array start.
# Shuts down cleanly on SIGTERM/SIGINT when array stops.
#
# Tier 1 — Strict monitoring (configured containers only)
# Memory hard limits — immediate restart if exceeded
# CPU thresholds — strike system, restart after CPU_FAIL_LIMIT strikes
# HTTP responsiveness — strike system, restart after RESP_FAIL_LIMIT strikes
# Required containers — must always be running, strike system with skip list
# ── TIER 1 — STRICT MONITORING ────────────────────────────────────────────────────────────────
# Applies only to explicitly configured containers (HOST*_WATCHDOG_CONTAINERS etc.)
#
# Tier 2 — Global health scan (all running containers)
# Unhealthy status — Docker HEALTHCHECK unhealthy → restart
# OOM killed — kernel killed container → restart + notify
# Crash loop detection — RestartCount climbing → notify, critical above limit
# Dead containers — remove and restart
# Unexpected exits — non-zero exit code → restart
# Memory hard limits — immediate restart if container exceeds configured MB ceiling
# Memory soft threshold — warn at SOFT_MEM_THRESHOLD % of hard limit (no restart)
# CPU thresholds — strike system: warn at SOFT_CPU_THRESHOLD, restart after
# CPU_FAIL_LIMIT consecutive strikes at HARD_CPU_THRESHOLD
# HTTP responsiveness — strike system: restart after RESP_FAIL_LIMIT consecutive failures
# Required containers — must always be running; strike system before restart;
# skip list after WATCHDOG_CONTAINER_RESTART_LIMIT in window;
# auto-clears when container recovers
#
# Cross-cutting intelligence:
# Startup grace period — skip restarts while system is still booting
# Dependency ordering — restart database before app
# Restart loop protect — stop restarting after X restarts in X hours → skip list
# Skip list auto-clear — clears when container recovers
# Notification batching — one clean summary per cycle, not one ping per event
# Quiet when healthy — only logs when something needs attention
# Parity awareness — skips restarts during parity check
# ── TIER 2 — GLOBAL HEALTH SCAN ───────────────────────────────────────────────────────────────
# Scans ALL running containers when WATCHDOG_SCAN_ALL=true.
# Containers in WATCHDOG_SCAN_IGNORE are excluded from Tier 2.
#
# Continuous loop:
# Checks run every DOCKER_WATCHDOG_INTERVAL seconds (default 900 = 15min)
# Clean shutdown on SIGTERM/SIGINT — sent by array stop
# Variables scoped per-cycle — no state accumulation between cycles
# Unhealthy status — Docker HEALTHCHECK unhealthy → safe_restart()
# OOM killed — kernel OOM killed → safe_restart() + notify
# OOM state tracked per-session to prevent restart loop
# Crash loop detection — RestartCount climbing → notify; above WATCHDOG_CRASH_LIMIT
# → safe_restart() → skip list if restart limit hit
# Dead containers — safe_restart() via remove + start
# Unexpected exits — non-zero exit code → safe_restart()
#
# State files:
# WATCHDOG_STATE_FILE — strike counts (/tmp — resets on reboot)
# SYS_WATCHDOG_FAILED_FILE — persistent skip list (/boot — survives reboots)
# WATCHDOG_CONTAINER_RESTART_LOG — restart history for loop detection
# ── CROSS-CUTTING INTELLIGENCE ────────────────────────────────────────────────────────────────
# Startup grace period — no restarts for WATCHDOG_STARTUP_GRACE seconds after boot
# Dependency ordering — waits for dependencies before restarting a dependent container
# Restart loop protect — skip list after WATCHDOG_CONTAINER_RESTART_LIMIT in rolling window
# Skip list auto-clear — clears when container is seen running again
# Notification batching — one summary per cycle, not one ping per event
# Parity awareness — skips restart actions during parity check
# Timeout protection — all docker commands wrapped in timeout — daemon hangs cannot
# stall the watchdog and leave containers unmonitored
# Docker daemon check — first check every cycle; hung daemon → strike system →
# restart daemon via /etc/rc.d/rc.docker → verify recovery
# system_watchdog.sh handles escalation if restart fails
# Quiet when healthy — only logs when something needs attention (plus heartbeat)
#
# All configuration in Master.conf under Docker Watchdog section.
# Supports --dry-run and --status.
# -----------------------------------------------------------------------------------------------
# ── STATE FILES ───────────────────────────────────────────────────────────────────────────────
# WATCHDOG_STATE_FILE — strike counts (/tmp — resets on reboot, correct)
# SYS_WATCHDOG_FAILED_FILE — skip list (/boot — survives reboots, intentional)
# WATCHDOG_CONTAINER_RESTART_LOG — restart history for loop detection
#
# ── CONFIGURATION (master_host*.conf) ─────────────────────────────────────────────────────────
# HOST*_WATCHDOG_CONTAINERS — memory hard limits per container
# HOST*_WATCHDOG_CONTAINER_URLS — HTTP health check URLs
# HOST*_WATCHDOG_REQUIRED_CONTAINERS — must always be running
# HOST*_WATCHDOG_SCAN_IGNORE — skip in Tier 2 scan
# HOST*_WATCHDOG_DEPENDENCIES — dependency ordering for restart decisions
# All aliased by detect_hosts() — script uses unprefixed names
#
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
# SOFT_CPU_THRESHOLD / HARD_CPU_THRESHOLD / CPU_FAIL_LIMIT
# SOFT_MEM_THRESHOLD
# RESP_FAIL_LIMIT / CURL_TIMEOUT
# DOCKER_WATCHDOG_INTERVAL
# DOCKER_WATCHDOG_HEARTBEAT / DOCKER_WATCHDOG_HEARTBEAT_HOURS
# WATCHDOG_SCAN_ALL
# WATCHDOG_RESTART_UNHEALTHY / WATCHDOG_RESTART_DEAD / WATCHDOG_RESTART_CRASHED
# WATCHDOG_NOTIFY_OOM / WATCHDOG_NOTIFY_CRASHLOOP
# WATCHDOG_CRASH_LIMIT
# WATCHDOG_STARTUP_GRACE
# WATCHDOG_CONTAINER_RESTART_LIMIT / WATCHDOG_CONTAINER_RESTART_WINDOW
# WATCHDOG_BATCH_NOTIFY
# WATCHDOG_STATE_FILE / SYS_WATCHDOG_FAILED_FILE / WATCHDOG_CONTAINER_RESTART_LOG
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
# docker_watchdog.sh — normal start (continuous loop)
# docker_watchdog.sh --dry-run — preview without restarting
# docker_watchdog.sh --status — show config and exit
# docker_watchdog.sh --log — verbose output
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../Master.conf"
source "$SCRIPT_DIR/../common.sh"
source "$SCRIPT_DIR/../load_config.sh"
parse_args "$@"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Setup — runs once at start ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Setup — runs once at start ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_GEAR Setup ━━━"
@@ -58,74 +95,70 @@ if [[ "$EUID" -ne 0 ]]; then
error "Must be run as root"
exit 1
fi
success "Running as root"
# Continuous mode — skip gracefully if healthy instance already running
acquire_lock "continuous"
# Select correct per-host watchdog lists — done once at startup
# detect_hosts() sets MY_ID and aliases all HOST*_WATCHDOG_* arrays
detect_hosts
if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then
WATCHDOG_REQUIRED_CONTAINERS=("${HOST1_WATCHDOG_REQUIRED_CONTAINERS[@]}")
declare -A WATCHDOG_CONTAINER_URLS
for key in "${!HOST1_WATCHDOG_CONTAINER_URLS[@]}"; do
WATCHDOG_CONTAINER_URLS["$key"]="${HOST1_WATCHDOG_CONTAINER_URLS[$key]}"
done
else
WATCHDOG_REQUIRED_CONTAINERS=("${HOST2_WATCHDOG_REQUIRED_CONTAINERS[@]}")
declare -A WATCHDOG_CONTAINER_URLS
for key in "${!HOST2_WATCHDOG_CONTAINER_URLS[@]}"; do
WATCHDOG_CONTAINER_URLS["$key"]="${HOST2_WATCHDOG_CONTAINER_URLS[$key]}"
done
fi
info "Watchdog running as: $LOCAL_SERVER_NAME"
info "Check interval: ${DOCKER_WATCHDOG_INTERVAL}s"
if ! command -v docker >/dev/null 2>&1; then
error "Docker not found"
error "Docker not found — cannot start watchdog"
exit 1
fi
success "Docker found"
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be restarted"
# Validate unRAID-specific commands used by this script
# If rc.docker is missing or changed, daemon restart will fail — better to know now
validate_unraid_cmd "/etc/rc.d/rc.docker" "" "" "Docker rc.d script" || warn "rc.docker not found — daemon restart unavailable if needed"
validate_unraid_cmd "/usr/local/emhttp/plugins/dynamix/scripts/notify" "" "" "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
# Ensure state files exist
touch "$WATCHDOG_STATE_FILE" "$WATCHDOG_CONTAINER_RESTART_LOG" \
"$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Status ━━━
# -----------------------------------------------------------------------------------------------
# Timeout for all docker commands — prevents hung daemon from stalling the watchdog
DOCKER_TIMEOUT=15
# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
echo ""
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
echo "$ICON_CONTAINERS Watched: ${!WATCHDOG_CONTAINERS[@]}"
echo "$ICON_CONTAINERS Required: ${WATCHDOG_REQUIRED_CONTAINERS[*]}"
echo "$ICON_WATCHDOG Scan all: $WATCHDOG_SCAN_ALL"
echo "$ICON_WATCHDOG Interval: ${DOCKER_WATCHDOG_INTERVAL}s"
echo "$ICON_WATCHDOG Startup grace: ${WATCHDOG_STARTUP_GRACE}s"
echo "$ICON_WATCHDOG Restart limit: $WATCHDOG_CONTAINER_RESTART_LIMIT in ${WATCHDOG_CONTAINER_RESTART_WINDOW}h"
echo "$ICON_WATCHDOG Batch notify: $WATCHDOG_BATCH_NOTIFY"
echo "$ICON_WATCHDOG Ignore list: ${WATCHDOG_SCAN_IGNORE[*]:-none}"
echo "$ICON_GEAR Dry Run: $DRY_RUN"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_CONTAINERS Watched: ${!WATCHDOG_CONTAINERS[*]:-none}"
echo "$ICON_CONTAINERS Required: ${WATCHDOG_REQUIRED_CONTAINERS[*]:-none}"
echo "$ICON_WATCHDOG Scan all: $WATCHDOG_SCAN_ALL"
echo "$ICON_WATCHDOG Ignore: ${WATCHDOG_SCAN_IGNORE[*]:-none}"
echo "$ICON_WATCHDOG Interval: ${DOCKER_WATCHDOG_INTERVAL}s"
echo "$ICON_WATCHDOG Startup grace: ${WATCHDOG_STARTUP_GRACE}s"
echo "$ICON_WATCHDOG Restart limit: $WATCHDOG_CONTAINER_RESTART_LIMIT in ${WATCHDOG_CONTAINER_RESTART_WINDOW}h"
echo "$ICON_WATCHDOG Batch notify: $WATCHDOG_BATCH_NOTIFY"
echo "$ICON_WATCHDOG Docker timeout: ${DOCKER_TIMEOUT}s"
echo "$ICON_GEAR Dry Run: $DRY_RUN"
UPTIME_S=$(awk '{print int($1)}' /proc/uptime)
echo "$ICON_TIME System uptime: $(format_duration $UPTIME_S)"
echo "$ICON_TIME System uptime: $(format_duration $UPTIME_S)"
[[ "$UPTIME_S" -lt "$WATCHDOG_STARTUP_GRACE" ]] && \
warn "Within startup grace period — restarts suppressed"
echo "━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
fi
# -----------------------------------------------------------------------------------------------
# HELPERS — defined once, used every cycle
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ── HELPER FUNCTIONS ──────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Get strike count for a container from state file
get_strikes() {
grep "^${1}:" "${2}" 2>/dev/null | cut -d: -f2 || echo "0"
}
# Set strike count for a container in state file
set_strikes() {
local container="$1" count="$2" file="$3"
if grep -q "^${container}:" "$file" 2>/dev/null; then
@@ -135,10 +168,12 @@ set_strikes() {
fi
}
# Check if container is on the persistent skip list
is_skipped() {
grep -q "^${1}$" "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null
}
# Add container to persistent skip list — manual intervention required to recover
add_to_skip_list() {
local container="$1" reason="$2"
if ! is_skipped "$container"; then
@@ -148,11 +183,13 @@ add_to_skip_list() {
fi
}
# Remove container from skip list — called when container is seen running again
remove_from_skip_list() {
sed -i "/^${1}$/d" "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null
success "$1 removed from skip list — recovered"
warn "$1 recovered — removed from skip list ✅"
}
# Log a restart event to the rolling restart history file
log_restart() {
local container="$1"
local now
@@ -160,12 +197,14 @@ log_restart() {
local cutoff
cutoff=$(date -d "${WATCHDOG_CONTAINER_RESTART_WINDOW} hours ago" '+%Y-%m-%d %H:%M:%S')
echo "${container}|${now}" >> "$WATCHDOG_CONTAINER_RESTART_LOG"
# Trim entries older than the rolling window
local tmp="${WATCHDOG_CONTAINER_RESTART_LOG}.tmp"
awk -F'|' -v cutoff="$cutoff" '$2 >= cutoff' \
"$WATCHDOG_CONTAINER_RESTART_LOG" > "$tmp" && \
mv "$tmp" "$WATCHDOG_CONTAINER_RESTART_LOG"
}
# Get number of times a container was restarted within the rolling window
get_restart_count() {
local container="$1"
local cutoff
@@ -174,13 +213,15 @@ get_restart_count() {
'$1==c && $2>=cutoff' "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | wc -l
}
# Check if all dependencies of a container are currently running.
# Returns 0 if all deps running (or no deps), 1 if any dep is down.
dependencies_satisfied() {
local container="$1"
local deps="${WATCHDOG_DEPENDENCIES[$container]:-}"
[[ -z "$deps" ]] && return 0
for dep in $deps; do
local status
status=$(docker inspect -f '{{.State.Running}}' "$dep" 2>/dev/null)
status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' "$dep" 2>/dev/null)
if [[ "$status" != "true" ]]; then
warn "$container — dependency $dep is not running — skipping restart this cycle"
return 1
@@ -189,8 +230,18 @@ dependencies_satisfied() {
return 0
}
# Safe restart with all guards:
# - Restart loop protection (skip list after limit)
# - Dependency check (don't restart if deps down)
# - Startup grace period (no restarts while booting)
# - Dry run support
# - Timeout protection on docker restart
#
# Returns: 0=restarted, 1=skipped, 2=added to skip list
safe_restart() {
local container="$1" reason="$2"
# Restart loop protection — skip list if over limit
local restart_count
restart_count=$(get_restart_count "$container")
if [[ "$restart_count" -ge "$WATCHDOG_CONTAINER_RESTART_LIMIT" ]]; then
@@ -198,34 +249,42 @@ safe_restart() {
"restarted $restart_count times in ${WATCHDOG_CONTAINER_RESTART_WINDOW}h"
return 2
fi
# Dependency check
dependencies_satisfied "$container" || return 1
# Startup grace period
local uptime_s
uptime_s=$(awk '{print int($1)}' /proc/uptime)
if [[ "$uptime_s" -lt "$WATCHDOG_STARTUP_GRACE" ]]; then
warn "$container — within startup grace period — skipping restart"
warn "$container — within startup grace period (${uptime_s}s < ${WATCHDOG_STARTUP_GRACE}s) — skipping"
return 1
fi
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would restart $container ($reason)"
return 0
fi
info "Restarting $container ($reason) [restart $((restart_count + 1))/$WATCHDOG_CONTAINER_RESTART_LIMIT in window]..."
if docker restart "$container" >/dev/null 2>&1; then
log "Restarting $container ($reason) [restart $((restart_count + 1))/$WATCHDOG_CONTAINER_RESTART_LIMIT in window]"
if timeout "$DOCKER_TIMEOUT" docker restart "$container" >/dev/null 2>&1; then
success "$ICON_STARTED $container restarted"
log_restart "$container"
return 0
else
error "Failed to restart $container"
error "Failed to restart $container (timeout or error)"
return 1
fi
}
# Queue a notification event for batch sending at end of cycle
queue_notify() {
local message="$1" severity="${2:-warning}"
NOTIFY_EVENTS+=("${severity}|${message}")
log "Queued: $message"
}
# Send all queued notifications — one batched summary or individual per event
flush_notify() {
[[ ${#NOTIFY_EVENTS[@]} -eq 0 ]] && return
if [[ "$WATCHDOG_BATCH_NOTIFY" == "true" ]]; then
@@ -251,28 +310,153 @@ flush_notify() {
NOTIFY_EVENTS=()
}
# Returns 0 if parity check is currently running
is_parity_running() {
grep -q "progress" /var/local/emhttp/parity-date.txt 2>/dev/null
}
# -----------------------------------------------------------------------------------------------
# CLEAN SHUTDOWN — trap SIGTERM/SIGINT from array stop
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ── DOCKER DAEMON HEALTH CHECK ────────────────────────────────────────────────────────────────
# ==============================================================================================
# Checks Docker daemon responsiveness at the start of every cycle.
# A hung daemon makes all container operations useless — check first, short-circuit if down.
#
# Strike system:
# Each consecutive failed check adds a strike
# At WATCHDOG_DAEMON_STRIKE_LIMIT → attempt daemon restart via rc.docker
# After restart → wait WATCHDOG_DAEMON_RESTART_WAIT seconds → verify
# If verified → clear strikes, continue cycle ✅
# If still hung → notify critical, skip cycle → system_watchdog.sh escalates from here
#
# Returns: 0 = daemon healthy | 1 = daemon down, skip this cycle
WATCHDOG_DAEMON_STRIKE_LIMIT=3 # consecutive failed checks before restart attempt
WATCHDOG_DAEMON_RESTART_WAIT=30 # seconds to wait after restart before verifying
WATCHDOG_DAEMON_STRIKES=0 # persists across cycles — reset when daemon recovers
WATCHDOG_DAEMON_RESTARTED=false # tracks if we already attempted restart this session
# ==============================================================================================
# ── SYSTEM WATCHDOG COORDINATION ──────────────────────────────────────────────────────────────
# ==============================================================================================
# Reads system_watchdog.sh state file to check if a RAM emergency shutdown is active.
# During RAM emergency: system_watchdog.sh has stopped non-essential containers to free RAM.
# docker_watchdog.sh must not restart them — that would undo the emergency shutdown and
# prevent RAM from recovering, creating an infinite restart/shutdown loop.
#
# Returns:
# 0 = normal — run all checks
# 1 = RAM emergency active — defer container management this cycle
check_system_watchdog_state() {
# Returns 0 = normal operation | 1 = defer, RAM emergency active
local state_file="$SYS_WATCHDOG_STATE_FILE"
# No state file = system_watchdog not running or not yet written — assume normal
[[ ! -f "$state_file" ]] && return 0
local mem_shutdown
mem_shutdown=$(grep "^mem_shutdown_active=" "$state_file" 2>/dev/null | cut -d= -f2)
[[ "$mem_shutdown" != "true" ]] && return 0
# ── Stale state guard ─────────────────────────────────────────────────────────────────────
# If mem_shutdown_active=true but state file hasn't been updated in > 2 hours,
# system_watchdog.sh may have died — don't be silenced forever by a stale flag.
local state_mtime now age_seconds stale_limit=7200 # 2 hours
state_mtime=$(stat -c %Y "$state_file" 2>/dev/null || echo 0)
now=$(date +%s)
age_seconds=$(( now - state_mtime ))
if [[ "$age_seconds" -gt "$stale_limit" ]]; then
warn "mem_shutdown_active=true but state file is ${age_seconds}s old — may be stale"
warn "system_watchdog.sh may not be running — resuming normal container management"
warn "If RAM is still low this will be caught on next system_watchdog.sh cycle"
return 0 # Resume normal — don't defer indefinitely on stale state
fi
return 1 # Defer — RAM emergency confirmed and state is fresh
}
check_docker_daemon() {
# docker info is more definitive than docker ps for daemon health
if timeout "$DOCKER_TIMEOUT" docker info >/dev/null 2>&1; then
if [[ "$WATCHDOG_DAEMON_STRIKES" -gt 0 ]]; then
info "$ICON_STARTED Docker daemon recovered — clearing strikes"
queue_notify "Docker daemon recovered on $(hostname)" "normal"
WATCHDOG_DAEMON_STRIKES=0
WATCHDOG_DAEMON_RESTARTED=false
fi
return 0
fi
WATCHDOG_DAEMON_STRIKES=$(( WATCHDOG_DAEMON_STRIKES + 1 ))
warn "$ICON_WATCHDOG Docker daemon not responding (strike $WATCHDOG_DAEMON_STRIKES/$WATCHDOG_DAEMON_STRIKE_LIMIT)"
if [[ "$WATCHDOG_DAEMON_STRIKES" -lt "$WATCHDOG_DAEMON_STRIKE_LIMIT" ]]; then
warn "Skipping monitoring cycle — waiting for daemon to recover"
return 1
fi
if [[ "$WATCHDOG_DAEMON_RESTARTED" == true ]]; then
error "Docker daemon still unresponsive after restart attempt"
error "system_watchdog.sh will handle further escalation"
queue_notify "Docker daemon hung on $(hostname) — restart failed — manual intervention needed" "critical"
flush_notify
return 1
fi
error "Docker daemon unresponsive — attempting restart"
notify "Docker daemon hung on $(hostname) — attempting restart" "Docker Watchdog" "warning"
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would restart Docker daemon via /etc/rc.d/rc.docker restart"
return 1
fi
# Restart daemon — unRAID uses rc.d scripts, not systemd
WATCHDOG_DAEMON_RESTARTED=true
if /etc/rc.d/rc.docker restart >/dev/null 2>&1; then
info "Docker daemon restart issued — waiting ${WATCHDOG_DAEMON_RESTART_WAIT}s..."
sleep "$WATCHDOG_DAEMON_RESTART_WAIT"
if timeout "$DOCKER_TIMEOUT" docker info >/dev/null 2>&1; then
success "Docker daemon restarted successfully ✅"
notify "Docker daemon restarted successfully on $(hostname)" "Docker Watchdog" "normal"
WATCHDOG_DAEMON_STRIKES=0
WATCHDOG_DAEMON_RESTARTED=false
return 0
else
error "Docker daemon did not recover after restart"
queue_notify "Docker daemon restart failed on $(hostname) — system_watchdog.sh escalating" "critical"
flush_notify
return 1
fi
else
error "Failed to issue Docker daemon restart — /etc/rc.d/rc.docker not found or failed"
queue_notify "Docker daemon restart command failed on $(hostname) — manual intervention needed" "critical"
flush_notify
return 1
fi
}
# ==============================================================================================
# ── CLEAN SHUTDOWN ────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
WATCHDOG_RUNNING=true
cleanup() {
echo ""
info "Docker watchdog received shutdown signal — stopping cleanly"
warn "Docker watchdog received shutdown signal — stopping cleanly"
WATCHDOG_RUNNING=false
exit 0
}
trap cleanup SIGTERM SIGINT
# -----------------------------------------------------------------------------------------------
# ━━━ CONTINUOUS MONITORING LOOP ━━━
# -----------------------------------------------------------------------------------------------
info "Docker watchdog started — checking every ${DOCKER_WATCHDOG_INTERVAL}s"
# ==============================================================================================
# ━━━ Continuous Monitoring Loop ━━━
# ==============================================================================================
info "$ICON_WATCHDOG Docker watchdog started — $MY_ID — checking every ${DOCKER_WATCHDOG_INTERVAL}s"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
CYCLE=0
@@ -281,59 +465,79 @@ while [[ "$WATCHDOG_RUNNING" == true ]]; do
((CYCLE++))
CYCLE_START=$(date +%s)
# Re-source Master.conf each cycle — picks up any config changes without restart
source "$SCRIPT_DIR/../Master.conf"
# ── Re-source config each cycle ──────────────────────────────────────────────────────────
# Picks up config changes (new containers, threshold adjustments) without restart.
# detect_hosts() re-aliases all HOST*_WATCHDOG_* arrays after re-source.
source "$SCRIPT_DIR/../load_config.sh"
detect_hosts
# Rebuild per-host lists after re-source in case they changed
if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then
WATCHDOG_REQUIRED_CONTAINERS=("${HOST1_WATCHDOG_REQUIRED_CONTAINERS[@]}")
for key in "${!HOST1_WATCHDOG_CONTAINER_URLS[@]}"; do
WATCHDOG_CONTAINER_URLS["$key"]="${HOST1_WATCHDOG_CONTAINER_URLS[$key]}"
done
else
WATCHDOG_REQUIRED_CONTAINERS=("${HOST2_WATCHDOG_REQUIRED_CONTAINERS[@]}")
for key in "${!HOST2_WATCHDOG_CONTAINER_URLS[@]}"; do
WATCHDOG_CONTAINER_URLS["$key"]="${HOST2_WATCHDOG_CONTAINER_URLS[$key]}"
done
fi
# Per-cycle variables — cleared each iteration, no accumulation
# ── Per-cycle state — cleared each iteration ──────────────────────────────────────────────
NOTIFY_EVENTS=()
T1_RESTARTS=0
T1_WARNINGS=0
T2_RESTARTS=0
T2_WARNINGS=0
# Rebuild ignore map each cycle in case config was updated
# OOM handled set — tracks containers already handled for OOM this cycle
# Prevents restart loop from OOMKilled flag persisting after restart
declare -A OOM_HANDLED
# Rebuild ignore map each cycle (config may have changed)
declare -A IGNORE_MAP
for c in "${WATCHDOG_SCAN_IGNORE[@]}"; do
for c in "${WATCHDOG_SCAN_IGNORE[@]:-}"; do
[[ -n "$c" ]] && IGNORE_MAP["$c"]=1
done
# Startup grace check
UPTIME_SECONDS=$(awk '{print int($1)}' /proc/uptime)
IN_GRACE_PERIOD=false
if [[ "$UPTIME_SECONDS" -lt "$WATCHDOG_STARTUP_GRACE" ]]; then
IN_GRACE_PERIOD=true
# ── Docker daemon health check — first check every cycle ────────────────────────────────
# If daemon is hung all container operations will fail — check first, skip cycle if down
if ! check_docker_daemon; then
sleep "$DOCKER_WATCHDOG_INTERVAL"
continue
fi
# Parity check — skip restarts during parity to avoid I/O interference
# ── Parity check — skip restarts during parity ───────────────────────────────────────────
if is_parity_running; then
log "Parity check in progress — skipping restart actions this cycle"
sleep "$DOCKER_WATCHDOG_INTERVAL"
continue
fi
# ── TIER 1 — Strict Monitoring ──────────────────────────────────────────────────────────
# ── RAM emergency check — system_watchdog.sh managing containers ───────────────────────────
# If system_watchdog.sh has triggered an emergency RAM shutdown, defer all container
# management this cycle. Docker daemon health checks continue — system still needs
# monitoring even during RAM crisis. Restarts deferred to prevent undoing shutdown.
if ! check_system_watchdog_state; then
MEM_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo)
MEM_GB=$(( MEM_KB / 1024 / 1024 ))
warn "RAM emergency active (${MEM_GB}GB free) — system_watchdog.sh managing containers"
warn "Deferring all container restart logic this cycle"
log "Waiting for RAM to recover above ${SYS_WATCHDOG_MEM_RECOVER_GB}GB before resuming"
sleep "$DOCKER_WATCHDOG_INTERVAL" &
wait $!
continue
fi
# Required containers
# ── Startup grace period ──────────────────────────────────────────────────────────────────
UPTIME_SECONDS=$(awk '{print int($1)}' /proc/uptime)
IN_GRACE_PERIOD=false
[[ "$UPTIME_SECONDS" -lt "$WATCHDOG_STARTUP_GRACE" ]] && IN_GRACE_PERIOD=true
# ==========================================================================================
# ── TIER 1 — Strict Monitoring ────────────────────────────────────────────────────────────
# ==========================================================================================
# ── Required containers ───────────────────────────────────────────────────────────────────
# Must always be running — strike system before restart, skip list after limit
if [[ ${#WATCHDOG_REQUIRED_CONTAINERS[@]} -gt 0 ]]; then
for container in "${WATCHDOG_REQUIRED_CONTAINERS[@]}"; do
[[ -z "$container" ]] && continue
STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \
"$container" 2>/dev/null)
if is_skipped "$container"; then
if [[ "$STATUS" == "true" ]]; then
# Container recovered — remove from skip list and clear its history
remove_from_skip_list "$container"
set_strikes "$container" 0 "$WATCHDOG_STATE_FILE"
sed -i "/^${container}|/d" "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null
@@ -345,6 +549,7 @@ while [[ "$WATCHDOG_RUNNING" == true ]]; do
fi
if [[ "$STATUS" == "true" ]]; then
# Running — clear any strikes
set_strikes "$container" 0 "$WATCHDOG_STATE_FILE"
else
STRIKES=$(get_strikes "$container" "$WATCHDOG_STATE_FILE")
@@ -360,7 +565,7 @@ while [[ "$WATCHDOG_RUNNING" == true ]]; do
0) set_strikes "$container" 0 "$WATCHDOG_STATE_FILE"
((T1_RESTARTS++))
queue_notify "$container was down and restarted on $(hostname)" "warning" ;;
2) : ;;
2) : ;; # Added to skip list — already notified
*) queue_notify "$container failed to restart on $(hostname)" "warning" ;;
esac
fi
@@ -368,9 +573,9 @@ while [[ "$WATCHDOG_RUNNING" == true ]]; do
done
fi
# Memory and CPU monitoring
# ── Memory and CPU monitoring ─────────────────────────────────────────────────────────────
if [[ ${#WATCHDOG_CONTAINERS[@]} -gt 0 ]]; then
STATS=$(docker stats --no-stream \
STATS=$(timeout "$DOCKER_TIMEOUT" docker stats --no-stream \
--format "{{.Name}}|{{.MemUsage}}|{{.CPUPerc}}" 2>/dev/null)
TOTAL_CORES=$(nproc 2>/dev/null || echo 1)
@@ -379,6 +584,7 @@ while [[ "$WATCHDOG_RUNNING" == true ]]; do
CONTAINER_STATS=$(echo "$STATS" | grep "^${container}|" | head -1)
[[ -z "$CONTAINER_STATS" ]] && continue
# ── Memory ────────────────────────────────────────────────────────────────────────
MEM_USAGE=$(echo "$CONTAINER_STATS" | cut -d'|' -f2 | awk '{print $1}')
MEM_UNIT=$(echo "$MEM_USAGE" | grep -oE '[A-Za-z]+')
MEM_VALUE=$(echo "$MEM_USAGE" | grep -oE '[0-9.]+')
@@ -390,34 +596,50 @@ while [[ "$WATCHDOG_RUNNING" == true ]]; do
*) MEM_MB=0 ;;
esac
# Soft memory threshold — warn when approaching hard limit
SOFT_MEM_MB=$(awk "BEGIN {printf \"%.0f\", $MEM_LIMIT_MB * $SOFT_MEM_THRESHOLD / 100}")
if [[ "$MEM_MB" -ge "$MEM_LIMIT_MB" ]]; then
# Hard limit exceeded — immediate restart
error "$container — memory ${MEM_MB}MB exceeded hard limit ${MEM_LIMIT_MB}MB"
safe_restart "$container" "memory hard limit exceeded"
((T1_RESTARTS++))
queue_notify "$container exceeded memory hard limit on $(hostname) — restarted" "warning"
elif [[ "$MEM_MB" -ge "$SOFT_MEM_MB" ]]; then
# Soft threshold — warn only, no restart
warn "$container — memory ${MEM_MB}MB approaching limit (${SOFT_MEM_THRESHOLD}% of ${MEM_LIMIT_MB}MB)"
((T1_WARNINGS++))
fi
# ── CPU ───────────────────────────────────────────────────────────────────────────
CPU_RAW=$(echo "$CONTAINER_STATS" | cut -d'|' -f3 | tr -d '%')
CPU_NORM=$(awk "BEGIN {printf \"%.1f\", $CPU_RAW / $TOTAL_CORES}")
CPU_INT=$(printf "%.0f" "$CPU_NORM")
if [[ "$MEM_MB" -ge "$MEM_LIMIT_MB" ]]; then
error "$container — memory exceeded hard limit ${MEM_LIMIT_MB}MB"
safe_restart "$container" "memory hard limit exceeded"
((T1_RESTARTS++))
queue_notify "$container exceeded memory limit on $(hostname) — restarted" "warning"
fi
if [[ "$CPU_INT" -ge "$HARD_CPU_THRESHOLD" ]]; then
# Hard CPU threshold — strike system → restart
CPU_STRIKES=$(get_strikes "${container}_cpu" "$WATCHDOG_STATE_FILE")
CPU_STRIKES=$(( CPU_STRIKES + 1 ))
set_strikes "${container}_cpu" "$CPU_STRIKES" "$WATCHDOG_STATE_FILE"
warn "$container — CPU ${CPU_NORM}% (strike $CPU_STRIKES/$CPU_FAIL_LIMIT)"
if [[ "$CPU_STRIKES" -ge "$CPU_FAIL_LIMIT" ]]; then
safe_restart "$container" "CPU threshold exceeded"
safe_restart "$container" "CPU hard threshold exceeded ${CPU_NORM}%"
set_strikes "${container}_cpu" 0 "$WATCHDOG_STATE_FILE"
((T1_RESTARTS++))
queue_notify "$container CPU ${CPU_NORM}% on $(hostname) — restarted" "warning"
fi
elif [[ "$CPU_INT" -ge "$SOFT_CPU_THRESHOLD" ]]; then
# Soft CPU threshold — warn only, no restart, clear strikes
warn "$container — CPU ${CPU_NORM}% (above soft threshold ${SOFT_CPU_THRESHOLD}%)"
((T1_WARNINGS++))
set_strikes "${container}_cpu" 0 "$WATCHDOG_STATE_FILE"
else
# Normal — clear CPU strikes
set_strikes "${container}_cpu" 0 "$WATCHDOG_STATE_FILE"
fi
done
fi
# HTTP responsiveness
# ── HTTP responsiveness ───────────────────────────────────────────────────────────────────
if [[ ${#WATCHDOG_CONTAINER_URLS[@]} -gt 0 ]]; then
for container in "${!WATCHDOG_CONTAINER_URLS[@]}"; do
URL="${WATCHDOG_CONTAINER_URLS[$container]}"
@@ -430,102 +652,133 @@ while [[ "$WATCHDOG_RUNNING" == true ]]; do
warn "$container — not responding at $URL (strike $HTTP_STRIKES/$RESP_FAIL_LIMIT)"
((T1_WARNINGS++))
if [[ "$HTTP_STRIKES" -ge "$RESP_FAIL_LIMIT" ]]; then
safe_restart "$container" "HTTP unresponsive"
set_strikes "${container}_http" 0 "$WATCHDOG_STATE_FILE"
((T1_RESTARTS++))
queue_notify "$container unresponsive at $URL on $(hostname) — restarted" "warning"
result=0
safe_restart "$container" "HTTP unresponsive at $URL" || result=$?
if [[ $result -eq 0 ]]; then
set_strikes "${container}_http" 0 "$WATCHDOG_STATE_FILE"
((T1_RESTARTS++))
queue_notify "$container unresponsive at $URL on $(hostname) — restarted" "warning"
fi
fi
fi
done
fi
# ── TIER 2 — Global Health Scan ─────────────────────────────────────────────────────────
# ==========================================================================================
# ── TIER 2 — Global Health Scan ───────────────────────────────────────────────────────────
# ==========================================================================================
if [[ "$WATCHDOG_SCAN_ALL" == "true" ]]; then
ALL_CONTAINERS=$(docker ps --format "{{.Names}}" 2>/dev/null)
# Unhealthy containers
ALL_CONTAINERS=$(timeout "$DOCKER_TIMEOUT" docker ps --format "{{.Names}}" 2>/dev/null)
# ── Unhealthy containers ──────────────────────────────────────────────────────────────
if [[ "$WATCHDOG_RESTART_UNHEALTHY" == "true" ]]; then
UNHEALTHY=$(docker ps --filter health=unhealthy --format "{{.Names}}" 2>/dev/null)
UNHEALTHY=$(timeout "$DOCKER_TIMEOUT" docker ps \
--filter health=unhealthy --format "{{.Names}}" 2>/dev/null)
while IFS= read -r container; do
[[ -z "$container" ]] && continue
[[ -n "${IGNORE_MAP[$container]:-}" ]] && continue
is_skipped "$container" && continue
error "$container — unhealthy"
error "$container — Docker HEALTHCHECK unhealthy"
((T2_WARNINGS++))
result=0
safe_restart "$container" "unhealthy health status" || result=$?
[[ $result -eq 0 ]] && ((T2_RESTARTS++)) && \
if [[ $result -eq 0 ]]; then
((T2_RESTARTS++))
queue_notify "$container unhealthy on $(hostname) — restarted" "warning"
fi
done <<< "$UNHEALTHY"
fi
# OOM killed
# ── OOM killed ────────────────────────────────────────────────────────────────────────
# OOMKilled flag persists after restart — track handled containers per-cycle
# to prevent the same container triggering a restart loop every cycle
if [[ "$WATCHDOG_NOTIFY_OOM" == "true" ]]; then
while IFS= read -r container; do
[[ -z "$container" ]] && continue
[[ -n "${IGNORE_MAP[$container]:-}" ]] && continue
is_skipped "$container" && continue
OOM=$(docker inspect -f '{{.State.OOMKilled}}' "$container" 2>/dev/null)
[[ -n "${OOM_HANDLED[$container]:-}" ]] && continue # already handled this cycle
OOM=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
'{{.State.OOMKilled}}' "$container" 2>/dev/null)
if [[ "$OOM" == "true" ]]; then
error "$container — OOM killed"
error "$container — OOM killed by kernel"
((T2_WARNINGS++))
OOM_HANDLED["$container"]=1
result=0
safe_restart "$container" "OOM killed" || result=$?
[[ $result -eq 0 ]] && ((T2_RESTARTS++)) && \
if [[ $result -eq 0 ]]; then
((T2_RESTARTS++))
queue_notify "$container OOM killed on $(hostname) — restarted" "warning"
fi
done <<< "$ALL_CONTAINERS"
fi
# Crash loop detection
if [[ "$WATCHDOG_NOTIFY_CRASHLOOP" == "true" ]]; then
while IFS= read -r container; do
[[ -z "$container" ]] && continue
[[ -n "${IGNORE_MAP[$container]:-}" ]] && continue
is_skipped "$container" && continue
RESTART_COUNT=$(docker inspect -f '{{.RestartCount}}' "$container" 2>/dev/null || echo 0)
PREV_COUNT=$(grep "^${container}_docker:" "$WATCHDOG_STATE_FILE" 2>/dev/null | cut -d: -f2 || echo 0)
set_strikes "${container}_docker" "$RESTART_COUNT" "$WATCHDOG_STATE_FILE"
if [[ "$RESTART_COUNT" -gt "$PREV_COUNT" && "$RESTART_COUNT" -gt 0 ]]; then
((T2_WARNINGS++))
if [[ "$RESTART_COUNT" -ge "$WATCHDOG_CRASH_LIMIT" ]]; then
error "$container — crash loop CRITICAL: $RESTART_COUNT restarts"
queue_notify "$container crash loop CRITICAL on $(hostname) — manual intervention needed" "critical"
else
warn "$container — restarted since last check (total: $RESTART_COUNT)"
queue_notify "$container restarted on $(hostname) — count: $RESTART_COUNT" "warning"
fi
fi
done <<< "$ALL_CONTAINERS"
fi
# Dead containers
# ── Crash loop detection ──────────────────────────────────────────────────────────────
# Tracks Docker's own RestartCount climbing between cycles.
# Below WATCHDOG_CRASH_LIMIT: notify only — Docker's restart policy is handling it.
# At or above WATCHDOG_CRASH_LIMIT: safe_restart() which will add to skip list
# if WATCHDOG_CONTAINER_RESTART_LIMIT is also hit — ensures eventual quarantine.
if [[ "$WATCHDOG_NOTIFY_CRASHLOOP" == "true" ]]; then
while IFS= read -r container; do
[[ -z "$container" ]] && continue
[[ -n "${IGNORE_MAP[$container]:-}" ]] && continue
is_skipped "$container" && continue
RESTART_COUNT=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
'{{.RestartCount}}' "$container" 2>/dev/null || echo 0)
PREV_COUNT=$(grep "^${container}_docker:" "$WATCHDOG_STATE_FILE" \
2>/dev/null | cut -d: -f2 || echo 0)
set_strikes "${container}_docker" "$RESTART_COUNT" "$WATCHDOG_STATE_FILE"
if [[ "$RESTART_COUNT" -gt "$PREV_COUNT" && "$RESTART_COUNT" -gt 0 ]]; then
((T2_WARNINGS++))
if [[ "$RESTART_COUNT" -ge "$WATCHDOG_CRASH_LIMIT" ]]; then
error "$container — crash loop CRITICAL: $RESTART_COUNT restarts"
# Attempt restart via safe_restart — will add to skip list if over limit
result=0
safe_restart "$container" "crash loop — $RESTART_COUNT restarts" || result=$?
case $result in
0) ((T2_RESTARTS++))
queue_notify "$container crash loop on $(hostname) — restarted" "critical" ;;
2) : ;; # Added to skip list
*) queue_notify "$container crash loop CRITICAL on $(hostname) — manual intervention needed" "critical" ;;
esac
else
warn "$container — restarted since last check (Docker count: $RESTART_COUNT)"
queue_notify "$container restarted on $(hostname) — Docker count: $RESTART_COUNT" "warning"
fi
fi
done <<< "$ALL_CONTAINERS"
fi
# ── Dead containers ───────────────────────────────────────────────────────────────────
# Routes through safe_restart() — ensures restart loop protection applies
if [[ "$WATCHDOG_RESTART_DEAD" == "true" ]]; then
DEAD=$(docker ps -a --filter status=dead --format "{{.Names}}" 2>/dev/null)
DEAD=$(timeout "$DOCKER_TIMEOUT" docker ps -a \
--filter status=dead --format "{{.Names}}" 2>/dev/null)
while IFS= read -r container; do
[[ -z "$container" ]] && continue
[[ -n "${IGNORE_MAP[$container]:-}" ]] && continue
is_skipped "$container" && continue
error "$container — dead"
((T2_WARNINGS++))
RESTART_COUNT=$(get_restart_count "$container")
if [[ "$RESTART_COUNT" -ge "$WATCHDOG_CONTAINER_RESTART_LIMIT" ]]; then
add_to_skip_list "$container" "dead — restarted $RESTART_COUNT times"
elif [[ "$DRY_RUN" == false ]]; then
docker rm "$container" >/dev/null 2>&1
if docker start "$container" >/dev/null 2>&1; then
success "$container removed from dead state and restarted"
log_restart "$container"
((T2_RESTARTS++))
queue_notify "$container was dead on $(hostname) — restarted" "warning"
fi
if [[ "$DRY_RUN" == false ]]; then
timeout "$DOCKER_TIMEOUT" docker rm "$container" >/dev/null 2>&1
fi
result=0
safe_restart "$container" "dead container" || result=$?
if [[ $result -eq 0 ]]; then
((T2_RESTARTS++))
queue_notify "$container was dead on $(hostname) — removed and restarted" "warning"
fi
done <<< "$DEAD"
fi
# Unexpected exits
# ── Unexpected exits ──────────────────────────────────────────────────────────────────
# Only non-zero exit codes — exit 0 is a clean stop, not a crash
# Skips containers already covered by WATCHDOG_REQUIRED_CONTAINERS (handled in Tier 1)
if [[ "$WATCHDOG_RESTART_CRASHED" == "true" ]]; then
CRASHED=$(docker ps -a \
CRASHED=$(timeout "$DOCKER_TIMEOUT" docker ps -a \
--filter status=exited \
--format "{{.Names}}|{{.Status}}" 2>/dev/null | \
grep -v "Exited (0)")
@@ -533,30 +786,34 @@ while [[ "$WATCHDOG_RUNNING" == true ]]; do
[[ -z "$container" ]] && continue
[[ -n "${IGNORE_MAP[$container]:-}" ]] && continue
is_skipped "$container" && continue
SKIP=false
for req in "${WATCHDOG_REQUIRED_CONTAINERS[@]}"; do
[[ "$container" == "$req" ]] && SKIP=true && break
# Skip containers already monitored by required containers (Tier 1)
local already_required=false
for req in "${WATCHDOG_REQUIRED_CONTAINERS[@]:-}"; do
[[ "$container" == "$req" ]] && already_required=true && break
done
[[ "$SKIP" == true ]] && continue
[[ "$already_required" == true ]] && continue
error "$container — $status (unexpected exit)"
((T2_WARNINGS++))
result=0
safe_restart "$container" "unexpected exit" || result=$?
[[ $result -eq 0 ]] && ((T2_RESTARTS++)) && \
safe_restart "$container" "unexpected exit: $status" || result=$?
if [[ $result -eq 0 ]]; then
((T2_RESTARTS++))
queue_notify "$container crashed on $(hostname) ($status) — restarted" "warning"
fi
done <<< "$CRASHED"
fi
fi
# Send notifications if any events this cycle
fi # WATCHDOG_SCAN_ALL
# ── Send notifications ────────────────────────────────────────────────────────────────────
flush_notify
# Only log summary if something happened — quiet when all healthy
# ── Cycle summary — quiet when healthy ───────────────────────────────────────────────────
TOTAL_RESTARTS=$(( T1_RESTARTS + T2_RESTARTS ))
TOTAL_WARNINGS=$(( T1_WARNINGS + T2_WARNINGS ))
CYCLE_END=$(date +%s)
if [[ "$TOTAL_RESTARTS" -gt 0 || "$TOTAL_WARNINGS" -gt 0 ]]; then
CYCLE_END=$(date +%s)
echo ""
echo "━━━ $ICON_WATCHDOG Cycle $CYCLE — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
echo "$ICON_WATCHDOG T1: $T1_RESTARTS restarts / $T1_WARNINGS warnings"
@@ -565,13 +822,16 @@ while [[ "$WATCHDOG_RUNNING" == true ]]; do
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
else
log "Cycle $CYCLE — all healthy ($(date '+%H:%M:%S'))"
# Heartbeat — periodic proof of life
# Heartbeat — periodic proof of life even when everything is healthy
if [[ "${DOCKER_WATCHDOG_HEARTBEAT:-true}" == true ]]; then
HB_SECONDS=$(( ${DOCKER_WATCHDOG_HEARTBEAT_HOURS:-1} * 3600 ))
UPTIME_SECONDS=$(( CYCLE * DOCKER_WATCHDOG_INTERVAL ))
if [[ "$HB_SECONDS" -gt 0 ]] && (( UPTIME_SECONDS % HB_SECONDS < DOCKER_WATCHDOG_INTERVAL )) && [[ "$UPTIME_SECONDS" -gt 0 ]]; then
HB_UPTIME_HR=$(( UPTIME_SECONDS / 3600 ))
info "♥ docker_watchdog alive — ~${HB_UPTIME_HR}hr uptime ($(date '+%H:%M:%S'))"
UPTIME_APPROX=$(( CYCLE * DOCKER_WATCHDOG_INTERVAL ))
if [[ "$HB_SECONDS" -gt 0 ]] && \
(( UPTIME_APPROX % HB_SECONDS < DOCKER_WATCHDOG_INTERVAL )) && \
[[ "$UPTIME_APPROX" -gt 0 ]]; then
HB_UPTIME_HR=$(( UPTIME_APPROX / 3600 ))
info "♥ docker_watchdog alive — $MY_ID — ~${HB_UPTIME_HR}hr uptime ($(date '+%H:%M:%S'))"
fi
fi
fi
Docker_Essentials/docker_weekly_restart.sh
+198 -60
View File
@@ -1,24 +1,64 @@
#!/bin/bash
# -----------------------------------------------------------------------------------------------
# --------------------------------- Docker Weekly Restart --------------------------------------
# -----------------------------------------------------------------------------------------------
# Restarts or starts specified Docker containers with retry logic.
# Containers are configured in Master.conf under WEEKLY_RESTART_CONTAINERS.
# Uses global RETRY_COUNT and SLEEP from Master.conf for retry behaviour.
# Sends notifications on completion or failure via common.sh notify().
# Supports --dry-run to preview what would be restarted without taking action.
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ================================= Docker Weekly Restart ======================================
# ==============================================================================================
# Restarts all running containers in HOST*_WEEKLY_RESTART_CONTAINERS.
# Called by weekly_sync_maintenance.sh via WEEKLY_MAINTENANCE_SCRIPTS every Sunday at 2:30am.
# Can also be run manually for ad hoc weekly restarts.
#
# ── CONTEXT ───────────────────────────────────────────────────────────────────────────────────
# weekly_sync_maintenance.sh stops containers before syncing and restarts them after.
# This script runs AFTER that restart — targeting a different set of less critical services
# that benefit from a weekly restart but don't need to be stopped for the sync itself.
# These containers are typically already running when this script executes.
#
# ── BEHAVIOUR ─────────────────────────────────────────────────────────────────────────────────
# Running containers → docker restart (graceful stop + start)
# Stopped containers → left stopped — was down intentionally, do not bring back up
# Missing containers → logged and skipped — not treated as fatal
# Each action uses RETRY_COUNT + SLEEP from master.conf for retry logic.
#
# The "was running → restart, was stopped → leave stopped" rule is consistent
# across the entire ecosystem — container state is always respected.
#
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
# Dependency ordering — containers restart in dependency-safe order using
# HOST*_WATCHDOG_DEPENDENCIES from master_host*.conf. Dependencies restart
# first with CONTAINER_DELAY before their dependents.
#
# Restart verification — after each restart, container state is checked after a short
# settle period. If the container fails to stay running it is marked as failed and
# a notification is sent rather than silently passing.
#
# Timeout protection — all docker commands are wrapped in a 30 second timeout.
# A hung Docker daemon cannot cause this script to hang indefinitely.
#
# ── CONFIGURATION (master_host*.conf) ─────────────────────────────────────────────────────────
# HOST*_WEEKLY_RESTART_CONTAINERS — list of containers to restart weekly
# HOST*_WATCHDOG_DEPENDENCIES — dependency ordering for restart sequence
# Set by detect_hosts() alias → WEEKLY_RESTART_CONTAINERS used by this script
#
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
# RETRY_COUNT — retry attempts before giving up on a container
# SLEEP — seconds between retry attempts
# CONTAINER_DELAY — seconds to wait between dependency and dependent restart
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
# docker_weekly_restart.sh — normal restart
# docker_weekly_restart.sh --dry-run — preview without restarting
# docker_weekly_restart.sh --log — verbose output
# docker_weekly_restart.sh --status — show config and exit
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../Master.conf"
source "$SCRIPT_DIR/../common.sh"
source "$SCRIPT_DIR/../load_config.sh"
parse_args "$@"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Setup ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_GEAR Setup ━━━"
@@ -26,7 +66,6 @@ if [[ "$EUID" -ne 0 ]]; then
error "Must be run as root"
exit 1
fi
success "Running as root"
acquire_lock
@@ -36,19 +75,28 @@ if ! command -v docker &>/dev/null; then
notify "Docker weekly restart failed — Docker not found on $(hostname)" "Docker Weekly Restart" "warning"
exit 1
fi
success "Docker found"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Status ━━━
# -----------------------------------------------------------------------------------------------
# detect_hosts() sets MY_ID and aliases HOST*_WEEKLY_RESTART_CONTAINERS → WEEKLY_RESTART_CONTAINERS
detect_hosts
if [[ ${#WEEKLY_RESTART_CONTAINERS[@]} -eq 0 ]]; then
warn "WEEKLY_RESTART_CONTAINERS is empty for $MY_ID — nothing to restart"
warn "Check HOST*_WEEKLY_RESTART_CONTAINERS in master_host*.conf"
exit 0
fi
# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
echo ""
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_CONTAINERS Containers: ${WEEKLY_RESTART_CONTAINERS[*]}"
echo "$ICON_RETRY Retries: $RETRY_COUNT"
echo "$ICON_TIME Sleep: ${SLEEP}s between retries"
echo "$ICON_NOTIFY Notifications: unRAID=${NOTIFY_UNRAID:-false} Discord=$([[ -n "${DISCORD_WEBHOOK:-}" ]] && echo enabled || echo disabled)"
echo "$ICON_NOTIFY Notify: unRAID=${NOTIFY_UNRAID:-false} Discord=$([[ -n "${MY_DISCORD_WEBHOOK:-}" ]] && echo enabled || echo disabled)"
echo "$ICON_GEAR Dry Run: $DRY_RUN"
echo "━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
@@ -56,19 +104,32 @@ fi
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be restarted"
# -----------------------------------------------------------------------------------------------
# FUNCTIONS
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ── FUNCTIONS ─────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Attempts a docker command up to RETRY_COUNT times with SLEEP seconds between attempts.
# Returns 0 on success, 1 if all attempts fail.
# Wraps docker commands with a 30 second timeout.
# Prevents a hung Docker daemon from causing the script to hang indefinitely.
DOCKER_TIMEOUT=30
docker_cmd() {
timeout "$DOCKER_TIMEOUT" "$@"
local exit_code=$?
if [[ "$exit_code" -eq 124 ]]; then
error "Docker command timed out after ${DOCKER_TIMEOUT}s: $*"
return 1
fi
return "$exit_code"
}
# Retries a docker command up to RETRY_COUNT times with SLEEP seconds between attempts.
# Uses docker_cmd wrapper for timeout protection on each attempt.
retry_docker() {
local attempt=1
while [[ "$attempt" -le "$RETRY_COUNT" ]]; do
info "$ICON_RETRY Attempt $attempt of $RETRY_COUNT: $*"
if "$@"; then
if docker_cmd "$@"; then
success "Succeeded on attempt $attempt"
return 0
else
@@ -82,9 +143,77 @@ retry_docker() {
return 1
}
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_CONTAINERS Weekly Restart ━━━
# -----------------------------------------------------------------------------------------------
# Verifies a container is still running after restart.
# Gives the container a short settle period before checking.
RESTART_VERIFY_WAIT=5
verify_running() {
local container="$1"
sleep "$RESTART_VERIFY_WAIT"
local state
state=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
if [[ "$state" != "true" ]]; then
error "$container failed to stay running after restart — may have crashed"
return 1
fi
return 0
}
# Builds a dependency-safe restart order from WEEKLY_RESTART_CONTAINERS.
# Containers that are dependencies of others restart first.
build_restart_order() {
ORDERED_RESTART=()
local remaining=("${WEEKLY_RESTART_CONTAINERS[@]}")
local placed=()
# First pass — add dependency containers that appear in our list
for container in "${remaining[@]}"; do
[[ -z "$container" ]] && continue
local is_dependency=false
for dependent in "${!WATCHDOG_DEPENDENCIES[@]:-}"; do
if [[ "${WATCHDOG_DEPENDENCIES[$dependent]}" == *"$container"* ]]; then
is_dependency=true
break
fi
done
if [[ "$is_dependency" == true ]]; then
local already=false
for p in "${placed[@]:-}"; do [[ "$p" == "$container" ]] && already=true && break; done
if [[ "$already" == false ]]; then
ORDERED_RESTART+=("$container")
placed+=("$container")
fi
fi
done
# Second pass — add remaining containers (dependents and independents)
for container in "${remaining[@]}"; do
[[ -z "$container" ]] && continue
local already=false
for p in "${placed[@]:-}"; do [[ "$p" == "$container" ]] && already=true && break; done
if [[ "$already" == false ]]; then
ORDERED_RESTART+=("$container")
placed+=("$container")
fi
done
log "Restart order: ${ORDERED_RESTART[*]}"
}
# Waits CONTAINER_DELAY if this container depends on the last restarted one.
check_dependency_delay() {
local container="$1"
local last="$2"
[[ -z "$last" ]] && return
local deps="${WATCHDOG_DEPENDENCIES[$container]:-}"
if [[ -n "$deps" ]] && [[ "$deps" == *"$last"* ]]; then
info "Waiting ${CONTAINER_DELAY}s — $container depends on $last..."
sleep "$CONTAINER_DELAY"
fi
}
# ==============================================================================================
# ━━━ Weekly Restart ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_CONTAINERS Weekly Restart — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
echo "$ICON_CONTAINERS Containers: ${WEEKLY_RESTART_CONTAINERS[*]}"
@@ -94,30 +223,48 @@ echo ""
START=$(date +%s)
FAILED=()
RESTARTED=()
STARTED=()
SKIPPED=()
for container in "${WEEKLY_RESTART_CONTAINERS[@]}"; do
# Build dependency-safe restart order
build_restart_order
echo "$ICON_GEAR Restart order: ${ORDERED_RESTART[*]}"
echo ""
LAST_RESTARTED=""
for container in "${ORDERED_RESTART[@]}"; do
[[ -z "$container" ]] && continue
echo "━━━ $ICON_CONTAINERS $container ━━━"
if ! docker inspect "$container" &>/dev/null; then
error "$container does not exist — skipping"
FAILED+=("$container")
if ! timeout "$DOCKER_TIMEOUT" docker inspect "$container" &>/dev/null; then
warn "$container does not exist — skipping"
echo ""
continue
fi
STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
case "$STATUS" in
true)
echo "$ICON_RUNNING $container is running — restarting..."
# Wait if this container depends on the last one restarted
check_dependency_delay "$container" "$LAST_RESTARTED"
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would restart $container"
RESTARTED+=("$container")
else
if retry_docker docker restart "$container"; then
echo "$ICON_STARTED $container restarted"
RESTARTED+=("$container")
if verify_running "$container"; then
echo "$ICON_STARTED $container restarted and running ✅"
RESTARTED+=("$container")
LAST_RESTARTED="$container"
else
error "$container restarted but crashed immediately"
notify "$container crashed after restart on $(hostname)" "Docker Weekly Restart" "warning"
FAILED+=("$container")
fi
else
error "Failed to restart $container after $RETRY_COUNT attempts"
notify "$container failed to restart on $(hostname)" "Docker Weekly Restart" "warning"
@@ -126,20 +273,10 @@ for container in "${WEEKLY_RESTART_CONTAINERS[@]}"; do
fi
;;
false)
echo "$ICON_NOT_RUNNING $container is stopped — starting..."
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would start $container"
else
if retry_docker docker start "$container"; then
echo "$ICON_STARTED $container started"
STARTED+=("$container")
else
error "Failed to start $container after $RETRY_COUNT attempts"
notify "$container failed to start on $(hostname)" "Docker Weekly Restart" "warning"
FAILED+=("$container")
fi
fi
# Container was stopped — leave it stopped
# Intentionally stopped containers are not restarted
echo "$ICON_NOT_RUNNING $container is stopped — skipping (respecting stopped state)"
SKIPPED+=("$container")
;;
*)
error "Unknown status for $container: $STATUS"
@@ -152,25 +289,26 @@ done
END=$(date +%s)
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Summary ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
echo "━━━━━ $ICON_SUMMARY WEEKLY RESTART SUMMARY ━━━━━"
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
[[ ${#RESTARTED[@]} -gt 0 ]] && echo "$ICON_STARTED Restarted: ${RESTARTED[*]}"
[[ ${#STARTED[@]} -gt 0 ]] && echo "$ICON_STARTED Started: ${STARTED[*]}"
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
[[ ${#RESTARTED[@]} -gt 0 ]] && echo "$ICON_STARTED Restarted: ${RESTARTED[*]}"
[[ ${#SKIPPED[@]} -gt 0 ]] && echo "$ICON_NOT_RUNNING Skipped: ${SKIPPED[*]} (were stopped)"
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}"
if [[ "$DRY_RUN" == true ]]; then
echo "$ICON_WARN Status: DRY RUN — no changes made"
elif [[ ${#FAILED[@]} -eq 0 ]]; then
echo "$ICON_DONE Status: $ICON_SUCCESS ALL DONE"
notify "Weekly restart complete — ${#RESTARTED[@]} restarted, ${#STARTED[@]} started on $(hostname)" "Docker Weekly Restart" "normal"
notify "Weekly restart complete — ${#RESTARTED[@]} restarted, ${#SKIPPED[@]} skipped (stopped) on $(hostname)" "Docker Weekly Restart" "normal"
else
echo "$ICON_ERROR Status: $ICON_ERROR ${#FAILED[@]} container(s) failed"
notify "Weekly restart completed with errors on $(hostname) — failed: ${FAILED[*]}" "Docker Weekly Restart" "warning"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
[[ ${#FAILED[@]} -gt 0 ]] && exit 1
exit 0
Docker_Essentials/downloaders_reset.sh
+157 -92
View File
@@ -1,40 +1,70 @@
#!/bin/bash
# -----------------------------------------------------------------------------------------------
# --------------------------------- Downloaders Reset ------------------------------------------
# -----------------------------------------------------------------------------------------------
# Daily maintenance reset for all download clients.
# Clears stuck states, purges old history, and prepares downloaders for a clean daily cycle.
# Run before container restarts in daily_sync_maintenance.sh via MEDIA_MANAGEMENT_JOBS.
# ==============================================================================================
# ================================= Downloaders Reset ==========================================
# ==============================================================================================
# Maintenance reset for all download clients on this server.
# Called every 15 minutes by critical_sync_maintenance.sh via CRITICAL_MAINTENANCE_SCRIPTS.
# Can also be run manually for ad hoc cleanup.
#
# Downloaders covered:
# slskd — clears stuck/errored searches, dead transfer records,
# purges expired failed imports (albums Lidarr rejected)
# SABnzbd — clears completed and failed history older than retention period,
# removes stalled/paused queue items
# qBittorrent — last chance failsafe delete for torrents older than
# QBIT_FAILSAFE_MIN_DAYS regardless of ratio
# ── DOWNLOADERS COVERED ───────────────────────────────────────────────────────────────────────
# slskd
# Stuck searches — clears Completed/Errored searches left by Soularr crashes
# prevents 409 Conflict on next Soularr startup
# Dead transfers — removes completed/errored/aborted transfer records per user
# prevents Soularr 404 loop when polling a user whose transfer is gone
# NEVER removes InProgress or Queued transfers
# Failed imports — purges albums Soularr downloaded but Lidarr rejected
# Soularr moves these to failed_imports/ and never cleans them up
#
# Safety:
# Always --dry-run first before scheduling
# slskd transfer cleanup skips any user with InProgress or Queued transfers
# qBittorrent only deletes if torrent age exceeds QBIT_FAILSAFE_MIN_DAYS
# SABnzbd only deletes history older than DOWNLOADER_RETENTION_DAYS
# qBittorrent deleteFiles=false — removes from qBit, leaves files for arrs to manage
# SABnzbd
# Completed history — removes completed download records older than DOWNLOADER_RETENTION_DAYS
# Failed history — removes failed download records older than DOWNLOADER_RETENTION_DAYS
# Stalled queue — removes Paused or Stuck queue items no longer progressing
# active downloading items are never touched
#
# Configuration in Master.conf under Docker Essentials — Downloaders Reset.
# Supports --dry-run to preview without making changes.
# -----------------------------------------------------------------------------------------------
# qBittorrent
# Age failsafe — removes torrents older than QBIT_FAILSAFE_MIN_DAYS
# deleteFiles=false — removes from qBit, leaves files for arrs to manage
# optional ratio requirement via QBIT_FAILSAFE_MIN_RATIO
#
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
# detect_hosts() sets MY_ID and aliases all HOST*_SLSKD_*, HOST*_SABNZBD_*, HOST*_QBIT_* vars.
# If a downloader URL is empty for this host — that section is skipped with a clear message.
# HOST2 currently has no downloaders configured — all sections skip cleanly on HOST2.
#
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
# slskd — skips users with InProgress or Queued transfers — never interrupts active downloads
# SABnzbd — age check before deletion — only removes items past retention threshold
# qBittorrent — age + optional ratio check — failsafe only removes old completed torrents
# All sections — skip gracefully if downloader is unreachable, no fatal exit
# acquire_lock "wait" — if previous run still active, waits briefly then exits cleanly
#
# ── CONFIGURATION (master_host*.conf) ─────────────────────────────────────────────────────────
# HOST*_SLSKD_URL / HOST*_SLSKD_API_KEY / HOST*_SLSKD_FAILED_IMPORTS_DIR
# HOST*_SABNZBD_URL / HOST*_SABNZBD_API_KEY
# HOST*_QBIT_URL / HOST*_QBIT_USERNAME / HOST*_QBIT_PASSWORD
#
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
# DOWNLOADER_RETENTION_DAYS — days before history entries are purged
# QBIT_FAILSAFE_MIN_DAYS — minimum torrent age before failsafe deletion
# QBIT_FAILSAFE_MIN_RATIO — minimum ratio requirement (0 = age only)
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
# downloaders_reset.sh — normal reset
# downloaders_reset.sh --dry-run — preview without making changes
# downloaders_reset.sh --log — verbose output
# downloaders_reset.sh --status — show config and exit
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../Master.conf"
source "$SCRIPT_DIR/../common.sh"
source "$SCRIPT_DIR/../load_config.sh"
parse_args "$@"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Setup ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_GEAR Downloaders Reset ━━━"
@@ -43,45 +73,54 @@ if [[ "$EUID" -ne 0 ]]; then
exit 1
fi
detect_hosts
# Lock first — wait mode since this runs every 15min and previous may still be finishing
acquire_lock "wait"
# detect_hosts() sets MY_ID and aliases all HOST*_SLSKD_*, HOST*_SABNZBD_*, HOST*_QBIT_* vars
detect_hosts
START_TIME=$(date +%s)
# Select host-specific config
if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then
SLSKD_URL="$HOST1_SLSKD_URL"
SLSKD_API_KEY="$HOST1_SLSKD_API_KEY"
SLSKD_FAILED_IMPORTS_DIR="$HOST1_SLSKD_FAILED_IMPORTS_DIR"
SABNZBD_URL="$HOST1_SABNZBD_URL"
SABNZBD_API_KEY="$HOST1_SABNZBD_API_KEY"
QBIT_URL="$HOST1_QBIT_URL"
QBIT_USERNAME="$HOST1_QBIT_USERNAME"
QBIT_PASSWORD="$HOST1_QBIT_PASSWORD"
else
# HOST2 placeholders — fill in when HOST2 is back online
SLSKD_URL="${HOST2_SLSKD_URL:-}"
SLSKD_API_KEY="${HOST2_SLSKD_API_KEY:-}"
SLSKD_FAILED_IMPORTS_DIR="${HOST2_SLSKD_FAILED_IMPORTS_DIR:-}"
SABNZBD_URL="${HOST2_SABNZBD_URL:-}"
SABNZBD_API_KEY="${HOST2_SABNZBD_API_KEY:-}"
QBIT_URL="${HOST2_QBIT_URL:-}"
QBIT_USERNAME="${HOST2_QBIT_USERNAME:-}"
QBIT_PASSWORD="${HOST2_QBIT_PASSWORD:-}"
fi
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
CUTOFF=$(( $(date +%s) - (DOWNLOADER_RETENTION_DAYS * 86400) ))
TOTAL_PASS=0
TOTAL_FAIL=0
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
echo ""
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_GEAR slskd: ${SLSKD_URL:-not configured}"
echo "$ICON_GEAR SABnzbd: ${SABNZBD_URL:-not configured}"
echo "$ICON_GEAR qBittorrent: ${QBIT_URL:-not configured}"
echo "$ICON_TIME Retention: ${DOWNLOADER_RETENTION_DAYS} days"
echo "$ICON_GEAR qBit age: ${QBIT_FAILSAFE_MIN_DAYS} days"
echo "$ICON_GEAR qBit ratio: ${QBIT_FAILSAFE_MIN_RATIO} (0=age only)"
echo "$ICON_NOTIFY Notify: unRAID=${NOTIFY_UNRAID:-false} Discord=$([[ -n "${MY_DISCORD_WEBHOOK:-}" ]] && echo enabled || echo disabled)"
echo "$ICON_GEAR Dry Run: $DRY_RUN"
echo "━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
fi
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
# Log which downloaders are active on this host
if [[ -z "$SLSKD_URL" ]] && [[ -z "$SABNZBD_URL" ]] && [[ -z "$QBIT_URL" ]]; then
warn "No downloaders configured for $MY_ID — nothing to reset"
exit 0
fi
[[ -n "$SLSKD_URL" ]] && log "slskd active on $MY_ID"
[[ -n "$SABNZBD_URL" ]] && log "SABnzbd active on $MY_ID"
[[ -n "$QBIT_URL" ]] && log "qBittorrent active on $MY_ID"
# ==============================================================================================
# ━━━ slskd — Stuck Searches ━━━
# Clears searches in Completed/Errored state left by Soularr crashes
# Prevents 409 Conflict error on next Soularr startup
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# Clears searches in Completed/Errored state left by Soularr crashes.
# Prevents 409 Conflict error on next Soularr startup when it tries to
# create a search with the same ID that already exists in a terminal state.
if [[ -n "$SLSKD_URL" ]] && [[ -n "$SLSKD_API_KEY" ]]; then
echo ""
echo "━━━ 🔍 slskd — Stuck Searches ━━━"
@@ -127,12 +166,13 @@ if [[ -n "$SLSKD_URL" ]] && [[ -n "$SLSKD_API_KEY" ]]; then
fi
fi
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ slskd — Dead Transfer Records ━━━
# Removes completed/errored/aborted transfer records per user
# Prevents Soularr 404 loop when polling a user whose transfer no longer exists
# NEVER removes transfers that are InProgress or Queued
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# Removes completed/errored/aborted transfer records per user.
# Prevents Soularr 404 loop when polling a user whose transfer no longer exists.
# Safety: NEVER removes transfers that are InProgress or Queued — active downloads protected.
if [[ -n "$SLSKD_URL" ]] && [[ -n "$SLSKD_API_KEY" ]]; then
echo ""
echo "━━━ 🔍 slskd — Dead Transfer Records ━━━"
@@ -152,6 +192,7 @@ if [[ -n "$SLSKD_URL" ]] && [[ -n "$SLSKD_API_KEY" ]]; then
SUCCESS=0; SKIPPED=0; FAIL=0
while IFS= read -r USER; do
[[ -z "$USER" ]] && continue
# Skip users with any active or queued transfers — never interrupt downloads
ACTIVE=$(echo "$TRANSFERS" | grep -o "\"username\":\"$USER\"[^}]*\"state\":\"[^\"]*\"" | \
grep -c "InProgress\|Queued")
if [[ "$ACTIVE" -gt 0 ]]; then
@@ -182,11 +223,13 @@ if [[ -n "$SLSKD_URL" ]] && [[ -n "$SLSKD_API_KEY" ]]; then
fi
fi
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ slskd — Purge Expired Failed Imports ━━━
# Removes albums Soularr downloaded but Lidarr rejected
# Soularr moves these to failed_imports/ and never cleans them up
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# Removes albums Soularr downloaded but Lidarr rejected.
# Soularr moves rejected albums to failed_imports/ and never cleans them up.
# Purges directories older than DOWNLOADER_RETENTION_DAYS to prevent unbounded growth.
if [[ -n "$SLSKD_FAILED_IMPORTS_DIR" ]]; then
echo ""
echo "━━━ 🔍 slskd — Failed Imports (older than ${DOWNLOADER_RETENTION_DAYS} days) ━━━"
@@ -217,10 +260,12 @@ if [[ -n "$SLSKD_FAILED_IMPORTS_DIR" ]]; then
fi
fi
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ SABnzbd — Clear Completed History ━━━
# Removes completed download history older than DOWNLOADER_RETENTION_DAYS
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# Removes completed download history older than DOWNLOADER_RETENTION_DAYS.
# Keeps recent history for reference — only purges what's past the retention window.
if [[ -n "$SABNZBD_URL" ]] && [[ -n "$SABNZBD_API_KEY" ]]; then
echo ""
echo "━━━ 🔍 SABnzbd — Completed History (older than ${DOWNLOADER_RETENTION_DAYS} days) ━━━"
@@ -261,10 +306,12 @@ if [[ -n "$SABNZBD_URL" ]] && [[ -n "$SABNZBD_API_KEY" ]]; then
fi
fi
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ SABnzbd — Clear Failed History ━━━
# Removes failed download history older than DOWNLOADER_RETENTION_DAYS
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# Removes failed download history older than DOWNLOADER_RETENTION_DAYS.
# Failed history is kept briefly for diagnosis but purged after the retention window.
if [[ -n "$SABNZBD_URL" ]] && [[ -n "$SABNZBD_API_KEY" ]]; then
echo ""
echo "━━━ 🔍 SABnzbd — Failed History (older than ${DOWNLOADER_RETENTION_DAYS} days) ━━━"
@@ -305,11 +352,14 @@ if [[ -n "$SABNZBD_URL" ]] && [[ -n "$SABNZBD_API_KEY" ]]; then
fi
fi
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ SABnzbd — Remove Stalled Queue Items ━━━
# Removes paused queue items no longer progressing
# Active downloading items are never touched
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# Removes queue items in Paused or Stuck state that are no longer progressing.
# Active downloading items (Downloading, Grabbing) are never touched.
# Paused items may be intentional pauses — but in an automated environment
# a Paused item sitting in the queue indefinitely is effectively stalled.
if [[ -n "$SABNZBD_URL" ]] && [[ -n "$SABNZBD_API_KEY" ]]; then
echo ""
echo "━━━ 🔍 SABnzbd — Stalled Queue Items ━━━"
@@ -331,15 +381,19 @@ if [[ -n "$SABNZBD_URL" ]] && [[ -n "$SABNZBD_API_KEY" ]]; then
[[ -z "$NZO_ID" ]] && continue
STATUS=$(echo "$QUEUE" | grep -A10 "$NZO_ID" | \
grep -o '"status":"[^"]*"' | sed 's/"status":"//;s/"//')
[[ "$STATUS" != "Paused" ]] && ((SKIPPED++)) && continue
# Only remove Paused or Stuck items — Downloading/Grabbing are active
if [[ "$STATUS" != "Paused" ]] && [[ "$STATUS" != "Stuck" ]]; then
((SKIPPED++))
continue
fi
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would remove stalled item: $NZO_ID"
warn "DRY RUN — would remove stalled item: $NZO_ID ($STATUS)"
((DELETED++))
else
curl -sf --max-time 10 \
"$SABNZBD_URL/api?mode=queue&name=delete&value=$NZO_ID&apikey=$SABNZBD_API_KEY" \
>/dev/null
info "$ICON_TRASH Removed stalled: $NZO_ID"
info "$ICON_TRASH Removed stalled ($STATUS): $NZO_ID"
((DELETED++))
fi
done <<< "$STALLED_IDS"
@@ -349,11 +403,17 @@ if [[ -n "$SABNZBD_URL" ]] && [[ -n "$SABNZBD_API_KEY" ]]; then
fi
fi
# -----------------------------------------------------------------------------------------------
# ━━━ qBittorrent — Last Chance Failsafe Cleanup ━━━
# Deletes torrents older than QBIT_FAILSAFE_MIN_DAYS
# deleteFiles=false — removes from qBit, leaves files for arrs to manage
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ qBittorrent — Age Failsafe Cleanup ━━━
# ==============================================================================================
# Last-chance cleanup for torrents that have been sitting in qBit past their useful life.
# deleteFiles=false — removes the torrent record from qBit but leaves files on disk.
# Radarr/Sonarr manage actual files independently — this only cleans up the qBit entry.
#
# Safety checks before deletion:
# Age must exceed QBIT_FAILSAFE_MIN_DAYS
# Ratio must meet QBIT_FAILSAFE_MIN_RATIO (0 = age only, no ratio requirement)
if [[ -n "$QBIT_URL" ]] && [[ -n "$QBIT_USERNAME" ]]; then
echo ""
echo "━━━ 🔍 qBittorrent — Failsafe (older than ${QBIT_FAILSAFE_MIN_DAYS} days) ━━━"
@@ -366,7 +426,8 @@ if [[ -n "$QBIT_URL" ]] && [[ -n "$QBIT_USERNAME" ]]; then
grep SID | awk '{print "SID="$NF}')
if [[ -z "$QBIT_COOKIE" ]]; then
error "Failed to authenticate with qBittorrent"
error "Failed to authenticate with qBittorrent — check QBIT_USERNAME/PASSWORD"
notify "qBittorrent auth failed on $(hostname) — check credentials in master_host*.conf" "Downloaders Reset" "warning"
((TOTAL_FAIL++))
else
TORRENTS=$(curl -sf --max-time 15 \
@@ -385,8 +446,11 @@ if [[ -n "$QBIT_URL" ]] && [[ -n "$QBIT_USERNAME" ]]; then
[[ -z "$HASH" || -z "$ADDED" ]] && continue
AGE_DAYS=$(( (NOW - ADDED) / 86400 ))
# Age check — must be old enough
[[ "$AGE_DAYS" -lt "$QBIT_FAILSAFE_MIN_DAYS" ]] && ((SKIPPED++)) && continue
# Ratio check — if configured
if [[ "$QBIT_FAILSAFE_MIN_RATIO" != "0" ]]; then
RATIO_INT="${RATIO%.*}"
MIN_RATIO_INT="${QBIT_FAILSAFE_MIN_RATIO%.*}"
@@ -411,14 +475,15 @@ if [[ -n "$QBIT_URL" ]] && [[ -n "$QBIT_USERNAME" ]]; then
fi
fi
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Summary ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
echo ""
echo "━━━━━ $ICON_SUMMARY DOWNLOADERS RESET SUMMARY ━━━━━"
echo "$ICON_TIME Duration: $(format_duration $(( $(date +%s) - START_TIME )))"
echo "$ICON_SUCCESS Actions: $TOTAL_PASS"
echo "$ICON_ERROR Failures: $TOTAL_FAIL"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_TIME Duration: $(format_duration $(( $(date +%s) - START_TIME )))"
echo "$ICON_SUCCESS Actions: $TOTAL_PASS"
echo "$ICON_ERROR Failures: $TOTAL_FAIL"
echo ""
if [[ "$DRY_RUN" == true ]]; then
Failover/README-Failover.md
+1051 -437
View File
File diff suppressed because it is too large Load Diff
Failover/failover.sh
+336 -287
View File
File diff suppressed because it is too large Load Diff
Failover/failover_test.sh
+199 -135
View File
@@ -1,47 +1,71 @@
#!/bin/bash
# -----------------------------------------------------------------------------------------------
# --------------------------------- Failover Test ----------------------------------------------
# -----------------------------------------------------------------------------------------------
# Controlled simulation of the failover scenario — validates the entire failover lifecycle
# ==============================================================================================
# ================================= Failover Test ==============================================
# ==============================================================================================
# Controlled simulation of the failover lifecycle — validates the entire failover sequence
# without waiting for a real outage.
#
# This script is a TEST HARNESS only — it does not contain failover logic.
# All failover logic lives in failover.sh and is called directly from here.
# Any changes to failover.sh are automatically reflected in this test.
# ── WHAT THIS SCRIPT IS ───────────────────────────────────────────────────────────────────────
# A test harness only — contains no failover logic.
# All failover logic lives in failover.sh and is exercised by this test.
# Any changes to failover.sh are automatically reflected here.
#
# Test sequence:
# 1. Pre-flight — verify both servers reachable, failover.sh exists, state is NORMAL
# 2. Block — add iptables rule dropping all traffic to remote IP
# 3. Detect — run failover.sh one cycle — confirm FAILOVER state detected
# 4. Start — verify failover containers started locally
# 5. Restore — remove iptables rule, remote becomes reachable again
# 6. Handback — wait for failover.sh to confirm handback strikes and hand back
# 7. Verify — confirm containers returned to remote, local copies stopped
# 8. Report — full pass/fail summary per phase
# ── TEST SEQUENCE ─────────────────────────────────────────────────────────────────────────────
# Phase 1 — Pre-flight verify both servers reachable, daemons healthy,
# version parity, failover.sh exists, state is NORMAL
# Phase 2 — Block Remote iptables rule drops all traffic to remote IP
# Phase 3 — Failover Detection wait for failover.sh to detect outage and enter FAILOVER
# Phase 4 — Container Start verify Tier 1 failover containers started locally
# Phase 5 — Restore remove iptables rule, remote becomes reachable
# Phase 6 — Handback wait for failover.sh to complete handback to NORMAL
# Phase 7 — Container Handback verify Tier 1 containers stopped locally after handback
# Phase 8 — Report full pass/fail summary per phase
#
# Safety: iptables rule is removed via trap on ANY exit — crash, error, ctrl-c, or normal.
# Remote connectivity is always restored regardless of test outcome.
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
# FAILOVER_ENABLED gate — aborts if failover monitoring is disabled
# iptables safety trap — rule ALWAYS removed on exit (crash, error, ctrl-c, normal)
# remote connectivity always restored regardless of outcome
# Version parity check — pre-flight verifies both servers on compatible unRAID versions
# Remote Docker daemon — pre-flight verifies remote daemon is responsive
# DOCKER_TIMEOUT — all docker calls protected against daemon hangs
# MY_ID-based routing — tier containers selected via MY_ID not hostname comparison
# Command validation — iptables and notify validated before use
# Dry-run safe — full sequence walkthrough without touching iptables or containers
#
# ⚠️ This script starts and stops real containers on both servers.
# ── WARNING ───────────────────────────────────────────────────────────────────────────────────
# ⚠️ This script starts and stops REAL containers on both servers.
# Run during a maintenance window — users will experience a brief service interruption.
# Use --dry-run to walk through the sequence without touching containers or iptables.
# Use --dry-run to walk through the sequence without any real changes.
#
# All configuration in Master.conf under Failover and Failover Test sections.
# -----------------------------------------------------------------------------------------------
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
# FAILOVER_TEST_BLOCK_WAIT — seconds to wait for failover.sh to detect outage
# FAILOVER_TEST_HANDBACK_WAIT — seconds to wait for failover.sh to complete handback
# FAILOVER_CHECK_INTERVAL — check interval of the running failover.sh (informational)
# FAILOVER_HANDBACK_STRIKES — strikes required before handback (informational)
# FAILOVER_STATE_FILE — state file path to read current state
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
# failover_test.sh — run full test sequence
# failover_test.sh --dry-run — walk through all phases without changes
# failover_test.sh --status — show current failover state and test config
# failover_test.sh --log — verbose output
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../Master.conf"
source "$SCRIPT_DIR/../common.sh"
source "$SCRIPT_DIR/../load_config.sh"
parse_args "$@"
FAILOVER_SCRIPT="$SCRIPT_DIR/failover.sh"
DOCKER_TIMEOUT=15
# ==============================================================================================
# ── SAFETY TRAP — always remove iptables rule on exit ─────────────────────────────────────────
# ==============================================================================================
# Fires on normal exit, error exit, ctrl-c, and script crashes.
# Remote connectivity is ALWAYS restored regardless of test outcome.
# -----------------------------------------------------------------------------------------------
# SAFETY TRAP — always remove iptables rule on exit
# Fires on normal exit, error exit, ctrl-c, and script crashes
# -----------------------------------------------------------------------------------------------
IPTABLES_RULE_ACTIVE=false
cleanup() {
@@ -51,7 +75,7 @@ cleanup() {
if [[ "$DRY_RUN" == false ]]; then
iptables -D OUTPUT -d "$REMOTE_SERVER" -j DROP 2>/dev/null
IPTABLES_RULE_ACTIVE=false
success "iptables rule removed — remote connectivity restored"
warn "iptables rule removed — remote connectivity restored"
else
warn "DRY RUN — would remove iptables rule"
fi
@@ -60,9 +84,9 @@ cleanup() {
trap cleanup EXIT
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Setup ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_GEAR Setup ━━━"
@@ -71,93 +95,127 @@ if [[ "$EUID" -ne 0 ]]; then
exit 1
fi
success "Running as root"
if ! command -v iptables >/dev/null 2>&1; then
error "iptables not found — required for connectivity simulation"
exit 1
# FAILOVER_ENABLED gate — no point testing if failover is disabled
if [[ "${FAILOVER_ENABLED:-false}" == false ]]; then
warn "FAILOVER_ENABLED=false — failover test aborted"
warn "Enable failover in master.conf before running this test"
exit 0
fi
success "iptables available"
acquire_lock # strict single instance — modifies iptables and containers
detect_hosts
resolve_remote_ip
# Validate commands used by this script
validate_unraid_cmd \
"$(which iptables 2>/dev/null || echo /sbin/iptables)" \
"--version" "iptables" \
"iptables" || { error "iptables not found — required for connectivity simulation"; exit 1; }
validate_unraid_cmd \
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
"" "" \
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
if [[ ! -f "$FAILOVER_SCRIPT" ]]; then
error "failover.sh not found at $FAILOVER_SCRIPT"
exit 1
fi
success "failover.sh found"
detect_hosts
resolve_remote_ip
log "failover.sh found at $FAILOVER_SCRIPT"
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no iptables rules or container changes will be made"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Status ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
local_ver=$(grep -oP '(?<=version=")[^"]+' /etc/unraid-version 2>/dev/null || echo "unknown")
echo ""
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
echo "$ICON_HOST Local: $LOCAL_SERVER_NAME"
echo "$ICON_HOST Remote: $REMOTE_SERVER_NAME ($REMOTE_SERVER)"
echo "$ICON_HOST My ID: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_HOST Remote ID: $REMOTE_ID ($REMOTE_SERVER_NAME — $REMOTE_SERVER)"
echo "$ICON_GEAR unRAID ver: $local_ver"
echo "$ICON_FAILOVER Block wait: ${FAILOVER_TEST_BLOCK_WAIT}s"
echo "$ICON_FAILOVER Handback wait: ${FAILOVER_TEST_HANDBACK_WAIT}s"
echo "$ICON_FAILOVER Check interval: ${FAILOVER_CHECK_INTERVAL}s"
echo "$ICON_FAILOVER Handback strikes: ${FAILOVER_HANDBACK_STRIKES}"
echo "$ICON_GEAR Dry Run: $DRY_RUN"
# Current failover state
if [[ -f "$FAILOVER_STATE_FILE" ]]; then
CURRENT_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
echo "$ICON_FAILOVER Current state: ${CURRENT_STATE:-unknown}"
else
echo "$ICON_FAILOVER Current state: no state file"
fi
# Show Tier 1 containers for this host
TIER1_VAR="FAILOVER_${MY_ID}_RUNS_FOR_${REMOTE_ID}_TIER1"
eval "TIER1_CONTAINERS=(\"\${${TIER1_VAR}[@]:-}\")"
echo "$ICON_CONTAINERS Tier 1 to test: ${TIER1_CONTAINERS[*]:-none configured}"
echo "━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
fi
# -----------------------------------------------------------------------------------------------
# PHASE TRACKING
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ── PHASE TRACKING ────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
PHASES_PASS=()
PHASES_FAIL=()
TOTAL_START=$(date +%s)
phase_pass() { PHASES_PASS+=("$1"); success "$ICON_DONE Phase: $1 — PASSED"; }
phase_fail() { PHASES_FAIL+=("$1"); error "$ICON_ERROR Phase: $1 — FAILED"; }
phase_pass() { PHASES_PASS+=("$1"); warn "$ICON_DONE Phase: $1 — PASSED ✅"; }
phase_fail() { PHASES_FAIL+=("$1"); error "Phase: $1 — FAILED ❌"; }
# -----------------------------------------------------------------------------------------------
# ━━━ PHASE 1 — Pre-flight ━━━
# -----------------------------------------------------------------------------------------------
# Get Tier 1 containers for this server's failover responsibility
TIER1_VAR="FAILOVER_${MY_ID}_RUNS_FOR_${REMOTE_ID}_TIER1"
eval "TIER1_CONTAINERS=(\"\${${TIER1_VAR}[@]:-}\")"
# ==============================================================================================
# ━━━ Phase 1 — Pre-flight ━━━
# ==============================================================================================
echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo " $ICON_SHIELD FAILOVER TEST — $(date '+%Y-%m-%d %H:%M:%S')"
echo " $ICON_HOST Local: $LOCAL_SERVER_NAME"
echo " $ICON_HOST Remote: $REMOTE_SERVER_NAME ($REMOTE_SERVER)"
echo " $ICON_HOST $MY_ID ($LOCAL_SERVER_NAME) → $REMOTE_ID ($REMOTE_SERVER_NAME)"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo ""
echo "━━━ $ICON_SHIELD Phase 1 — Pre-flight ━━━"
# Check remote reachable
info "Checking remote reachability..."
# Remote reachable
if ping_remote; then
success "Remote $REMOTE_SERVER_NAME is reachable"
log "$REMOTE_SERVER_NAME is reachable"
else
error "Remote $REMOTE_SERVER_NAME is not reachable — cannot run test"
error "$REMOTE_SERVER_NAME is not reachable — cannot run test"
phase_fail "Pre-flight"
exit 1
fi
# Check internet reachable
info "Checking internet connectivity..."
# Internet reachable
if ping_internet; then
success "Internet is reachable"
log "Internet is reachable"
else
error "No internet connectivity — cannot run test"
phase_fail "Pre-flight"
exit 1
fi
# Check current failover state is NORMAL
# Version parity — test may produce misleading results on mismatch
if ! check_unraid_version_parity; then
error "unRAID version mismatch — test aborted to prevent misleading results"
phase_fail "Pre-flight"
exit 1
fi
# Remote Docker daemon — must be responsive before test manipulates containers
if ! check_remote_docker_daemon; then
error "Remote Docker daemon not responsive — cannot run test"
phase_fail "Pre-flight"
exit 1
fi
# Failover state must be NORMAL before test
if [[ -f "$FAILOVER_STATE_FILE" ]]; then
CURRENT_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
if [[ "$CURRENT_STATE" != "NORMAL" ]]; then
@@ -165,28 +223,37 @@ if [[ -f "$FAILOVER_STATE_FILE" ]]; then
phase_fail "Pre-flight"
exit 1
fi
success "Failover state is NORMAL"
log "Failover state is NORMAL"
else
warn "No state file found — assuming NORMAL (first run)"
fi
# Tier 1 containers configured
if [[ ${#TIER1_CONTAINERS[@]} -eq 0 ]]; then
error "No Tier 1 containers configured for $MY_ID → $REMOTE_ID"
error "Check FAILOVER_${MY_ID}_RUNS_FOR_${REMOTE_ID}_TIER1 in master_host*.conf"
phase_fail "Pre-flight"
exit 1
fi
log "Tier 1 containers: ${TIER1_CONTAINERS[*]}"
phase_pass "Pre-flight"
# -----------------------------------------------------------------------------------------------
# ━━━ PHASE 2 — Block Remote ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Phase 2 — Block Remote Connectivity ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_PING Phase 2 — Block Remote Connectivity ━━━"
warn "Adding iptables rule — dropping all traffic to $REMOTE_SERVER"
warn "Adding iptables rule — dropping all traffic to $REMOTE_SERVER ($REMOTE_SERVER_NAME)"
if [[ "$DRY_RUN" == false ]]; then
iptables -I OUTPUT -d "$REMOTE_SERVER" -j DROP
IPTABLES_RULE_ACTIVE=true
success "iptables rule active — $REMOTE_SERVER_NAME appears unreachable"
# Verify block is working
sleep 2
if ! ping -c1 -W2 "$REMOTE_SERVER" &>/dev/null; then
success "Connectivity block confirmed — ping to remote fails as expected"
log "Connectivity block confirmed — ping to remote fails as expected"
phase_pass "Block Remote"
else
error "iptables rule did not block connectivity — ping still succeeds"
@@ -194,30 +261,29 @@ if [[ "$DRY_RUN" == false ]]; then
exit 1
fi
else
warn "DRY RUN — would block $REMOTE_SERVER with iptables"
warn "DRY RUN — would block $REMOTE_SERVER with iptables DROP rule"
phase_pass "Block Remote"
fi
# -----------------------------------------------------------------------------------------------
# ━━━ PHASE 3 — Failover Detection ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Phase 3 — Failover Detection ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_FAILOVER Phase 3 — Failover Detection ━━━"
info "Waiting ${FAILOVER_TEST_BLOCK_WAIT}s for failover.sh to detect outage..."
info "failover.sh check interval is ${FAILOVER_CHECK_INTERVAL}s"
warn "Waiting ${FAILOVER_TEST_BLOCK_WAIT}s for failover.sh to detect outage..."
log "failover.sh check interval: ${FAILOVER_CHECK_INTERVAL}s"
if [[ "$DRY_RUN" == false ]]; then
sleep "$FAILOVER_TEST_BLOCK_WAIT"
# Check state file updated to FAILOVER
if [[ -f "$FAILOVER_STATE_FILE" ]]; then
NEW_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
if [[ "$NEW_STATE" == "FAILOVER" ]]; then
success "State changed to FAILOVER — outage detected correctly"
log "State changed to FAILOVER — outage detected correctly ✅"
phase_pass "Failover Detection"
else
error "State is $NEW_STATE — expected FAILOVER after ${FAILOVER_TEST_BLOCK_WAIT}s"
warn "failover.sh may not be running — check User Scripts plugin"
warn "Is failover.sh running? Check User Scripts plugin"
phase_fail "Failover Detection"
fi
else
@@ -225,30 +291,25 @@ if [[ "$DRY_RUN" == false ]]; then
phase_fail "Failover Detection"
fi
else
warn "DRY RUN — would wait ${FAILOVER_TEST_BLOCK_WAIT}s and check for FAILOVER state"
warn "DRY RUN — would wait ${FAILOVER_TEST_BLOCK_WAIT}s then check for FAILOVER state"
phase_pass "Failover Detection"
fi
# -----------------------------------------------------------------------------------------------
# ━━━ PHASE 4 — Container Start Verification ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Phase 4 — Container Start Verification ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_CONTAINERS Phase 4 — Failover Containers Started ━━━"
# Determine which containers should have started on this host
if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then
EXPECTED_CONTAINERS=("${FAILOVER_HOST1_STARTS_FOR_HOST2[@]}")
else
EXPECTED_CONTAINERS=("${FAILOVER_HOST2_STARTS_FOR_HOST1[@]}")
fi
echo "━━━ $ICON_CONTAINERS Phase 4 — Tier 1 Containers Started Locally ━━━"
log "Checking Tier 1 containers: ${TIER1_CONTAINERS[*]}"
if [[ "$DRY_RUN" == false ]]; then
CONTAINERS_OK=true
for container in "${EXPECTED_CONTAINERS[@]}"; do
for container in "${TIER1_CONTAINERS[@]}"; do
[[ -z "$container" ]] && continue
STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \
"$container" 2>/dev/null)
if [[ "$STATUS" == "true" ]]; then
success "$ICON_RUNNING $container is running locally"
log "$ICON_RUNNING $container is running locally ✅"
else
error "$ICON_NOT_RUNNING $container is NOT running locally"
CONTAINERS_OK=false
@@ -261,29 +322,27 @@ if [[ "$DRY_RUN" == false ]]; then
phase_fail "Container Start"
fi
else
warn "DRY RUN — would verify these containers started: ${EXPECTED_CONTAINERS[*]}"
warn "DRY RUN — would verify these Tier 1 containers started: ${TIER1_CONTAINERS[*]}"
phase_pass "Container Start"
fi
# -----------------------------------------------------------------------------------------------
# ━━━ PHASE 5 — Restore Connectivity ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Phase 5 — Restore Remote Connectivity ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_PING Phase 5 — Restore Remote Connectivity ━━━"
info "Removing iptables block — remote becomes reachable again"
warn "Removing iptables block — $REMOTE_SERVER_NAME becomes reachable again"
if [[ "$DRY_RUN" == false ]]; then
iptables -D OUTPUT -d "$REMOTE_SERVER" -j DROP 2>/dev/null
IPTABLES_RULE_ACTIVE=false
success "iptables rule removed"
# Verify connectivity restored
sleep 3
if ping_remote; then
success "Remote $REMOTE_SERVER_NAME is reachable again"
log "$REMOTE_SERVER_NAME is reachable again ✅"
phase_pass "Restore Connectivity"
else
error "Remote still unreachable after removing iptables rule"
error "$REMOTE_SERVER_NAME still unreachable after removing iptables rule"
phase_fail "Restore Connectivity"
fi
else
@@ -291,14 +350,14 @@ else
phase_pass "Restore Connectivity"
fi
# -----------------------------------------------------------------------------------------------
# ━━━ PHASE 6 — Handback ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Phase 6 — Handback ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_FAILOVER Phase 6 — Handback ━━━"
info "Waiting ${FAILOVER_TEST_HANDBACK_WAIT}s for failover.sh to confirm handback..."
info "Requires $FAILOVER_HANDBACK_STRIKES consecutive remote-up checks at ${FAILOVER_CHECK_INTERVAL}s intervals"
info "Estimated minimum wait: $(( FAILOVER_HANDBACK_STRIKES * FAILOVER_CHECK_INTERVAL ))s"
warn "Waiting ${FAILOVER_TEST_HANDBACK_WAIT}s for failover.sh to complete handback..."
log "Requires $FAILOVER_HANDBACK_STRIKES consecutive checks at ${FAILOVER_CHECK_INTERVAL}s"
log "Minimum handback time: $(( FAILOVER_HANDBACK_STRIKES * FAILOVER_CHECK_INTERVAL ))s"
if [[ "$DRY_RUN" == false ]]; then
sleep "$FAILOVER_TEST_HANDBACK_WAIT"
@@ -306,10 +365,11 @@ if [[ "$DRY_RUN" == false ]]; then
if [[ -f "$FAILOVER_STATE_FILE" ]]; then
FINAL_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
if [[ "$FINAL_STATE" == "NORMAL" ]]; then
success "State returned to NORMAL — handback completed"
log "State returned to NORMAL — handback completed ✅"
phase_pass "Handback"
else
error "State is $FINAL_STATE — expected NORMAL after handback wait"
error "State is $FINAL_STATE — expected NORMAL after ${FAILOVER_TEST_HANDBACK_WAIT}s"
warn "Handback may still be in progress — check failover.sh output"
phase_fail "Handback"
fi
else
@@ -317,23 +377,25 @@ if [[ "$DRY_RUN" == false ]]; then
phase_fail "Handback"
fi
else
warn "DRY RUN — would wait ${FAILOVER_TEST_HANDBACK_WAIT}s and verify NORMAL state"
warn "DRY RUN — would wait ${FAILOVER_TEST_HANDBACK_WAIT}s then verify NORMAL state"
phase_pass "Handback"
fi
# -----------------------------------------------------------------------------------------------
# ━━━ PHASE 7 — Container Handback Verification ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Phase 7 — Container Handback Verification ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_CONTAINERS Phase 7 — Failover Containers Stopped Locally ━━━"
echo "━━━ $ICON_CONTAINERS Phase 7 — Tier 1 Containers Stopped Locally ━━━"
log "Verifying Tier 1 containers returned to $REMOTE_SERVER_NAME"
if [[ "$DRY_RUN" == false ]]; then
HANDBACK_OK=true
for container in "${EXPECTED_CONTAINERS[@]}"; do
for container in "${TIER1_CONTAINERS[@]}"; do
[[ -z "$container" ]] && continue
STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \
"$container" 2>/dev/null)
if [[ "$STATUS" != "true" ]]; then
success "$ICON_NOT_RUNNING $container stopped locally — handed back"
log "$ICON_NOT_RUNNING $container stopped locally — handed back ✅"
else
error "$ICON_RUNNING $container still running locally — handback may have failed"
HANDBACK_OK=false
@@ -346,20 +408,20 @@ if [[ "$DRY_RUN" == false ]]; then
phase_fail "Container Handback"
fi
else
warn "DRY RUN — would verify failover containers stopped locally after handback"
warn "DRY RUN — would verify Tier 1 containers stopped locally after handback"
phase_pass "Container Handback"
fi
TOTAL_END=$(date +%s)
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Test Report ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Test Report ━━━
# ==============================================================================================
echo ""
echo "━━━━━ $ICON_SUMMARY FAILOVER TEST REPORT ━━━━━"
echo "$ICON_HOST Local: $LOCAL_SERVER_NAME"
echo "$ICON_HOST Remote: $REMOTE_SERVER_NAME"
echo "$ICON_TIME Duration: $(format_duration $((TOTAL_END - TOTAL_START)))"
echo "$ICON_HOST My ID: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_HOST Remote: $REMOTE_ID ($REMOTE_SERVER_NAME)"
echo "$ICON_TIME Duration: $(format_duration $((TOTAL_END - TOTAL_START)))"
echo ""
echo " Phase Results:"
for phase in "${PHASES_PASS[@]}"; do
@@ -375,13 +437,15 @@ FAIL_COUNT=${#PHASES_FAIL[@]}
TOTAL_PHASES=$(( PASS_COUNT + FAIL_COUNT ))
if [[ "$DRY_RUN" == true ]]; then
echo "$ICON_WARN Status: DRY RUN — no changes made"
warn "DRY RUN — no changes made"
elif [[ "$FAIL_COUNT" -eq 0 ]]; then
echo "$ICON_DONE Status: $ICON_SUCCESS ALL $TOTAL_PHASES PHASES PASSED"
notify "Failover test PASSED on $(hostname) — all $TOTAL_PHASES phases completed successfully" "Failover Test" "normal"
warn "$ICON_DONE ALL $TOTAL_PHASES PHASES PASSED"
notify "Failover test PASSED on $(hostname) — all $TOTAL_PHASES phases completed" \
"Failover Test" "normal"
else
echo "$ICON_ERROR Status: $FAIL_COUNT/$TOTAL_PHASES PHASES FAILED"
notify "Failover test FAILED on $(hostname) — $FAIL_COUNT/$TOTAL_PHASES phases failed: ${PHASES_FAIL[*]}" "Failover Test" "warning"
error "$FAIL_COUNT/$TOTAL_PHASES PHASES FAILED"
notify "Failover test FAILED on $(hostname) — $FAIL_COUNT/$TOTAL_PHASES phases failed: ${PHASES_FAIL[*]}" \
"Failover Test" "warning"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
Media/README-Media.md
+907 -253
View File
File diff suppressed because it is too large Load Diff
Media/arrs_failed_stalled_recovery.sh
+236 -171
View File
@@ -1,67 +1,81 @@
#!/bin/bash
# -----------------------------------------------------------------------------------------------
# --------------------------------- Arrs Failed Stalled Recovery --------------------------------
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ========================= Arrs Failed / Stalled Recovery =====================================
# ==============================================================================================
# Automatically detects and recovers from failed imports and stalled downloads
# across Sonarr, Radarr, and Lidarr. Blocklists the bad release and triggers
# a new search — hands free recovery while you sleep.
# Schedule: 0 */6 * * * (every 6 hours)
# a new search — hands-free recovery while you sleep.
#
# What it checks (per-arr toggles in Master.conf):
# HOST1: Sonarr (Tv_Shows) — /api/v3/
# HOST1: Radarr (Movies) — /api/v3/
# HOST1: Lidarr (Music) — /api/v1/ ← HOST1 only, exits cleanly on HOST2
# HOST2: Sonarr (Anime_Shows) — /api/v3/
# HOST2: Radarr (Anime_Movies) — /api/v3/
#
# Targets four problem types from the queue API:
# ── WHAT IT CHECKS ────────────────────────────────────────────────────────────────────────────
# Four problem types from the arr queue API:
# importFailed — downloaded successfully but arr couldn't import the file
# importPending — downloaded, stuck waiting to import (won't self-resolve)
# error status — serious failure not covered by importFailed/importPending
# importPending — downloaded, stuck waiting to import (will not self-resolve)
# error status — serious failure not covered by the above two states
# stalled — download stuck with no connections or no progress
#
# Items newer than ARR_IMPORT_RECOVERY_AGE (6hr) are skipped — gives arr time to retry.
# Never touches items with state "downloading" or "imported" — safe to run anytime.
# Items newer than ARR_IMPORT_RECOVERY_AGE are skipped — gives arr time to retry first.
#
# Action per problem item:
# 1. Blocklist the release — prevents re-grabbing the same bad release
# 2. Remove from queue — cleans up the failed item
# 3. Trigger new search — finds a different release automatically
# ── WHAT IT DOES PER PROBLEM ITEM ─────────────────────────────────────────────────────────────
# 1. Blocklist the release — prevents re-grabbing the same bad release
# 2. Remove from queue — cleans up the failed item
# 3. Trigger new search — finds a different release automatically
#
# Configuration in Master.conf:
# ARR_IMPORT_RECOVERY_AGE — hours before item is eligible
# HOST1/2_SONARR_RECOVERY — enable/disable per arr
# HOST1/2_RADARR_RECOVERY — enable/disable per arr
# HOST1_LIDARR_RECOVERY — enable/disable Lidarr (HOST1 only)
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
# detect_hosts() sets MY_ID and aliases all arr vars:
# SONARR_URL / SONARR_API_KEY / SONARR_RECOVERY
# RADARR_URL / RADARR_API_KEY / RADARR_RECOVERY
# LIDARR_URL / LIDARR_API_KEY / LIDARR_RECOVERY (HOST1 only — exits cleanly on HOST2)
# No manual HOST1/HOST2 comparisons needed — MY_ID routes automatically.
#
# Age threshold (ARR_IMPORT_RECOVERY_AGE):
# Items newer than threshold are skipped — gives the arr time to retry on its own
# Items older than threshold have not self-resolved — safe to intervene
# Default: 12 hours
# ── API VERSION SAFETY ────────────────────────────────────────────────────────────────────────
# check_arr_version() verifies the running arr matches the tested major version in master.conf.
# If the API structure changed after an upgrade — exits rather than silently misoperating.
# Sonarr v4 → /api/v3/ (v3 endpoint retained in v4)
# Radarr v6 → /api/v3/ (v3 endpoint retained in v6)
# Lidarr v3 → /api/v1/ (different from Sonarr/Radarr)
#
# API versions:
# Sonarr v4 → /api/v3/ (v3 endpoint retained in v4)
# Radarr v6 → /api/v3/ (v3 endpoint retained in v6)
# Lidarr v3 → /api/v1/ (different from Sonarr/Radarr)
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
# acquire_lock — prevents concurrent runs overlapping
# jq validation — exits if jq not installed (required for JSON parsing)
# API pre-flight — checks each arr is reachable before querying queue
# Version check — verifies arr major version matches tested version in master.conf
# Age threshold — skips items newer than ARR_IMPORT_RECOVERY_AGE (default 6hr)
# Silent by default — only problems produce output, clean arrs stay silent
#
# Per-arr enable/disable toggles in Master.conf.
# Lidarr runs on HOST1 only — exits cleanly on HOST2.
# detect_hosts() selects correct URL and API key per server at runtime.
# ── CONFIGURATION (master_host*.conf) ─────────────────────────────────────────────────────────
# HOST*_SONARR_URL / HOST*_SONARR_API_KEY / HOST*_SONARR_RECOVERY
# HOST*_RADARR_URL / HOST*_RADARR_API_KEY / HOST*_RADARR_RECOVERY
# HOST1_LIDARR_URL / HOST1_LIDARR_API_KEY / HOST1_LIDARR_RECOVERY
# All aliased by detect_hosts() — script uses unprefixed names
#
# Recommended schedule: 0 5 * * * (5am daily)
# Supports --dry-run to show what would be actioned without making changes.
# -----------------------------------------------------------------------------------------------
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
# ARR_IMPORT_RECOVERY_AGE — hours before item is eligible (default 6)
# SONARR_VERSION_MAJOR — expected Sonarr major version (e.g. 4)
# RADARR_VERSION_MAJOR — expected Radarr major version (e.g. 6)
# LIDARR_VERSION_MAJOR — expected Lidarr major version (e.g. 3)
# ARR_RECOVERY_STATS — stats file path (read by coffee report)
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
# arrs_failed_stalled_recovery.sh — normal run
# arrs_failed_stalled_recovery.sh --dry-run — show what would be actioned, no changes
# arrs_failed_stalled_recovery.sh --log — verbose output
# arrs_failed_stalled_recovery.sh --status — show config and exit
#
# ── SCHEDULE ──────────────────────────────────────────────────────────────────────────────────
# Recommended: 0 5 * * * (5am daily)
# Or every 6hr: 0 */6 * * * (matches ARR_IMPORT_RECOVERY_AGE default)
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../Master.conf"
source "$SCRIPT_DIR/../common.sh"
source "$SCRIPT_DIR/../load_config.sh"
parse_args "$@"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Setup ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_GEAR Setup ━━━"
@@ -70,41 +84,73 @@ if [[ "$EUID" -ne 0 ]]; then
exit 1
fi
success "Running as root"
acquire_lock
# detect_hosts() sets MY_ID and aliases SONARR_*, RADARR_*, LIDARR_* vars
detect_hosts
# jq is required — not optional — for JSON parsing
if ! command -v jq >/dev/null 2>&1; then
error "jq is not installed — required for arr API JSON parsing"
error "Install: apt-get install jq or brew install jq"
notify "arrs_failed_stalled_recovery failed on $(hostname) — jq not installed" \
"Arr Recovery" "warning"
exit 1
fi
log "jq found"
validate_unraid_cmd \
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
"" "" \
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no items will be blocklisted or searched"
# Age threshold in seconds for comparison
# Age threshold in seconds
AGE_THRESHOLD_SECONDS=$(( ARR_IMPORT_RECOVERY_AGE * 3600 ))
# Tracking totals
TOTAL_ACTIONED=0
TOTAL_SKIPPED=0
ARR_SUMMARIES=()
# -----------------------------------------------------------------------------------------------
# CORE FUNCTIONS
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
echo ""
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_SYNC Sonarr: ${SONARR_URL:-not configured} (recovery: ${SONARR_RECOVERY:-true})"
echo "$ICON_SYNC Radarr: ${RADARR_URL:-not configured} (recovery: ${RADARR_RECOVERY:-true})"
echo "$ICON_SYNC Lidarr: ${LIDARR_URL:-not configured on this host} (recovery: ${LIDARR_RECOVERY:-false})"
echo "$ICON_TIME Age thresh: ${ARR_IMPORT_RECOVERY_AGE}hr"
echo "$ICON_GEAR Sonarr ver: v${SONARR_VERSION_MAJOR} expected"
echo "$ICON_GEAR Radarr ver: v${RADARR_VERSION_MAJOR} expected"
echo "$ICON_GEAR Lidarr ver: v${LIDARR_VERSION_MAJOR} expected"
echo "$ICON_NOTIFY Notify: unRAID=${NOTIFY_UNRAID:-false} Discord=$([[ -n "${MY_DISCORD_WEBHOOK:-}" ]] && echo enabled || echo disabled)"
echo "$ICON_GEAR Dry Run: $DRY_RUN"
echo "━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
fi
# ==============================================================================================
# ── HELPER FUNCTIONS ──────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Check if a queue item is older than ARR_IMPORT_RECOVERY_AGE
# Returns 0 (old enough) or 1 (too new — skip)
item_is_old_enough() {
local added="$1"
[[ -z "$added" ]] && return 0 # no date = treat as old enough
[[ -z "$added" ]] && return 0 # no date = treat as old enough, safe to act
local added_epoch
added_epoch=$(date -d "$added" +%s 2>/dev/null) || return 0
local now_epoch
now_epoch=$(date +%s)
local age_seconds=$(( now_epoch - added_epoch ))
local age_seconds=$(( $(date +%s) - added_epoch ))
[[ "$age_seconds" -ge "$AGE_THRESHOLD_SECONDS" ]]
}
# Query the arr queue and return failed/stalled items
# Query the arr queue API and return all records
# Args: url, api_key, api_version
get_problem_items() {
get_queue_data() {
local url="$1" api_key="$2" api_version="$3"
curl -sf --max-time 15 \
-H "X-Api-Key: $api_key" \
@@ -112,7 +158,7 @@ get_problem_items() {
2>/dev/null
}
# Blocklist and remove item from queue
# Blocklist and remove a queue item
# Args: url, api_key, api_version, queue_id
blocklist_item() {
local url="$1" api_key="$2" api_version="$3" queue_id="$4"
@@ -127,18 +173,18 @@ blocklist_item() {
>/dev/null 2>&1
}
# Trigger new search
# Trigger a new search for the media item
# Args: url, api_key, api_version, arr_type, media_id
trigger_search() {
local url="$1" api_key="$2" api_version="$3" arr_type="$4" media_id="$5"
local command body
case "$arr_type" in
sonarr) command="EpisodeSearch"; body="{\"name\":\"EpisodeSearch\",\"episodeIds\":[$media_id]}" ;;
radarr) command="MoviesSearch"; body="{\"name\":\"MoviesSearch\",\"movieIds\":[$media_id]}" ;;
lidarr) command="AlbumSearch"; body="{\"name\":\"AlbumSearch\",\"albumIds\":[$media_id]}" ;;
sonarr) command="EpisodeSearch"; body="{\"name\":\"EpisodeSearch\",\"episodeIds\":[$media_id]}" ;;
radarr) command="MoviesSearch"; body="{\"name\":\"MoviesSearch\",\"movieIds\":[$media_id]}" ;;
lidarr) command="AlbumSearch"; body="{\"name\":\"AlbumSearch\",\"albumIds\":[$media_id]}" ;;
esac
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would trigger $command for ID $media_id"
warn "DRY RUN — would trigger $command for media ID $media_id"
return 0
fi
curl -sf --max-time 15 \
@@ -150,34 +196,64 @@ trigger_search() {
>/dev/null 2>&1
}
# -----------------------------------------------------------------------------------------------
# PROCESS AN ARR
# Args: arr_name, arr_type, url, api_key, api_version, enabled
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ── PROCESS AN ARR ────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Args: display_name, arr_type, url, api_key, api_version, enabled,
# version_major, version_api_prefix
#
# Exits cleanly if disabled.
# Checks API reachability and version before touching queue.
# Processes each problem item: blocklist + trigger new search.
# Silent when clean — only warns when problems found or actioned.
process_arr() {
local arr_name="$1" arr_type="$2" url="$3" api_key="$4" api_version="$5" enabled="$6"
local actioned=0 skipped=0 too_new=0
local arr_name="$1"
local arr_type="$2"
local url="$3"
local api_key="$4"
local api_version="$5"
local enabled="$6"
local version_major="$7"
local version_api_prefix="$8"
local actioned=0 skipped_new=0
echo ""
echo "━━━ $ICON_SYNC $arr_name ━━━"
# Disabled — skip cleanly
if [[ "$enabled" != "true" ]]; then
info "$arr_name recovery disabled — skipping"
log "$arr_name recovery disabled — skipping"
ARR_SUMMARIES+=("$arr_name: disabled")
return
fi
# API pre-flight
# URL not configured on this host — skip cleanly
if [[ -z "$url" ]]; then
log "$arr_name not configured on $MY_ID — skipping"
ARR_SUMMARIES+=("$arr_name: not configured on $MY_ID")
return
fi
# API reachability
if ! check_api "$url" "$arr_name" 10; then
warn "$arr_name API unreachable — skipping"
ARR_SUMMARIES+=("$arr_name: unreachable")
return
fi
# Get queue
# Version check — exit if API structure may have changed
if ! check_arr_version "$url" "$api_key" "$version_api_prefix" \
"$version_major" "$arr_name"; then
ARR_SUMMARIES+=("$arr_name: version mismatch — skipped")
return
fi
# Fetch queue
local queue_data
queue_data=$(get_problem_items "$url" "$api_key" "$api_version")
if [[ -z "$queue_data" ]] || ! command -v jq >/dev/null 2>&1; then
queue_data=$(get_queue_data "$url" "$api_key" "$api_version")
if [[ -z "$queue_data" ]]; then
warn "$arr_name — could not retrieve queue data"
ARR_SUMMARIES+=("$arr_name: queue fetch failed")
return
@@ -185,13 +261,9 @@ process_arr() {
local total_records
total_records=$(echo "$queue_data" | jq '.totalRecords // 0' 2>/dev/null)
info "Queue: $total_records total items"
log "$arr_name queue: $total_records total items"
# Filter for problem items — never touch downloading or imported
# importFailed = tried to import, actually failed
# importPending = downloaded, stuck waiting to import (won't self-resolve)
# error status = serious failure not covered by above states
# stalled = download stuck with no connections or progress
local problem_items
problem_items=$(echo "$queue_data" | jq -c '
.records // [] |
@@ -205,48 +277,46 @@ process_arr() {
.trackedDownloadStatus == "error" or
(.status == "warning" and (
(.errorMessage // "" | ascii_downcase | contains("stalled")) or
(.statusMessages // [] | .[] | .messages // [] | .[] | ascii_downcase | contains("stalled"))
(.statusMessages // [] | .[] | .messages // [] | .[] |
ascii_downcase | contains("stalled"))
))
)
)
' 2>/dev/null)
if [[ -z "$problem_items" ]]; then
success "$arr_name — no failed imports or stalled downloads found"
log "$arr_name — clean ✅ no failed imports or stalled downloads"
ARR_SUMMARIES+=("$arr_name: clean ✅")
return
fi
local problem_count
problem_count=$(echo "$problem_items" | wc -l)
info "Found $problem_count problem item(s)"
warn "$arr_name — found $problem_count problem item(s)"
# Process each problem item
while IFS= read -r item; do
[[ -z "$item" ]] && continue
local queue_id title added problem_type media_id
queue_id=$(echo "$item" | jq -r '.id // empty' 2>/dev/null)
title=$(echo "$item" | jq -r '.title // "Unknown"' 2>/dev/null)
added=$(echo "$item" | jq -r '.added // empty' 2>/dev/null)
local queue_id title added tracked_state tracked_status problem_type media_id
# Determine problem type for display
local tracked_state tracked_status
queue_id=$(echo "$item" | jq -r '.id // empty' 2>/dev/null)
title=$(echo "$item" | jq -r '.title // "Unknown"' 2>/dev/null)
added=$(echo "$item" | jq -r '.added // empty' 2>/dev/null)
tracked_state=$(echo "$item" | jq -r '.trackedDownloadState // ""' 2>/dev/null)
tracked_status=$(echo "$item" | jq -r '.trackedDownloadStatus // ""' 2>/dev/null)
# Human-readable problem type
case "$tracked_state" in
importFailed) problem_type="import failed" ;;
importPending) problem_type="import pending/stuck" ;;
*)
if [[ "$tracked_status" == "error" ]]; then
problem_type="error"
else
problem_type="stalled"
fi
[[ "$tracked_status" == "error" ]] && \
problem_type="error" || problem_type="stalled"
;;
esac
# Get media ID for search trigger (episode, movie, or album)
# Media ID for search trigger
case "$arr_type" in
sonarr) media_id=$(echo "$item" | jq -r '.episodeId // .episode.id // empty' 2>/dev/null) ;;
radarr) media_id=$(echo "$item" | jq -r '.movieId // .movie.id // empty' 2>/dev/null) ;;
@@ -255,129 +325,124 @@ process_arr() {
[[ -z "$queue_id" ]] && continue
# Age check
# Age check — skip items that are too new to have self-resolved
if ! item_is_old_enough "$added"; then
info " $ICON_TIME Skipping (too new < ${ARR_IMPORT_RECOVERY_AGE}hr): $title"
((too_new++))
((skipped++))
log " Skipping (too new < ${ARR_IMPORT_RECOVERY_AGE}hr): $title"
(( skipped_new++ ))
(( TOTAL_SKIPPED++ ))
continue
fi
info " $ICON_TRASH $problem_type: $title"
warn " $ICON_TRASH $problem_type — $title"
# Blocklist + remove
if blocklist_item "$url" "$api_key" "$api_version" "$queue_id"; then
log " Blocklisted queue item: $queue_id"
else
# Step 1: Blocklist + remove from queue
if ! blocklist_item "$url" "$api_key" "$api_version" "$queue_id"; then
warn " Failed to blocklist: $title"
((skipped++))
(( TOTAL_SKIPPED++ ))
continue
fi
log " Blocklisted: $queue_id"
# Trigger new search if we have a media ID
# Step 2: Trigger new search
if [[ -n "$media_id" ]]; then
if trigger_search "$url" "$api_key" "$api_version" "$arr_type" "$media_id"; then
log " New search triggered for: $title"
((actioned++))
((TOTAL_ACTIONED++))
log " New search triggered: $title"
else
warn " Blocklisted but search trigger failed: $title"
((actioned++))
((TOTAL_ACTIONED++))
fi
else
warn " Blocklisted but no media ID found for search: $title"
((actioned++))
((TOTAL_ACTIONED++))
warn " Blocklisted but no media ID found — search not triggered: $title"
fi
(( actioned++ ))
(( TOTAL_ACTIONED++ ))
done <<< "$problem_items"
if [[ "$DRY_RUN" == true ]]; then
info "$arr_name — dry run complete"
if [[ "$actioned" -gt 0 ]]; then
warn "$arr_name — actioned: $actioned | skipped (too new): $skipped_new"
else
success "$arr_name — actioned: $actioned | skipped (too new): $too_new"
log "$arr_name — nothing actioned | skipped (too new): $skipped_new"
fi
ARR_SUMMARIES+=("$arr_name: actioned $actioned | too new $too_new")
TOTAL_SKIPPED=$(( TOTAL_SKIPPED + skipped ))
ARR_SUMMARIES+=("$arr_name: actioned $actioned | too new $skipped_new")
}
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SYNC Process Each Arr ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_SYNC Arrs Failed Stalled Recovery — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
echo "$ICON_TIME Age threshold: ${ARR_IMPORT_RECOVERY_AGE}hr (items newer than this are skipped)"
# ==============================================================================================
# ━━━ Process Each Arr ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_SYNC Arrs Failed/Stalled Recovery — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME)"
log "Age threshold: ${ARR_IMPORT_RECOVERY_AGE}hr"
START=$(date +%s)
# Sonarr
if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then
process_arr "Sonarr (Tv_Shows)" "sonarr" \
"$HOST1_SONARR_URL" "$HOST1_SONARR_API_KEY" "v3" \
"${HOST1_SONARR_RECOVERY:-true}"
else
process_arr "Sonarr (Anime_Shows)" "sonarr" \
"$HOST2_SONARR_URL" "$HOST2_SONARR_API_KEY" "v3" \
"${HOST2_SONARR_RECOVERY:-true}"
fi
# Sonarr — uses aliased vars set by detect_hosts()
process_arr \
"Sonarr" \
"sonarr" \
"${SONARR_URL:-}" \
"${SONARR_API_KEY:-}" \
"v3" \
"${SONARR_RECOVERY:-true}" \
"${SONARR_VERSION_MAJOR:-4}" \
"v3"
# Radarr
if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then
process_arr "Radarr (Movies)" "radarr" \
"$HOST1_RADARR_URL" "$HOST1_RADARR_API_KEY" "v3" \
"${HOST1_RADARR_RECOVERY:-true}"
else
process_arr "Radarr (Anime_Movies)" "radarr" \
"$HOST2_RADARR_URL" "$HOST2_RADARR_API_KEY" "v3" \
"${HOST2_RADARR_RECOVERY:-true}"
fi
# Radarr — uses aliased vars set by detect_hosts()
process_arr \
"Radarr" \
"radarr" \
"${RADARR_URL:-}" \
"${RADARR_API_KEY:-}" \
"v3" \
"${RADARR_RECOVERY:-true}" \
"${RADARR_VERSION_MAJOR:-6}" \
"v3"
# Lidarr — HOST1 only
if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then
process_arr "Lidarr (Music)" "lidarr" \
"$HOST1_LIDARR_URL" "$HOST1_LIDARR_API_KEY" "v1" \
"${HOST1_LIDARR_RECOVERY:-true}"
else
info "Lidarr runs on $HOST1 only — skipping on $LOCAL_SERVER_NAME"
fi
# Lidarr — HOST1 only, LIDARR_URL empty on HOST2 → exits cleanly via "not configured" guard
process_arr \
"Lidarr" \
"lidarr" \
"${LIDARR_URL:-}" \
"${LIDARR_API_KEY:-}" \
"v1" \
"${LIDARR_RECOVERY:-false}" \
"${LIDARR_VERSION_MAJOR:-3}" \
"v1"
END=$(date +%s)
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Summary ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
echo ""
echo "━━━━━ $ICON_SUMMARY ARR IMPORT RECOVERY SUMMARY ━━━━━"
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
echo "$ICON_TRASH Actioned: $TOTAL_ACTIONED items blocklisted + searched"
echo "$ICON_RUNNING Skipped: $TOTAL_SKIPPED items (too new or unreachable)"
echo "━━━━━ $ICON_SUMMARY ARR RECOVERY SUMMARY ━━━━━"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
echo "$ICON_TRASH Actioned: $TOTAL_ACTIONED items blocklisted + searched"
echo "$ICON_SKIP Skipped: $TOTAL_SKIPPED items (too new)"
echo ""
echo " Problem types detected: importFailed | importPending | error | stalled"
echo ""
for summary in "${ARR_SUMMARIES[@]}"; do
echo " $ICON_SUMMARY $summary"
done
echo ""
if [[ "$DRY_RUN" == true ]]; then
echo "$ICON_WARN Status: DRY RUN — no changes made"
warn "DRY RUN — no changes made"
elif [[ "$TOTAL_ACTIONED" -gt 0 ]]; then
echo "$ICON_DONE Status: $ICON_SUCCESS DONE"
notify "Arr import recovery on $(hostname) — $TOTAL_ACTIONED item(s) blocklisted and re-searched. Check arrs for new downloads." "Arr Recovery" "normal"
warn "$ICON_DONE Done — $TOTAL_ACTIONED item(s) blocklisted and re-searched"
notify "Arr recovery on $(hostname) — $TOTAL_ACTIONED item(s) blocklisted and re-searched" \
"Arr Recovery" "warning"
else
echo "$ICON_DONE Status: $ICON_SUCCESS DONE — nothing to recover"
log "$ICON_DONE Done — nothing to recover (all arrs clean)"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
# Write stats for sunday_morning_coffee_report.sh
if [[ "$DRY_RUN" == false ]] && [[ -n "${ARR_RECOVERY_STATS:-}" ]]; then
DATE=$(date '+%Y-%m-%d')
TIME=$(date '+%H:%M')
echo "${DATE}|${TIME}|${TOTAL_ACTIONED}|${TOTAL_SKIPPED}" \
echo "$(date '+%Y-%m-%d')|$(date '+%H:%M')|${TOTAL_ACTIONED}|${TOTAL_SKIPPED}" \
>> "$ARR_RECOVERY_STATS" 2>/dev/null || true
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
Media/lidarr_cleanup.sh
+265 -213
View File
@@ -1,91 +1,118 @@
#!/bin/bash
# -----------------------------------------------------------------------------------------------
# --------------------------------- Lidarr Cleanup Script --------------------------------------
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ================================= Lidarr Cleanup =============================================
# ==============================================================================================
# Removes orphaned music files from the library that Lidarr no longer tracks.
# Uses the Lidarr API to build a complete list of tracked file paths then compares
# against what exists on disk — anything not tracked and older than LIDARR_ORPHAN_AGE
# against what exists on disk — anything untracked and older than LIDARR_ORPHAN_AGE
# days is considered an orphan and deleted.
#
# File classification:
# ── FILE CLASSIFICATION ───────────────────────────────────────────────────────────────────────
# TRACKED — Lidarr API knows about this exact file path → leave it alone
# PROTECTED — matches LIDARR_PROTECTED_PATTERNS → never delete (cover art, .nfo etc.)
# PROTECTED — matches LIDARR_PROTECTED_PATTERNS → never delete (cover art, .nfo, .lrc etc.)
# ORPHAN — music file, not tracked, older than LIDARR_ORPHAN_AGE days → delete
# JUNK — not a music extension, not protected → delete regardless of age
# RECENT — not tracked, under LIDARR_ORPHAN_AGE days old → skip (may be mid-import)
#
# Why protected patterns matter:
# Lidarr generates cover art (*.jpg), metadata (*.nfo) and lyrics (*.lrc) but does
# not include these in its tracked file API response. Without protection these would
# be classified as orphans and deleted — breaking Lidarr and Emby metadata display.
# ── WHY PROTECTED PATTERNS MATTER ─────────────────────────────────────────────────────────────
# Lidarr generates cover art (*.jpg), metadata (*.nfo) and lyrics (*.lrc) but does NOT
# include these in its tracked file API response. Without protection these would be
# classified as orphans and deleted — breaking Lidarr and Emby metadata display.
#
# Safety layers — all must pass before any file is touched:
# 1. Container must be running and healthy
# ── SAFETY LAYERS — ALL MUST PASS BEFORE ANY FILE IS TOUCHED ─────────────────────────────────
# 1. Container must be running and not starting/unhealthy
# 2. API must be reachable
# 3. Artist count must be > 0
# 4. Tracked file count must be > 0
# 5. Tracked count must be >= LIDARR_MIN_TRACKED_PCT% of last known count
# 6. Deletion size must be < LIDARR_MAX_DELETE_GB — or --i-know-what-im-doing required
# 3. API version must match tested major version in master.conf
# 4. Artist count must be > 0
# 5. Tracked file count must be > 0
# 6. Tracked count must be >= LIDARR_MIN_TRACKED_PCT % of last known count
# 7. Deletion size must be < LIDARR_MAX_DELETE_GB — or --i-know-what-im-doing required
#
# --i-know-what-im-doing flag:
# Required when deletion would exceed LIDARR_MAX_DELETE_GB
# Long and annoying by design — cannot be added accidentally
# ── OVERRIDE FLAGS ────────────────────────────────────────────────────────────────────────────
# --i-know-what-im-doing
# Required when deletion would exceed LIDARR_MAX_DELETE_GB.
# Long and annoying by design — cannot be added accidentally.
#
# --skip-strike-list flag:
# Bypasses the LIDARR_ORPHAN_AGE age check — deletes recent files too
# Combined with --i-know-what-im-doing activates NUCLEAR MODE:
# Age check bypassed, size threshold bypassed, deletes on first pass
# Use when Soularr/other tool has filled the gaps — clean one-pass wipe
# ⚠️ The script author takes NO responsibility for data loss with both flags active
# The user accepts full responsibility — this is 100% intentional by design
# --skip-strike-list
# Bypasses the LIDARR_ORPHAN_AGE age check — deletes recent files too.
#
# NUCLEAR MODE — both flags active together:
# Age check bypassed, size threshold bypassed, deletes on first pass.
# Use when Soularr has filled the gaps and you want a clean one-pass wipe.
# ⚠️ Script author takes NO responsibility for data loss with both flags active.
# The user accepts full responsibility — this is 100% intentional by design.
#
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
# Lidarr runs on HOST1 only — music library is HOST1's source of truth.
# If run on HOST2 this script exits cleanly with no action.
# All configuration in Master.conf under Arr Cleanup section.
# Supports --dry-run to preview what would be deleted without making changes.
# -----------------------------------------------------------------------------------------------
# detect_hosts() sets MY_ID — if MY_ID != HOST1 script exits cleanly with no action.
# LIDARR_URL, LIDARR_API_KEY, LIDARR_MUSIC_ROOT aliased by detect_hosts() automatically.
#
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
# acquire_lock "wait" — large scans take time, wait for previous run to finish
# jq + curl validation — exits if either tool missing
# DOCKER_TIMEOUT — container checks protected against daemon hangs
# 7 safety layers — all must pass before any file is touched
# Duplicate detection — temp file of tracked paths, grep before delete
# validate_unraid_cmd — notify script validated before use
# Silent by default — orphans/junk warn(), clean library logs silently
#
# ── CONFIGURATION (master_host*.conf) ─────────────────────────────────────────────────────────
# HOST1_LIDARR_URL / HOST1_LIDARR_API_KEY / HOST1_LIDARR_MUSIC_ROOT
# HOST1_LIDARR_PATH_MAP — container path → host path translation
# All aliased by detect_hosts() — script uses unprefixed names
#
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
# LIDARR_LOCK_WARN_AGE — override default lock warning age (large libraries)
# LIDARR_ORPHAN_AGE — days before untracked file is eligible for deletion
# LIDARR_MAX_DELETE_GB — require --i-know-what-im-doing above this
# LIDARR_MIN_TRACKED_PCT — abort if tracked count drops below this % of last run
# LIDARR_TRACKED_COUNT_FILE — persistent baseline file path
# LIDARR_EXTENSIONS — music file extensions to consider for orphan classification
# LIDARR_PROTECTED_PATTERNS — file patterns that are never deleted
# LIDARR_VERSION_MAJOR — expected Lidarr major version for API safety check
# ARR_CLEANUP_STATS — stats file path (read by coffee report)
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
# lidarr_cleanup.sh — normal run
# lidarr_cleanup.sh --dry-run — preview, no deletions
# lidarr_cleanup.sh --log — verbose output
# lidarr_cleanup.sh --status — show config and exit
# lidarr_cleanup.sh --i-know-what-im-doing — bypass size threshold
# lidarr_cleanup.sh --i-know-what-im-doing --skip-strike-list — NUCLEAR MODE
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../Master.conf"
source "$SCRIPT_DIR/../common.sh"
source "$SCRIPT_DIR/../load_config.sh"
# Check for --i-know-what-im-doing and --skip-strike-list flags before parse_args
# These flags are filtered out before parse_args sees them to avoid unknown flag errors
# ── Special flag pre-processing ───────────────────────────────────────────────────────────────
# Filter --i-know-what-im-doing and --skip-strike-list before parse_args
# to avoid unknown flag errors — these are handled separately below.
I_KNOW=false
SKIP_STRIKES=false
FILTERED_ARGS=()
for arg in "$@"; do
if [[ "$arg" == "--i-know-what-im-doing" ]]; then
I_KNOW=true
elif [[ "$arg" == "--skip-strike-list" ]]; then
SKIP_STRIKES=true
else
FILTERED_ARGS+=("$arg")
fi
case "$arg" in
--i-know-what-im-doing) I_KNOW=true ;;
--skip-strike-list) SKIP_STRIKES=true ;;
*) FILTERED_ARGS+=("$arg") ;;
esac
done
parse_args "${FILTERED_ARGS[@]}"
# Nuclear mode — both override flags active
# Strike system AND size threshold bypassed — deletes on first pass
# Script author takes no responsibility for data loss when both flags are used.
# This combination is 100% intentional and the user accepts full responsibility.
# ── Nuclear mode warning ──────────────────────────────────────────────────────────────────────
if [[ "$I_KNOW" == true ]] && [[ "$SKIP_STRIKES" == true ]] && [[ "$DRY_RUN" != true ]]; then
echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "⚠️ WARNING — NUCLEAR MODE ACTIVE"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo " Flags: --i-know-what-im-doing --skip-strike-list"
echo " Strike system: BYPASSED — deletes on first pass"
echo " Size threshold: BYPASSED — no GB limit"
echo " Data recovery: NOT POSSIBLE after deletion"
echo " Flags: --i-know-what-im-doing --skip-strike-list"
echo " Strike system: BYPASSED — deletes on first pass"
echo " Size threshold: BYPASSED — no GB limit"
echo " Data recovery: NOT POSSIBLE after deletion"
echo ""
echo " The script author takes no responsibility for data"
echo " loss when both flags are used together. This is a"
echo " 100% intentional action by the user."
echo ""
echo " Review the dry run output before proceeding."
echo " Review --dry-run output before proceeding."
echo " You have 10 seconds to cancel (Ctrl+C)..."
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
sleep 10
@@ -93,9 +120,9 @@ if [[ "$I_KNOW" == true ]] && [[ "$SKIP_STRIKES" == true ]] && [[ "$DRY_RUN" !=
echo ""
fi
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Setup ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_GEAR Setup ━━━"
@@ -104,8 +131,7 @@ if [[ "$EUID" -ne 0 ]]; then
exit 1
fi
success "Running as root"
# Tool validation — both required, fail fast
if ! command -v curl >/dev/null 2>&1; then
error "curl not found — required for Lidarr API calls"
exit 1
@@ -117,110 +143,115 @@ if ! command -v jq >/dev/null 2>&1; then
exit 1
fi
# Lidarr runs on HOST1 only — exit cleanly if running on HOST2
validate_unraid_cmd \
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
"" "" \
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
# Lock before detect_hosts — large library scans take time, wait mode appropriate
[[ -n "${LIDARR_LOCK_WARN_AGE:-}" ]] && LOCK_WARN_AGE="$LIDARR_LOCK_WARN_AGE"
acquire_lock "wait"
# detect_hosts() sets MY_ID and aliases LIDARR_URL, LIDARR_API_KEY, LIDARR_MUSIC_ROOT
detect_hosts
if [[ "$LOCAL_SERVER_NAME" != "$HOST1" ]]; then
info "Lidarr runs on $HOST1 only — skipping on $LOCAL_SERVER_NAME"
# Lidarr is HOST1 only — exit cleanly on any other host
if [[ "$MY_ID" != "HOST1" ]]; then
log "Lidarr runs on HOST1 only — skipping on $MY_ID ($LOCAL_SERVER_NAME)"
exit 0
fi
LIDARR_URL="$HOST1_LIDARR_URL"
LIDARR_API_KEY="$HOST1_LIDARR_API_KEY"
LIDARR_MUSIC_ROOT="$HOST1_LIDARR_MUSIC_ROOT"
LIDARR_CONTAINER="Lidarr"
DOCKER_TIMEOUT=15
LIDARR_CONTAINER="Lidarr" # container name on HOST1
# Load path map — translates container paths from API to host paths on disk
# Build path map from HOST1_LIDARR_PATH_MAP for translate_path()
declare -A ARR_PATH_MAP
if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then
for key in "${!HOST1_LIDARR_PATH_MAP[@]}"; do
ARR_PATH_MAP["$key"]="${HOST1_LIDARR_PATH_MAP[$key]}"
done
else
for key in "${!HOST2_LIDARR_PATH_MAP[@]}"; do
ARR_PATH_MAP["$key"]="${HOST2_LIDARR_PATH_MAP[$key]}"
done
fi
info "Lidarr instance: $LOCAL_SERVER_NAME → $LIDARR_URL"
info "Music root: $LIDARR_MUSIC_ROOT"
for key in "${!HOST1_LIDARR_PATH_MAP[@]}"; do
ARR_PATH_MAP["$key"]="${HOST1_LIDARR_PATH_MAP[$key]}"
done
# Validate required vars — detect_hosts() should have set these
require_var LIDARR_URL
require_var LIDARR_API_KEY
require_var LIDARR_MUSIC_ROOT
if [[ ! -d "$LIDARR_MUSIC_ROOT" ]]; then
error "Music root not found: $LIDARR_MUSIC_ROOT"
notify "Lidarr cleanup failed on $(hostname) — music root not found: $LIDARR_MUSIC_ROOT" "Lidarr Cleanup" "warning"
notify "Lidarr cleanup failed on $(hostname) — music root not found: $LIDARR_MUSIC_ROOT" \
"Lidarr Cleanup" "warning"
exit 1
fi
# Large library scans take time — override default lock warn age
[[ -n "${LIDARR_LOCK_WARN_AGE:-}" ]] && LOCK_WARN_AGE="$LIDARR_LOCK_WARN_AGE"
log "Lidarr URL: $LIDARR_URL"
log "Music root: $LIDARR_MUSIC_ROOT"
acquire_lock "wait"
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be deleted"
[[ "$I_KNOW" == true ]] && warn "OVERRIDE — --i-know-what-im-doing active"
[[ "$SKIP_STRIKES" == true ]] && warn "OVERRIDE — --skip-strike-list active — age check bypassed"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SHIELD Safety Layer 1 — Container Health ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_SHIELD Safety Checks ━━━"
# Check container is running
CONTAINER_RUNNING=$(docker inspect -f '{{.State.Running}}' "$LIDARR_CONTAINER" 2>/dev/null)
if [[ "$CONTAINER_RUNNING" != "true" ]]; then
error "$LIDARR_CONTAINER container is not running — aborting"
notify "Lidarr cleanup aborted on $(hostname) — Lidarr container not running" "Lidarr Cleanup" "warning"
exit 1
fi
success "$LIDARR_CONTAINER is running"
# Check container health — only fail if explicitly unhealthy
CONTAINER_HEALTH=$(docker inspect -f '{{.State.Health.Status}}' "$LIDARR_CONTAINER" 2>/dev/null)
case "$CONTAINER_HEALTH" in
healthy)
success "$LIDARR_CONTAINER is healthy" ;;
"")
info "$LIDARR_CONTAINER has no health check configured — proceeding" ;;
starting)
error "$LIDARR_CONTAINER is still starting — aborting"
notify "Lidarr cleanup aborted on $(hostname) — Lidarr container still starting" "Lidarr Cleanup" "warning"
exit 1 ;;
unhealthy)
error "$LIDARR_CONTAINER is unhealthy — aborting"
notify "Lidarr cleanup aborted on $(hostname) — Lidarr container unhealthy" "Lidarr Cleanup" "warning"
exit 1 ;;
*)
warn "$LIDARR_CONTAINER health status: $CONTAINER_HEALTH — proceeding with caution" ;;
esac
success "All safety checks passed"
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be deleted"
[[ "$I_KNOW" == true ]] && warn "OVERRIDE — --i-know-what-im-doing flag active"
[[ "$SKIP_STRIKES" == true ]] && warn "OVERRIDE — --skip-strike-list flag active — strike system bypassed"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Status ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
echo ""
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
echo "$ICON_GEAR Lidarr URL: $LIDARR_URL"
echo "$ICON_GEAR Music root: $LIDARR_MUSIC_ROOT"
echo "$ICON_TIME Orphan age: ${LIDARR_ORPHAN_AGE} days"
echo "$ICON_GEAR Max delete: ${LIDARR_MAX_DELETE_GB}GB (requires --i-know-what-im-doing)"
echo "$ICON_GEAR Min tracked %: ${LIDARR_MIN_TRACKED_PCT}% of last run"
echo "$ICON_GEAR Extensions: ${LIDARR_EXTENSIONS[*]}"
echo "$ICON_GEAR Protected patterns: ${LIDARR_PROTECTED_PATTERNS[*]}"
echo "$ICON_GEAR Dry Run: $DRY_RUN"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_GEAR Lidarr URL: $LIDARR_URL"
echo "$ICON_GEAR Music root: $LIDARR_MUSIC_ROOT"
echo "$ICON_TIME Orphan age: ${LIDARR_ORPHAN_AGE} days"
echo "$ICON_GEAR Max delete: ${LIDARR_MAX_DELETE_GB}GB (requires --i-know-what-im-doing)"
echo "$ICON_GEAR Min tracked %: ${LIDARR_MIN_TRACKED_PCT}%"
echo "$ICON_GEAR Lidarr ver: v${LIDARR_VERSION_MAJOR} expected"
echo "$ICON_GEAR Extensions: ${LIDARR_EXTENSIONS[*]}"
echo "$ICON_GEAR Protected patterns: ${LIDARR_PROTECTED_PATTERNS[*]}"
echo "$ICON_GEAR Dry Run: $DRY_RUN"
echo "$ICON_GEAR I know: $I_KNOW"
echo "$ICON_GEAR Skip strikes: $SKIP_STRIKES"
echo "━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
fi
# -----------------------------------------------------------------------------------------------
# HELPERS
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Safety Layer 1 — Container Health ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_SHIELD Safety Checks ━━━"
CONTAINER_RUNNING=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
'{{.State.Running}}' "$LIDARR_CONTAINER" 2>/dev/null)
if [[ "$CONTAINER_RUNNING" != "true" ]]; then
error "$LIDARR_CONTAINER is not running — aborting"
notify "Lidarr cleanup aborted on $(hostname) — container not running" \
"Lidarr Cleanup" "warning"
exit 1
fi
CONTAINER_HEALTH=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
'{{.State.Health.Status}}' "$LIDARR_CONTAINER" 2>/dev/null)
case "$CONTAINER_HEALTH" in
healthy) log "$LIDARR_CONTAINER is healthy" ;;
"") log "$LIDARR_CONTAINER has no health check — proceeding" ;;
starting)
error "$LIDARR_CONTAINER is still starting — aborting"
notify "Lidarr cleanup aborted on $(hostname) — container still starting" \
"Lidarr Cleanup" "warning"
exit 1 ;;
unhealthy)
error "$LIDARR_CONTAINER is unhealthy — aborting"
notify "Lidarr cleanup aborted on $(hostname) — container unhealthy" \
"Lidarr Cleanup" "warning"
exit 1 ;;
*) warn "$LIDARR_CONTAINER health: $CONTAINER_HEALTH — proceeding with caution" ;;
esac
log "Safety layer 1 passed — container healthy"
# ==============================================================================================
# ── HELPER FUNCTIONS ──────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Lidarr API call with HTTP status check
# Usage: lidarr_api "artist" | lidarr_api "trackFile?artistId=123"
lidarr_api() {
local endpoint="$1"
local response http_code body
@@ -235,13 +266,13 @@ lidarr_api() {
body=$(echo "$response" | head -n -1)
if [[ "$http_code" != "200" ]]; then
error "Lidarr API returned HTTP $http_code for endpoint: $endpoint"
error "Lidarr API HTTP $http_code for: $endpoint"
return 1
fi
echo "$body"
}
# Check if a file extension is a tracked music format
is_music_file() {
local ext="${1##*.}"
ext="${ext,,}"
@@ -251,6 +282,7 @@ is_music_file() {
return 1
}
# Check if a file matches any protected pattern
is_protected_file() {
local filename
filename=$(basename "$1")
@@ -263,38 +295,43 @@ is_protected_file() {
return 1
}
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SYNC Fetching Lidarr Tracked Files ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Fetch Lidarr Tracked Files ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_SYNC Fetching Lidarr Tracked Files ━━━"
# Safety Layer 2 — API reachability
check_api "$LIDARR_URL" "Lidarr" || {
if ! check_api "$LIDARR_URL" "Lidarr" 10; then
notify "Lidarr cleanup aborted on $(hostname) — API unreachable" "Lidarr Cleanup" "warning"
exit 1
}
fi
info "Querying Lidarr API: $LIDARR_URL"
# Safety Layer 3 — API version check
check_arr_version "$LIDARR_URL" "$LIDARR_API_KEY" "v1" "$LIDARR_VERSION_MAJOR" "Lidarr" || exit 1
# Step 1 — Get all artists
log "Querying Lidarr API: $LIDARR_URL"
# Fetch all artists
ARTIST_RESPONSE=$(lidarr_api "artist") || {
error "Failed to fetch artists from Lidarr — check URL and API key"
notify "Lidarr cleanup failed on $(hostname) — could not fetch artists" "Lidarr Cleanup" "warning"
error "Failed to fetch artists from Lidarr"
notify "Lidarr cleanup failed on $(hostname) — could not fetch artists" \
"Lidarr Cleanup" "warning"
exit 1
}
ARTIST_IDS=$(echo "$ARTIST_RESPONSE" | jq -r '.[].id' 2>/dev/null)
ARTIST_COUNT=$(echo "$ARTIST_IDS" | grep -c "[0-9]" 2>/dev/null || echo 0)
# Safety Layer 3 — artist count
# Safety Layer 4 — artist count > 0
if [[ "$ARTIST_COUNT" -eq 0 ]]; then
error "No artists returned from Lidarr API — aborting to prevent mass deletion"
notify "Lidarr cleanup aborted on $(hostname) — API returned 0 artists" "Lidarr Cleanup" "warning"
error "API returned 0 artists — aborting to prevent mass deletion"
notify "Lidarr cleanup aborted on $(hostname) — 0 artists returned" \
"Lidarr Cleanup" "warning"
exit 1
fi
info "Found $ARTIST_COUNT artists — fetching track files..."
log "Found $ARTIST_COUNT artists — fetching track files..."
TMP_DIR="/tmp/lidarr_cleanup_$$"
mkdir -p "$TMP_DIR"
@@ -306,57 +343,65 @@ TRACKED_FILE="$TMP_DIR/tracked_paths.txt"
while IFS= read -r artist_id; do
[[ -z "$artist_id" ]] && continue
ARTIST_TRACKS=$(lidarr_api "trackFile?artistId=${artist_id}" 2>/dev/null)
if [[ -n "$ARTIST_TRACKS" ]]; then
if [[ -n "$ARTIST_TRACKS" ]]; then
while IFS= read -r api_path; do
[[ -z "$api_path" ]] && continue
translate_path "$api_path" >> "$TRACKED_FILE"
done < <(echo "$ARTIST_TRACKS" | jq -r '.[].path' 2>/dev/null)
fi
fi
done <<< "$ARTIST_IDS"
sort -u "$TRACKED_FILE" -o "$TRACKED_FILE"
# Build in-memory lookup map — O(1) per lookup vs O(n) grep per file
# Eliminates the main performance bottleneck for large libraries
declare -A TRACKED_MAP
while IFS= read -r _tracked_path; do
[[ -n "$_tracked_path" ]] && TRACKED_MAP["$_tracked_path"]=1
done < "$TRACKED_FILE"
unset _tracked_path
log "Built in-memory lookup map: ${#TRACKED_MAP[@]} tracked paths"
TRACKED_COUNT=$(wc -l < "$TRACKED_FILE")
# Safety Layer 4 — tracked file count
# Safety Layer 5 — tracked count > 0
if [[ "$TRACKED_COUNT" -eq 0 ]]; then
error "API returned 0 tracked files — aborting to prevent mass deletion"
notify "Lidarr cleanup aborted on $(hostname) — API returned 0 tracked files" "Lidarr Cleanup" "warning"
notify "Lidarr cleanup aborted on $(hostname) — 0 tracked files returned" \
"Lidarr Cleanup" "warning"
exit 1
fi
success "Lidarr tracks $TRACKED_COUNT files across $ARTIST_COUNT artists"
warn "Lidarr tracks $TRACKED_COUNT files across $ARTIST_COUNT artists"
# Safety Layer 5 — percentage drop check against last known count
# Safety Layer 6 — percentage drop vs last known count
if [[ -f "$LIDARR_TRACKED_COUNT_FILE" ]]; then
LAST_COUNT=$(cat "$LIDARR_TRACKED_COUNT_FILE" 2>/dev/null || echo 0)
if [[ "$LAST_COUNT" -gt 0 ]]; then
PCT=$(awk "BEGIN {printf \"%d\", ($TRACKED_COUNT / $LAST_COUNT) * 100}")
if [[ "$PCT" -lt "$LIDARR_MIN_TRACKED_PCT" ]]; then
error "Tracked file count dropped to ${PCT}% of last run ($TRACKED_COUNT vs $LAST_COUNT)"
error "This suggests an API issue — aborting to prevent mass deletion"
error "If this is expected (large library removal), delete: $LIDARR_TRACKED_COUNT_FILE"
notify "Lidarr cleanup aborted on $(hostname) — tracked count dropped to ${PCT}% of last run" "Lidarr Cleanup" "warning"
error "Tracked count dropped to ${PCT}% of last run ($TRACKED_COUNT vs $LAST_COUNT)"
error "Suggests API issue — aborting to prevent mass deletion"
error "If expected (large library removal) delete: $LIDARR_TRACKED_COUNT_FILE"
notify "Lidarr cleanup aborted on $(hostname) — tracked count dropped to ${PCT}%" \
"Lidarr Cleanup" "warning"
exit 1
fi
info "Tracked count check: ${PCT}% of last run ($TRACKED_COUNT vs $LAST_COUNT) ✅"
log "Tracked count: ${PCT}% of last run ($TRACKED_COUNT vs $LAST_COUNT) ✅"
fi
else
info "No previous count on record — first run, saving baseline"
log "No previous count on record — first run, saving baseline"
fi
# Save current count for next run comparison
echo "$TRACKED_COUNT" > "$LIDARR_TRACKED_COUNT_FILE"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_CLEAN Scanning Music Root ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Scan Music Root ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_CLEAN Scanning Music Root ━━━"
info "Root: $LIDARR_MUSIC_ROOT"
info "Orphan age: ${LIDARR_ORPHAN_AGE} days"
info "Protected: ${LIDARR_PROTECTED_PATTERNS[*]}"
log "Root: $LIDARR_MUSIC_ROOT"
log "Orphan age: ${LIDARR_ORPHAN_AGE} days"
log "Protected: ${LIDARR_PROTECTED_PATTERNS[*]}"
echo ""
START=$(date +%s)
@@ -374,14 +419,16 @@ MAX_DELETE_BYTES=$(awk "BEGIN {printf \"%d\", $LIDARR_MAX_DELETE_GB * 1073741824
while IFS= read -r filepath; do
[[ -z "$filepath" ]] && continue
if grep -qF "$filepath" "$TRACKED_FILE" 2>/dev/null; then
# Tracked — leave alone
if [[ -n "${TRACKED_MAP[$filepath]:-}" ]]; then
log "TRACKED: $filepath"
continue
fi
# Protected — never delete
if is_protected_file "$filepath"; then
log "$ICON_PROTECTED PROTECTED: $filepath"
((PROTECTED_COUNT++))
(( PROTECTED_COUNT++ ))
continue
fi
@@ -393,17 +440,17 @@ while IFS= read -r filepath; do
if [[ "$FILE_AGE" -lt "$AGE_SECONDS" ]] && [[ "$SKIP_STRIKES" != true ]]; then
log "RECENT (skipping): $filepath"
((RECENT_COUNT++))
(( RECENT_COUNT++ ))
continue
fi
warn "$ICON_TRASH ORPHAN: $filepath"
((ORPHAN_COUNT++))
ORPHAN_BYTES=$((ORPHAN_BYTES + FILE_SIZE))
(( ORPHAN_COUNT++ ))
ORPHAN_BYTES=$(( ORPHAN_BYTES + FILE_SIZE ))
else
log "JUNK: $filepath"
((JUNK_COUNT++))
JUNK_BYTES=$((JUNK_BYTES + FILE_SIZE))
(( JUNK_COUNT++ ))
JUNK_BYTES=$(( JUNK_BYTES + FILE_SIZE ))
fi
done < <(find "$LIDARR_MUSIC_ROOT" -type f 2>/dev/null)
@@ -411,47 +458,48 @@ done < <(find "$LIDARR_MUSIC_ROOT" -type f 2>/dev/null)
TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES ))
TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT ))
# -----------------------------------------------------------------------------------------------
# Safety Layer 6 — deletion size threshold
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Safety Layer 7 — Deletion Size Threshold ━━━
# ==============================================================================================
if [[ "$TOTAL_DELETE_BYTES" -gt "$MAX_DELETE_BYTES" ]]; then
TOTAL_HUMAN=$(awk "BEGIN {printf \"%.1fGB\", $TOTAL_DELETE_BYTES / 1073741824}")
if [[ "$I_KNOW" != true ]]; then
echo ""
error "Deletion would exceed ${LIDARR_MAX_DELETE_GB}GB threshold — $TOTAL_HUMAN would be deleted"
error "Review the ORPHAN lines above carefully before proceeding"
error "If this is expected, rerun with: --i-know-what-im-doing"
error "To also bypass age check and delete on first pass: add --skip-strike-list"
notify "Lidarr cleanup halted on $(hostname) — ${TOTAL_HUMAN} deletion requires --i-know-what-im-doing" "Lidarr Cleanup" "warning"
error "Deletion would exceed ${LIDARR_MAX_DELETE_GB}GB — $TOTAL_HUMAN would be deleted"
error "Review ORPHAN lines above carefully before proceeding"
error "Rerun with: --i-know-what-im-doing"
error "To also bypass age check: add --skip-strike-list"
notify "Lidarr cleanup halted on $(hostname) — ${TOTAL_HUMAN} requires --i-know-what-im-doing" \
"Lidarr Cleanup" "warning"
exit 1
else
warn "OVERRIDE — deletion is ${TOTAL_HUMAN} — proceeding because --i-know-what-im-doing"
warn "OVERRIDE — deletion is $TOTAL_HUMAN — proceeding with --i-know-what-im-doing"
fi
fi
# All safety layers passed — execute deletions
# ── Execute Deletions ─────────────────────────────────────────────────────────────────────────
# All safety layers passed — delete orphans and junk
if [[ "$DRY_RUN" == false ]]; then
while IFS= read -r filepath; do
[[ -z "$filepath" ]] && continue
if grep -qF "$filepath" "$TRACKED_FILE" 2>/dev/null; then continue; fi
if is_protected_file "$filepath"; then continue; fi
[[ -n "${TRACKED_MAP[$filepath]:-}" ]] && continue
is_protected_file "$filepath" && continue
FILE_MTIME=$(stat -c %Y "$filepath" 2>/dev/null || echo 0)
FILE_AGE=$(( NOW - FILE_MTIME ))
if is_music_file "$filepath"; then
[[ "$FILE_AGE" -lt "$AGE_SECONDS" ]] && [[ "$SKIP_STRIKES" != true ]] && continue
[[ "$FILE_AGE" -lt "$AGE_SECONDS" ]] && \
[[ "$SKIP_STRIKES" != true ]] && continue
fi
rm -f "$filepath" 2>/dev/null || error "Failed to delete: $filepath"
done < <(find "$LIDARR_MUSIC_ROOT" -type f 2>/dev/null)
echo ""
info "Cleaning up empty folders..."
log "Cleaning up empty folders..."
find "$LIDARR_MUSIC_ROOT" -mindepth 1 -type d -empty -delete 2>/dev/null
success "Empty folders removed"
log "Empty folders removed"
fi
END=$(date +%s)
@@ -467,35 +515,39 @@ format_bytes() {
fi
}
ORPHAN_HUMAN=$(format_bytes $ORPHAN_BYTES)
JUNK_HUMAN=$(format_bytes $JUNK_BYTES)
ORPHAN_HUMAN=$(format_bytes "$ORPHAN_BYTES")
JUNK_HUMAN=$(format_bytes "$JUNK_BYTES")
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Summary ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
echo ""
echo "━━━━━ $ICON_SUMMARY LIDARR CLEANUP SUMMARY ━━━━━"
echo "$ICON_SYNC Tracked by Lidarr: $TRACKED_COUNT files ($ARTIST_COUNT artists)"
echo "$ICON_SHIELD Protected: $PROTECTED_COUNT files (cover art, metadata)"
echo "$ICON_TRASH Orphans: $ORPHAN_COUNT files ($ORPHAN_HUMAN)"
echo "$ICON_TRASH Junk: $JUNK_COUNT files ($JUNK_HUMAN)"
echo "$ICON_TIME Recent skipped: $RECENT_COUNT files (under ${LIDARR_ORPHAN_AGE} days)"
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_SYNC Tracked: $TRACKED_COUNT files ($ARTIST_COUNT artists)"
echo "$ICON_SHIELD Protected: $PROTECTED_COUNT files (cover art, metadata)"
echo "$ICON_TRASH Orphans: $ORPHAN_COUNT files ($ORPHAN_HUMAN)"
echo "$ICON_TRASH Junk: $JUNK_COUNT files ($JUNK_HUMAN)"
echo "$ICON_SKIP Recent skipped: $RECENT_COUNT files (under ${LIDARR_ORPHAN_AGE} days)"
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
echo ""
if [[ "$DRY_RUN" == true ]]; then
echo "$ICON_WARN Status: DRY RUN — no files deleted"
warn "DRY RUN — no files deleted"
elif [[ "$TOTAL_REMOVED" -eq 0 ]]; then
echo "$ICON_DONE Status: $ICON_SUCCESS CLEAN — nothing to remove"
notify "Lidarr cleanup complete on $(hostname) — library is clean" "Lidarr Cleanup" "normal"
log "$ICON_DONE Clean — nothing to remove"
else
echo "$ICON_DONE Status: $ICON_SUCCESS DONE — $TOTAL_REMOVED files removed (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)"
notify "Lidarr cleanup on $(hostname) — removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" "Lidarr Cleanup" "normal"
warn "$ICON_DONE Removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)"
notify "Lidarr cleanup on $(hostname) — removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" "Lidarr Cleanup" "warning"
# Notify Emby to clean missing files — removes ghost entries immediately
notify_emby_scan
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
# Write stats for sunday_morning_coffee_report.sh
if [[ "$DRY_RUN" == false ]] && [[ -n "${ARR_CLEANUP_STATS:-}" ]]; then
DATE=$(date '+%Y-%m-%d')
echo "${DATE}|lidarr|${ORPHAN_COUNT}|${ORPHAN_BYTES}|${JUNK_COUNT}|${JUNK_BYTES}|${RECENT_COUNT}|${TRACKED_COUNT}" \
echo "$(date '+%Y-%m-%d')|lidarr|${ORPHAN_COUNT}|${ORPHAN_BYTES}|${JUNK_COUNT}|${JUNK_BYTES}|${RECENT_COUNT}|${TRACKED_COUNT}" \
>> "$ARR_CLEANUP_STATS" 2>/dev/null || true
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
Media/media_cleaner.sh
+120 -58
View File
@@ -1,42 +1,81 @@
#!/bin/bash
# -----------------------------------------------------------------------------------------------
# --------------------------------- Media Cleaner Script ---------------------------------------
# -----------------------------------------------------------------------------------------------
# Removes unwanted files from media share folders using configurable file patterns.
# Supports two profiles: anime and media — each with their own folder list and file patterns.
# Profiles and patterns are configured in Master.conf.
# Supports --dry-run to preview what would be deleted without making changes.
# ==============================================================================================
# ================================= Media Cleaner ==============================================
# ==============================================================================================
# Removes unwanted junk files from media share folders using configurable file patterns.
# Two profiles — anime and media — each with their own folder list and file patterns.
# Runs daily via DAILY_MAINTENANCE_SCRIPTS after media_shares_permissions.sh.
#
# Usage:
# media_cleaner.sh anime — clean anime shares
# media_cleaner.sh media — clean media shares
# media_cleaner.sh anime --dry-run — preview anime clean
# -----------------------------------------------------------------------------------------------
# ── PROFILES ──────────────────────────────────────────────────────────────────────────────────
# anime — cleans ANIME_CLEAN_FOLDERS using ANIME_FILE_PATTERNS
# typical targets: *.sfv *.nfo *.url *.rar *.zip *.sample* etc.
#
# media — cleans MEDIA_CLEAN_FOLDERS using MEDIA_FILE_PATTERNS
# same patterns plus *.iso *.lrc (media-specific extras)
#
# ── WHAT IT REMOVES ───────────────────────────────────────────────────────────────────────────
# Junk files left behind by download clients, scene releases, and various tools:
# *.sfv *.md5 *.sha1 — checksum verification files — useless post-download
# *.nfo *.url *.lnk — scene info files — not needed in media library
# *.rar *.zip — archives — source files not needed after extraction
# *.sample* *.proof* — scene samples — never needed
# *sync-conflict* — Syncthing conflict files
# *.scr *.exe — executables — should never be in a media folder
# *.torrent — torrent files left by download clients
# *.log *.json — tool output files
#
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
# detect_hosts() sets MY_ID and aliases HOST*_ANIME_CLEAN_FOLDERS → ANIME_CLEAN_FOLDERS
# and HOST*_MEDIA_CLEAN_FOLDERS → MEDIA_CLEAN_FOLDERS.
# Each server only cleans the shares it owns — correct folders per host automatically.
#
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
# acquire_lock "wait" — wait if previous run still active
# detect_hosts() — correct folder lists per host via MY_ID aliases
# Empty array guards — warns and exits cleanly if no folders or patterns configured
# Folder existence — skips missing folders with warning, continues others
# validate_unraid_cmd — notify script validated before use
# Silent by default — only problems and removals produce output
#
# ── CONFIGURATION (master_host*.conf) ─────────────────────────────────────────────────────────
# HOST*_ANIME_CLEAN_FOLDERS — folders cleaned by the anime profile on this host
# HOST*_MEDIA_CLEAN_FOLDERS — folders cleaned by the media profile on this host
# Aliased by detect_hosts() — script uses ANIME_CLEAN_FOLDERS / MEDIA_CLEAN_FOLDERS
#
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
# ANIME_FILE_PATTERNS — file patterns removed by the anime profile
# MEDIA_FILE_PATTERNS — file patterns removed by the media profile
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
# media_cleaner.sh anime — clean anime shares
# media_cleaner.sh media — clean media shares
# media_cleaner.sh anime --dry-run — preview anime clean, no deletions
# media_cleaner.sh media --dry-run — preview media clean, no deletions
# media_cleaner.sh anime --log — verbose output
# media_cleaner.sh anime --status — show config and exit
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../Master.conf"
source "$SCRIPT_DIR/../common.sh"
source "$SCRIPT_DIR/../load_config.sh"
# -----------------------------------------------------------------------------------------------
# Separate profile argument from flags
# -----------------------------------------------------------------------------------------------
# ── Separate profile argument from flags ──────────────────────────────────────────────────────
# Profile (anime|media) is a positional arg — separate before parse_args sees flags
PROFILE=""
RAW_ARGS=()
for ARG in "$@"; do
case "$ARG" in
--*|*=*) RAW_ARGS+=("$ARG") ;;
anime|media) PROFILE="$ARG" ;;
*) RAW_ARGS+=("$ARG") ;;
*) RAW_ARGS+=("$ARG") ;;
esac
done
parse_args "${RAW_ARGS[@]}"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Setup ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_GEAR Setup ━━━"
@@ -45,19 +84,21 @@ if [[ "$EUID" -ne 0 ]]; then
exit 1
fi
success "Running as root"
if [[ -z "$PROFILE" ]]; then
error "No profile specified"
error "Usage: media_cleaner.sh <anime|media> [--dry-run] [--log] [--status]"
exit 1
fi
acquire_lock "wait"
if ! command -v find >/dev/null 2>&1; then
error "find command not found — check findutils installation"
exit 1
fi
# detect_hosts() sets MY_ID and aliases HOST*_ANIME/MEDIA_CLEAN_FOLDERS
detect_hosts
if [[ -z "$PROFILE" ]]; then
error "No profile specified. Usage: media_cleaner.sh <anime|media> [--dry-run]"
exit 1
fi
validate_unraid_cmd \
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
"" "" \
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
# Resolve profile folders and patterns
case "$PROFILE" in
@@ -75,18 +116,35 @@ case "$PROFILE" in
;;
esac
info "$ICON_GEAR Profile: $PROFILE"
info "$ICON_CLEAN Folders: ${#CLEAN_FOLDERS[@]}"
info "$ICON_TRASH Patterns: ${#FILE_PATTERNS[@]}"
# Empty array guards
if [[ ${#CLEAN_FOLDERS[@]} -eq 0 ]]; then
warn "No folders configured for profile '$PROFILE' on $MY_ID"
warn "Check HOST*_${PROFILE^^}_CLEAN_FOLDERS in master_host*.conf"
exit 0
fi
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Status ━━━
# -----------------------------------------------------------------------------------------------
if [[ ${#FILE_PATTERNS[@]} -eq 0 ]]; then
warn "No file patterns configured for profile '$PROFILE'"
warn "Check ${PROFILE^^}_FILE_PATTERNS in master.conf"
exit 0
fi
log "Profile: $PROFILE"
log "Folders: ${#CLEAN_FOLDERS[@]}"
log "Patterns: ${#FILE_PATTERNS[@]}"
# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
echo ""
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_GEAR Profile: $PROFILE"
echo "$ICON_CLEAN Folders: ${CLEAN_FOLDERS[*]}"
echo "$ICON_CLEAN Folders:"
for f in "${CLEAN_FOLDERS[@]}"; do
echo " $f"
done
echo "$ICON_TRASH Patterns: ${FILE_PATTERNS[*]}"
echo "$ICON_GEAR Dry Run: $DRY_RUN"
echo "━━━━━━━━━━━━━━━━━━━━━━━"
@@ -95,11 +153,11 @@ fi
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be deleted"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_CLEAN Media Cleaner ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Media Cleaner ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_CLEAN Media Cleaner — $PROFILE ━━━"
echo "━━━ $ICON_CLEAN Media Cleaner — $PROFILE — $MY_ID ━━━"
echo ""
START=$(date +%s)
@@ -120,7 +178,7 @@ for FOLDER in "${CLEAN_FOLDERS[@]}"; do
# Build find command dynamically from FILE_PATTERNS array
CMD=(find "$FOLDER" -type f \()
for ((i = 0; i < ${#FILE_PATTERNS[@]}; i++)); do
for (( i = 0; i < ${#FILE_PATTERNS[@]}; i++ )); do
CMD+=(-iname "${FILE_PATTERNS[i]}")
if [[ $i -lt $(( ${#FILE_PATTERNS[@]} - 1 )) ]]; then
CMD+=(-o)
@@ -132,12 +190,12 @@ for FOLDER in "${CLEAN_FOLDERS[@]}"; do
FILE_COUNT=$("${CMD[@]}" 2>/dev/null | wc -l)
if [[ "$FILE_COUNT" -eq 0 ]]; then
success "$FOLDER_NAME — no matching files found"
log "$FOLDER_NAME — clean ✅"
echo ""
continue
fi
info "$ICON_TRASH $FILE_COUNT file(s) found in $FOLDER_NAME"
warn "$ICON_TRASH $FILE_COUNT file(s) to remove from $FOLDER_NAME"
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — files that would be deleted:"
@@ -147,8 +205,8 @@ for FOLDER in "${CLEAN_FOLDERS[@]}"; do
else
CLEAN_CMD=("${CMD[@]}" -exec rm -f {} +)
if "${CLEAN_CMD[@]}" 2>/dev/null; then
success "$FOLDER_NAME — $FILE_COUNT file(s) removed"
TOTAL_REMOVED=$((TOTAL_REMOVED + FILE_COUNT))
log "$FOLDER_NAME — $FILE_COUNT file(s) removed"
TOTAL_REMOVED=$(( TOTAL_REMOVED + FILE_COUNT ))
else
error "$FOLDER_NAME — cleanup failed"
FAILED+=("$FOLDER_NAME")
@@ -160,24 +218,28 @@ done
END=$(date +%s)
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Summary ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
echo "━━━━━ $ICON_SUMMARY MEDIA CLEANER SUMMARY ━━━━━"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_GEAR Profile: $PROFILE"
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
[[ ${#SKIPPED[@]} -gt 0 ]] && echo "$ICON_WARN Skipped: ${SKIPPED[*]}"
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
[[ ${#SKIPPED[@]} -gt 0 ]] && warn "Skipped: ${SKIPPED[*]} (folders not found)"
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}"
if [[ "$DRY_RUN" == true ]]; then
echo "$ICON_WARN Status: DRY RUN — no files deleted"
warn "DRY RUN — no files deleted"
elif [[ ${#FAILED[@]} -gt 0 ]]; then
echo "$ICON_ERROR Status: $ICON_ERROR SOME FOLDERS FAILED"
notify "Media cleaner ($PROFILE) failed on $(hostname) — ${FAILED[*]}" "Media Cleaner" "warning"
echo "$ICON_ERROR Status: SOME FOLDERS FAILED"
notify "Media cleaner ($PROFILE) failed on $(hostname) — ${FAILED[*]}" \
"Media Cleaner" "warning"
elif [[ "$TOTAL_REMOVED" -eq 0 ]]; then
log "$ICON_DONE Status: clean — nothing to remove"
else
echo "$ICON_TRASH Removed: $TOTAL_REMOVED file(s)"
echo "$ICON_DONE Status: $ICON_SUCCESS DONE"
notify "Media cleaner ($PROFILE) complete on $(hostname) — $TOTAL_REMOVED file(s) removed" "Media Cleaner" "normal"
warn "$ICON_TRASH Removed: $TOTAL_REMOVED file(s)"
notify "Media cleaner ($PROFILE) on $(hostname) — $TOTAL_REMOVED file(s) removed" \
"Media Cleaner" "warning"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
Media/media_shares_permissions.sh
+175 -55
View File
@@ -1,22 +1,78 @@
#!/bin/bash
# -----------------------------------------------------------------------------------------------
# --------------------------------- Media Permissions Script -----------------------------------
# -----------------------------------------------------------------------------------------------
# Applies permissions and ownership to all configured media shares.
# Shares, permissions mode and owner are configured in Master.conf.
# Supports --dry-run to preview what would be changed without making changes.
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ============================= Media Shares Permissions =======================================
# ==============================================================================================
# Applies correct ownership and permissions to all configured media shares.
# Runs daily via DAILY_MAINTENANCE_SCRIPTS — first job before arr cleanup scripts.
# Arr cleanup depends on correct ownership to rename and delete files safely.
#
# ── WHY THIS EXISTS ───────────────────────────────────────────────────────────────────────────
# Originally a band-aid for 777 permissions caused by containers running as root.
# Now a proper daily failsafe — even with correct container config, files can arrive
# with wrong ownership from:
# - rsync without --chown (brings source server's ownership)
# - Manual admin copies (creates root:root files)
# - New containers not yet configured with correct PUID/PGID
# - unRAID updates that reset container environments
#
# ── PERMISSIONS MODEL ─────────────────────────────────────────────────────────────────────────
# Directories: 755 nobody:users
# Owner (nobody) — rwx enter, list, create files ✅
# Group (users) — r-x enter and list ✅
# Others — r-x Samba guests can browse ✅
# No world-write — prevents accidental deletion by unauthenticated access
#
# Files: 664 nobody:users
# Owner (nobody) — rw read + write ✅
# Group (users) — rw arrs can import/rename ✅
# Others — r Samba guests can read ✅
# No execute bit — media files are never executable ✅
#
# ── DIAGNOSTIC — HIGH CORRECTED COUNT ─────────────────────────────────────────────────────────
# If this script corrects many files every run, a container has wrong PUID/PGID:
# Correct values on unRAID: PUID=99 (nobody) PGID=100 (users)
# Add to each container's environment in its Docker template
# Common culprits: SABnzbd, qBittorrent, slskd — check these first
# Once fixed, this script should correct 0 files per run (pure failsafe)
#
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
# detect_hosts() sets MY_ID and aliases HOST*_MEDIA_PERMISSION_SHARES → MEDIA_PERMISSION_SHARES
# Each server only applies permissions to the shares it owns.
#
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
# acquire_lock "wait" — wait if previous run still active (large share scans take time)
# detect_hosts() — correct share list per host via MY_ID aliases
# Empty array guard — warns and exits cleanly if no shares configured
# Folder existence — skips missing shares with warning, continues others
# Separate passes — directories and files chmod'd separately for correctness
# validate_unraid_cmd — notify script validated before use
# Silent by default — only failures produce output, success is silent
#
# ── CONFIGURATION (master_host*.conf) ─────────────────────────────────────────────────────────
# HOST*_MEDIA_PERMISSION_SHARES — shares this host applies permissions to
# Aliased by detect_hosts() — script uses MEDIA_PERMISSION_SHARES
#
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
# PERMISSIONS_DIR_MODE — directory permissions (default 755)
# PERMISSIONS_FILE_MODE — file permissions (default 664)
# PERMISSIONS_OWNER — ownership applied to all files (default nobody:users)
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
# media_shares_permissions.sh — normal run
# media_shares_permissions.sh --dry-run — preview without making changes
# media_shares_permissions.sh --log — verbose output
# media_shares_permissions.sh --status — show config and exit
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../Master.conf"
source "$SCRIPT_DIR/../common.sh"
source "$SCRIPT_DIR/../load_config.sh"
parse_args "$@"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Setup ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_GEAR Setup ━━━"
@@ -25,95 +81,159 @@ if [[ "$EUID" -ne 0 ]]; then
exit 1
fi
success "Running as root"
validate_unraid_cmd \
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
"" "" \
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Status ━━━
# -----------------------------------------------------------------------------------------------
if [[ "$SHOW_STATUS" == true ]]; then
echo ""
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
echo "$ICON_PERMS Mode: $PERMISSIONS_MODE"
echo "$ICON_PERMS Owner: $PERMISSIONS_OWNER"
echo "$ICON_PERMS Shares: ${#MEDIA_PERMISSION_SHARES[@]}"
echo "$ICON_GEAR Dry Run: $DRY_RUN"
echo "━━━━━━━━━━━━━━━━━━━━━━━"
acquire_lock "wait"
# detect_hosts() sets MY_ID and aliases HOST*_MEDIA_PERMISSION_SHARES
detect_hosts
# Empty array guard
if [[ ${#MEDIA_PERMISSION_SHARES[@]} -eq 0 ]]; then
warn "MEDIA_PERMISSION_SHARES is empty for $MY_ID — nothing to do"
warn "Check HOST*_MEDIA_PERMISSION_SHARES in master_host*.conf"
exit 0
fi
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
acquire_lock "wait"
# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
echo ""
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_PERMS Dir mode: ${PERMISSIONS_DIR_MODE:-755}"
echo "$ICON_PERMS File mode: ${PERMISSIONS_FILE_MODE:-664}"
echo "$ICON_PERMS Owner: $PERMISSIONS_OWNER"
echo "$ICON_PERMS Shares: ${#MEDIA_PERMISSION_SHARES[@]}"
echo ""
for share in "${MEDIA_PERMISSION_SHARES[@]}"; do
local_status="missing"
[[ -d "$share" ]] && local_status="exists"
echo " $share — $local_status"
done
echo "$ICON_GEAR Dry Run: $DRY_RUN"
echo "━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
fi
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_PERMS Media Permissions ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Apply Permissions ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_PERMS Media Permissions ━━━"
echo "$ICON_PERMS Mode: $PERMISSIONS_MODE"
echo "$ICON_PERMS Owner: $PERMISSIONS_OWNER"
echo "$ICON_PERMS Shares: ${#MEDIA_PERMISSION_SHARES[@]}"
echo "━━━ $ICON_PERMS Media Permissions — $MY_ID ━━━"
log "Dir mode: ${PERMISSIONS_DIR_MODE:-755}"
log "File mode: ${PERMISSIONS_FILE_MODE:-664}"
log "Owner: $PERMISSIONS_OWNER"
log "Shares: ${#MEDIA_PERMISSION_SHARES[@]}"
echo ""
START=$(date +%s)
FAILED=()
UPDATED=()
SKIPPED=()
TOTAL_DIRS_FIXED=0
TOTAL_FILES_FIXED=0
for SHARE in "${MEDIA_PERMISSION_SHARES[@]}"; do
SHARE_NAME=$(basename "$SHARE")
if [[ ! -d "$SHARE" ]]; then
warn "$ICON_PERMS $SHARE_NAME not found — skipping"
warn "$SHARE_NAME not found — skipping"
SKIPPED+=("$SHARE_NAME")
continue
fi
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would apply $PERMISSIONS_MODE $PERMISSIONS_OWNER to $SHARE"
# Count what would be changed without making changes
DIR_COUNT=$(find "$SHARE" -type d ! -perm "${PERMISSIONS_DIR_MODE:-755}" \
2>/dev/null | wc -l)
FILE_COUNT=$(find "$SHARE" -type f ! -perm "${PERMISSIONS_FILE_MODE:-664}" \
2>/dev/null | wc -l)
OWNER_COUNT=$(find "$SHARE" ! -user nobody -o ! -group users \
2>/dev/null | wc -l)
warn "DRY RUN — $SHARE_NAME: $DIR_COUNT dirs, $FILE_COUNT files, $OWNER_COUNT ownership fixes needed"
continue
fi
info "$ICON_PERMS Updating $SHARE_NAME..."
log "Updating $SHARE_NAME..."
CHMOD_OK=true
CHMOD_DIR_OK=true
CHMOD_FILE_OK=true
CHOWN_OK=true
chmod -R "$PERMISSIONS_MODE" "$SHARE" 2>/dev/null || CHMOD_OK=false
# Count files with wrong ownership before fixing (diagnostic)
WRONG_OWNER=$(find "$SHARE" \( ! -user nobody -o ! -group users \) \
2>/dev/null | wc -l)
# Apply ownership first — affects all files and directories
chown -R "$PERMISSIONS_OWNER" "$SHARE" 2>/dev/null || CHOWN_OK=false
if [[ "$CHMOD_OK" == true && "$CHOWN_OK" == true ]]; then
echo "$ICON_UNLOCKED $SHARE_NAME — permissions applied"
# Apply directory permissions — separate pass for correctness
# Directories need execute bit — different from files
find "$SHARE" -type d -exec chmod "${PERMISSIONS_DIR_MODE:-755}" {} + \
2>/dev/null || CHMOD_DIR_OK=false
# Apply file permissions — no execute bit on media files
find "$SHARE" -type f -exec chmod "${PERMISSIONS_FILE_MODE:-664}" {} + \
2>/dev/null || CHMOD_FILE_OK=false
if [[ "$CHMOD_DIR_OK" == true && \
"$CHMOD_FILE_OK" == true && \
"$CHOWN_OK" == true ]]; then
log "$ICON_UNLOCKED $SHARE_NAME — permissions applied"
UPDATED+=("$SHARE_NAME")
# Log diagnostic if many files had wrong ownership
if [[ "$WRONG_OWNER" -gt 0 ]]; then
warn "$SHARE_NAME — corrected $WRONG_OWNER file(s) with wrong ownership"
warn "If this is high, check container PUID/PGID settings (should be PUID=99 PGID=100)"
fi
TOTAL_DIRS_FIXED=$(( TOTAL_DIRS_FIXED + 1 ))
TOTAL_FILES_FIXED=$(( TOTAL_FILES_FIXED + WRONG_OWNER ))
else
error "$SHARE_NAME — permissions failed (chmod=$CHMOD_OK chown=$CHOWN_OK)"
error "$SHARE_NAME — permissions failed"
error " chown: $CHOWN_OK chmod dirs: $CHMOD_DIR_OK chmod files: $CHMOD_FILE_OK"
FAILED+=("$SHARE_NAME")
fi
done
END=$(date +%s)
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Summary ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
echo ""
echo "━━━━━ $ICON_SUMMARY MEDIA PERMISSIONS SUMMARY ━━━━━"
echo "$ICON_PERMS Mode: $PERMISSIONS_MODE"
echo "$ICON_PERMS Owner: $PERMISSIONS_OWNER"
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
echo ""
[[ ${#UPDATED[@]} -gt 0 ]] && echo " $ICON_UNLOCKED Updated: ${#UPDATED[@]}"
[[ ${#SKIPPED[@]} -gt 0 ]] && echo " $ICON_WARN Skipped: ${#SKIPPED[@]}"
[[ ${#FAILED[@]} -gt 0 ]] && echo " $ICON_ERROR Failed: ${#FAILED[@]} — ${FAILED[*]}"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_PERMS Dir mode: ${PERMISSIONS_DIR_MODE:-755}"
echo "$ICON_PERMS File mode: ${PERMISSIONS_FILE_MODE:-664}"
echo "$ICON_PERMS Owner: $PERMISSIONS_OWNER"
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
echo ""
[[ ${#UPDATED[@]} -gt 0 ]] && log "Updated: ${#UPDATED[@]} shares"
[[ ${#SKIPPED[@]} -gt 0 ]] && warn "Skipped: ${SKIPPED[*]} (not found)"
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}"
# Diagnostic — high correction count indicates container PUID/PGID issue
if [[ "$TOTAL_FILES_FIXED" -gt 50 ]]; then
warn "$TOTAL_FILES_FIXED files had wrong ownership this run"
warn "High count suggests a container is not set to PUID=99 PGID=100"
warn "Common culprits: SABnzbd, qBittorrent, slskd — check container env vars"
fi
if [[ "$DRY_RUN" == true ]]; then
echo "$ICON_WARN Status: DRY RUN — no changes made"
warn "DRY RUN — no changes made"
elif [[ ${#FAILED[@]} -gt 0 ]]; then
echo "$ICON_ERROR Status: $ICON_ERROR SOME SHARES FAILED"
notify "Media permissions failed on $(hostname) — ${FAILED[*]}" "Media Permissions" "warning"
echo "$ICON_ERROR Status: SOME SHARES FAILED"
notify "Media permissions failed on $(hostname) — ${FAILED[*]}" \
"Media Permissions" "warning"
else
echo "$ICON_DONE Status: $ICON_SUCCESS DONE"
notify "Media permissions applied on $(hostname) — ${#UPDATED[@]} shares updated" "Media Permissions" "normal"
log "$ICON_DONE Status: done — ${#UPDATED[@]} shares updated"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
Media/radarr_cleanup.sh
+284 -178
View File
@@ -1,67 +1,109 @@
#!/bin/bash
# -----------------------------------------------------------------------------------------------
# --------------------------------- Radarr Cleanup Script --------------------------------------
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ================================= Radarr Cleanup =============================================
# ==============================================================================================
# Removes orphaned movie files from the library that Radarr no longer tracks.
# Uses the Radarr API to build a complete list of tracked movie file paths then compares
# against what exists on disk — anything not tracked and older than RADARR_ORPHAN_AGE
# against what exists on disk — anything untracked and older than RADARR_ORPHAN_AGE
# days is considered an orphan and deleted.
#
# File classification:
# ── FILE CLASSIFICATION ───────────────────────────────────────────────────────────────────────
# TRACKED — Radarr API knows about this exact file path → leave it alone
# PROTECTED — matches RADARR_PROTECTED_PATTERNS → never delete (artwork, subtitles, .nfo)
# ORPHAN — video file, not tracked, older than RADARR_ORPHAN_AGE days → delete
# JUNK — not a video extension, not protected → delete regardless of age
# RECENT — not tracked, under RADARR_ORPHAN_AGE days old → skip (may be mid-import)
#
# Why protected patterns matter:
# Radarr generates movie artwork (*.jpg), metadata (*.nfo) and manages subtitles
# (*.srt, *.sub, *.ass) but does not include these in its tracked file API response.
# Without protection these would be classified as orphans and deleted.
# ── WHY PROTECTED PATTERNS MATTER ─────────────────────────────────────────────────────────────
# Radarr generates movie artwork (*.jpg), metadata (*.nfo) and manages subtitles (*.srt,
# *.sub, *.ass) but does NOT include these in its tracked file API response.
# Without protection these would be classified as orphans and deleted — breaking
# Radarr and Emby metadata display.
#
# ── SAFETY LAYERS — ALL MUST PASS BEFORE ANY FILE IS TOUCHED ─────────────────────────────────
# 1. Container must be running and not starting/unhealthy
# 2. API must be reachable
# 3. API version must match tested major version in master.conf
# 4. Movie count must be > 0
# 5. Tracked file count must be > 0
# 6. Deletion size must be < RADARR_MAX_DELETE_GB — or --i-know-what-im-doing required
#
# ── POST-DELETION ─────────────────────────────────────────────────────────────────────────────
# After files are deleted notify_emby_scan() triggers Emby "Clean Missing Files" task.
# Emby immediately removes ghost entries — no user-facing file-not-found errors.
#
# ── OVERRIDE FLAGS ────────────────────────────────────────────────────────────────────────────
# --i-know-what-im-doing required when deletion exceeds RADARR_MAX_DELETE_GB
# --skip-strike-list bypasses RADARR_ORPHAN_AGE age check
# NUCLEAR MODE — both active: age + size bypass, deletes on first pass
# ⚠️ User accepts full responsibility — no recovery possible after deletion
#
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
# HOST1 Radarr manages Movies. HOST2 Radarr manages Anime_Movies.
# detect_hosts() selects the correct URL, API key, and root path at runtime.
# All configuration in Master.conf under Arr Cleanup section.
# Supports --dry-run to preview what would be deleted without making changes.
# -----------------------------------------------------------------------------------------------
# detect_hosts() sets MY_ID and aliases RADARR_URL, RADARR_API_KEY, RADARR_MOVIES_ROOT.
# No manual HOST1/HOST2 comparisons — MY_ID routes correctly on any server.
#
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
# acquire_lock "wait" — large scans take time, wait for previous run to finish
# jq + curl validation — exits if either tool missing
# DOCKER_TIMEOUT — container checks protected against daemon hangs
# 6 safety layers — all must pass before any file is touched
# notify_emby_scan() — triggers Emby clean after deletion
# validate_unraid_cmd — notify script validated before use
# Silent by default — orphans/junk warn(), clean library logs silently
#
# ── CONFIGURATION (master_host*.conf) ─────────────────────────────────────────────────────────
# HOST*_RADARR_URL / HOST*_RADARR_API_KEY / HOST*_RADARR_MOVIES_ROOT
# HOST*_RADARR_PATH_MAP — container path → host path translation
# All aliased by detect_hosts() — script uses unprefixed names
#
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
# RADARR_ORPHAN_AGE — days before untracked file eligible for deletion
# RADARR_MAX_DELETE_GB — require --i-know-what-im-doing above this
# RADARR_EXTENSIONS — video file extensions considered for orphan classification
# RADARR_PROTECTED_PATTERNS — file patterns never deleted
# RADARR_VERSION_MAJOR — expected Radarr major version for API safety check
# ARR_CLEANUP_STATS — stats file path (read by coffee report)
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
# radarr_cleanup.sh — normal run
# radarr_cleanup.sh --dry-run — preview, no deletions
# radarr_cleanup.sh --log — verbose output
# radarr_cleanup.sh --status — show config and exit
# radarr_cleanup.sh --i-know-what-im-doing — bypass size threshold
# radarr_cleanup.sh --i-know-what-im-doing --skip-strike-list — NUCLEAR MODE
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../Master.conf"
source "$SCRIPT_DIR/../common.sh"
source "$SCRIPT_DIR/../load_config.sh"
# Check for --i-know-what-im-doing and --skip-strike-list flags before parse_args
# ── Special flag pre-processing ───────────────────────────────────────────────────────────────
I_KNOW=false
SKIP_STRIKES=false
FILTERED_ARGS=()
for arg in "$@"; do
if [[ "$arg" == "--i-know-what-im-doing" ]]; then
I_KNOW=true
elif [[ "$arg" == "--skip-strike-list" ]]; then
SKIP_STRIKES=true
else
FILTERED_ARGS+=("$arg")
fi
case "$arg" in
--i-know-what-im-doing) I_KNOW=true ;;
--skip-strike-list) SKIP_STRIKES=true ;;
*) FILTERED_ARGS+=("$arg") ;;
esac
done
parse_args "${FILTERED_ARGS[@]}"
# Nuclear mode disclaimer
# ── Nuclear mode warning ──────────────────────────────────────────────────────────────────────
if [[ "$I_KNOW" == true ]] && [[ "$SKIP_STRIKES" == true ]] && [[ "$DRY_RUN" != true ]]; then
echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "⚠️ WARNING — NUCLEAR MODE ACTIVE"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo " Flags: --i-know-what-im-doing --skip-strike-list"
echo " Strike system: BYPASSED — deletes on first pass"
echo " Size threshold: BYPASSED — no GB limit"
echo " Data recovery: NOT POSSIBLE after deletion"
echo " Flags: --i-know-what-im-doing --skip-strike-list"
echo " Strike system: BYPASSED — deletes on first pass"
echo " Size threshold: BYPASSED — no GB limit"
echo " Data recovery: NOT POSSIBLE after deletion"
echo ""
echo " The script author takes no responsibility for data"
echo " loss when both flags are used together. This is a"
echo " 100% intentional action by the user."
echo ""
echo " Review the dry run output before proceeding."
echo " Review --dry-run output before proceeding."
echo " You have 10 seconds to cancel (Ctrl+C)..."
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
sleep 10
@@ -69,9 +111,9 @@ if [[ "$I_KNOW" == true ]] && [[ "$SKIP_STRIKES" == true ]] && [[ "$DRY_RUN" !=
echo ""
fi
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Setup ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_GEAR Setup ━━━"
@@ -80,8 +122,6 @@ if [[ "$EUID" -ne 0 ]]; then
exit 1
fi
success "Running as root"
if ! command -v curl >/dev/null 2>&1; then
error "curl not found — required for Radarr API calls"
exit 1
@@ -93,33 +133,25 @@ if ! command -v jq >/dev/null 2>&1; then
exit 1
fi
# Select correct Radarr instance based on which server is running this script
validate_unraid_cmd \
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
"" "" \
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
acquire_lock "wait"
# detect_hosts() sets MY_ID and aliases RADARR_URL, RADARR_API_KEY, RADARR_MOVIES_ROOT
detect_hosts
if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then
RADARR_URL="$HOST1_RADARR_URL"
RADARR_API_KEY="$HOST1_RADARR_API_KEY"
RADARR_MOVIES_ROOT="$HOST1_RADARR_MOVIES_ROOT"
else
RADARR_URL="$HOST2_RADARR_URL"
RADARR_API_KEY="$HOST2_RADARR_API_KEY"
RADARR_MOVIES_ROOT="$HOST2_RADARR_MOVIES_ROOT"
fi
DOCKER_TIMEOUT=15
RADARR_CONTAINER="Radarr"
# Load path map for this host
# Build path map from MY_ID's Radarr path map
declare -A ARR_PATH_MAP
if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then
for key in "${!HOST1_RADARR_PATH_MAP[@]}"; do
ARR_PATH_MAP["$key"]="${HOST1_RADARR_PATH_MAP[$key]}"
done
else
for key in "${!HOST2_RADARR_PATH_MAP[@]}"; do
ARR_PATH_MAP["$key"]="${HOST2_RADARR_PATH_MAP[$key]}"
done
fi
info "Radarr instance: $LOCAL_SERVER_NAME → $RADARR_URL"
info "Movies root: $RADARR_MOVIES_ROOT"
local_path_map_var="${MY_ID}_RADARR_PATH_MAP"
eval "for key in \"\${!${local_path_map_var}[@]}\"; do
ARR_PATH_MAP[\"\$key\"]=\"\${${local_path_map_var}[\$key]}\"
done"
require_var RADARR_URL
require_var RADARR_API_KEY
@@ -127,35 +159,77 @@ require_var RADARR_MOVIES_ROOT
if [[ ! -d "$RADARR_MOVIES_ROOT" ]]; then
error "Movies root not found: $RADARR_MOVIES_ROOT"
notify "Radarr cleanup failed on $(hostname) — movies root not found: $RADARR_MOVIES_ROOT" "Radarr Cleanup" "warning"
notify "Radarr cleanup failed on $(hostname) — movies root not found: $RADARR_MOVIES_ROOT" \
"Radarr Cleanup" "warning"
exit 1
fi
acquire_lock "wait"
log "Radarr URL: $RADARR_URL"
log "Movies root: $RADARR_MOVIES_ROOT"
success "Config validated"
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be deleted"
[[ "$I_KNOW" == true ]] && warn "OVERRIDE — --i-know-what-im-doing active"
[[ "$SKIP_STRIKES" == true ]] && warn "OVERRIDE — --skip-strike-list active — age check bypassed"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Status ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
echo ""
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
echo "$ICON_GEAR Radarr URL: $RADARR_URL"
echo "$ICON_GEAR Movies root: $RADARR_MOVIES_ROOT"
echo "$ICON_TIME Orphan age: ${RADARR_ORPHAN_AGE} days"
echo "$ICON_GEAR Extensions: ${RADARR_EXTENSIONS[*]}"
echo "$ICON_GEAR Protected patterns: ${RADARR_PROTECTED_PATTERNS[*]}"
echo "$ICON_GEAR Dry Run: $DRY_RUN"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_GEAR Radarr URL: $RADARR_URL"
echo "$ICON_GEAR Movies root: $RADARR_MOVIES_ROOT"
echo "$ICON_TIME Orphan age: ${RADARR_ORPHAN_AGE} days"
echo "$ICON_GEAR Max delete: ${RADARR_MAX_DELETE_GB}GB (requires --i-know-what-im-doing)"
echo "$ICON_GEAR Radarr ver: v${RADARR_VERSION_MAJOR} expected"
echo "$ICON_GEAR Extensions: ${RADARR_EXTENSIONS[*]}"
echo "$ICON_GEAR Protected patterns: ${RADARR_PROTECTED_PATTERNS[*]}"
echo "$ICON_GEAR Dry Run: $DRY_RUN"
echo "$ICON_GEAR I know: $I_KNOW"
echo "$ICON_GEAR Skip strikes: $SKIP_STRIKES"
echo "━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
fi
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be deleted"
# ==============================================================================================
# ━━━ Safety Layer 1 — Container Health ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_SHIELD Safety Checks ━━━"
# -----------------------------------------------------------------------------------------------
# HELPERS
# -----------------------------------------------------------------------------------------------
CONTAINER_RUNNING=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
'{{.State.Running}}' "$RADARR_CONTAINER" 2>/dev/null)
if [[ "$CONTAINER_RUNNING" != "true" ]]; then
error "$RADARR_CONTAINER is not running — aborting"
notify "Radarr cleanup aborted on $(hostname) — container not running" \
"Radarr Cleanup" "warning"
exit 1
fi
CONTAINER_HEALTH=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
'{{.State.Health.Status}}' "$RADARR_CONTAINER" 2>/dev/null)
case "$CONTAINER_HEALTH" in
healthy) log "$RADARR_CONTAINER is healthy" ;;
"") log "$RADARR_CONTAINER has no health check — proceeding" ;;
starting)
error "$RADARR_CONTAINER is still starting — aborting"
notify "Radarr cleanup aborted on $(hostname) — container still starting" \
"Radarr Cleanup" "warning"
exit 1 ;;
unhealthy)
error "$RADARR_CONTAINER is unhealthy — aborting"
notify "Radarr cleanup aborted on $(hostname) — container unhealthy" \
"Radarr Cleanup" "warning"
exit 1 ;;
*) warn "$RADARR_CONTAINER health: $CONTAINER_HEALTH — proceeding with caution" ;;
esac
log "Safety layer 1 passed — container healthy"
# ==============================================================================================
# ── HELPER FUNCTIONS ──────────────────────────────────────────────────────────────────────────
# ==============================================================================================
radarr_api() {
local endpoint="$1"
@@ -171,10 +245,9 @@ radarr_api() {
body=$(echo "$response" | head -n -1)
if [[ "$http_code" != "200" ]]; then
error "Radarr API returned HTTP $http_code for endpoint: $endpoint"
error "Radarr API HTTP $http_code for: $endpoint"
return 1
fi
echo "$body"
}
@@ -199,36 +272,55 @@ is_protected_file() {
return 1
}
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SYNC Fetching Radarr Tracked Files ━━━
# -----------------------------------------------------------------------------------------------
format_bytes() {
local bytes=$1
if (( bytes > 1073741824 )); then
awk "BEGIN {printf \"%.1fGB\", $bytes / 1073741824}"
elif (( bytes > 1048576 )); then
awk "BEGIN {printf \"%.1fMB\", $bytes / 1048576}"
else
echo "${bytes}B"
fi
}
# ==============================================================================================
# ━━━ Fetch Radarr Tracked Files ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_SYNC Fetching Radarr Tracked Files ━━━"
check_api "$RADARR_URL" "Radarr" || {
# Safety Layer 2 — API reachability
if ! check_api "$RADARR_URL" "Radarr" 10; then
notify "Radarr cleanup aborted on $(hostname) — API unreachable" "Radarr Cleanup" "warning"
exit 1
}
fi
info "Querying Radarr API: $RADARR_URL"
# Safety Layer 3 — API version check
check_arr_version "$RADARR_URL" "$RADARR_API_KEY" "v3" "$RADARR_VERSION_MAJOR" "Radarr" || exit 1
# Step 1 — get all movies to extract IDs
log "Querying Radarr API: $RADARR_URL"
# Fetch all movies
MOVIES_RESPONSE=$(radarr_api "movie") || {
error "Failed to fetch movies from Radarr — check URL and API key"
notify "Radarr cleanup failed on $(hostname) — API unreachable" "Radarr Cleanup" "warning"
error "Failed to fetch movies from Radarr"
notify "Radarr cleanup failed on $(hostname) — could not fetch movies" \
"Radarr Cleanup" "warning"
exit 1
}
MOVIE_IDS=$(echo "$MOVIES_RESPONSE" | jq -r '.[].id' 2>/dev/null)
MOVIE_COUNT=$(echo "$MOVIE_IDS" | grep -c . 2>/dev/null || echo 0)
info "Movies in Radarr: $MOVIE_COUNT"
MOVIE_COUNT=$(echo "$MOVIE_IDS" | grep -c "." 2>/dev/null || echo 0)
# Safety Layer 4 — movie count > 0
if [[ "$MOVIE_COUNT" -eq 0 ]]; then
warn "No movies returned from Radarr — aborting to prevent mass deletion"
notify "Radarr cleanup aborted on $(hostname) — no movies returned from API" "Radarr Cleanup" "warning"
error "API returned 0 movies — aborting to prevent mass deletion"
notify "Radarr cleanup aborted on $(hostname) — 0 movies returned" \
"Radarr Cleanup" "warning"
exit 1
fi
log "Found $MOVIE_COUNT movies — fetching movie files..."
TMP_DIR="/tmp/radarr_cleanup_$$"
mkdir -p "$TMP_DIR"
trap "rm -rf $TMP_DIR" EXIT
@@ -236,12 +328,12 @@ trap "rm -rf $TMP_DIR" EXIT
TRACKED_FILE="$TMP_DIR/tracked_paths.txt"
> "$TRACKED_FILE"
# Step 2 — get movie files per movie ID
MOVIE_INDEX=0
while IFS= read -r movie_id; do
[[ -z "$movie_id" ]] && continue
((MOVIE_INDEX++))
[[ $(( MOVIE_INDEX % 100 )) -eq 0 ]] && info "Fetching files: $MOVIE_INDEX/$MOVIE_COUNT movies..."
(( MOVIE_INDEX++ ))
[[ $(( MOVIE_INDEX % 100 )) -eq 0 ]] && \
log "Fetching files: $MOVIE_INDEX/$MOVIE_COUNT movies..."
MOVIE_FILES=$(radarr_api "moviefile?movieId=${movie_id}" 2>/dev/null)
if [[ -n "$MOVIE_FILES" ]]; then
while IFS= read -r api_path; do
@@ -253,24 +345,34 @@ done <<< "$MOVIE_IDS"
sort -u "$TRACKED_FILE" -o "$TRACKED_FILE"
# Build in-memory lookup map — O(1) per lookup vs O(n) grep per file
# Eliminates the main performance bottleneck for large libraries
declare -A TRACKED_MAP
while IFS= read -r _tracked_path; do
[[ -n "$_tracked_path" ]] && TRACKED_MAP["$_tracked_path"]=1
done < "$TRACKED_FILE"
unset _tracked_path
log "Built in-memory lookup map: ${#TRACKED_MAP[@]} tracked paths"
TRACKED_COUNT=$(wc -l < "$TRACKED_FILE")
success "Radarr tracks $TRACKED_COUNT movie files"
# Safety Layer 5 — tracked count > 0
if [[ "$TRACKED_COUNT" -eq 0 ]]; then
warn "No tracked files returned — Radarr may not have scanned yet or library is empty"
warn "Aborting to prevent mass deletion"
notify "Radarr cleanup aborted on $(hostname) — no tracked files returned from API" "Radarr Cleanup" "warning"
error "API returned 0 tracked files — aborting to prevent mass deletion"
notify "Radarr cleanup aborted on $(hostname) — 0 tracked files returned" \
"Radarr Cleanup" "warning"
exit 1
fi
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_CLEAN Scanning Movies Root ━━━
# -----------------------------------------------------------------------------------------------
warn "Radarr tracks $TRACKED_COUNT movie files across $MOVIE_COUNT movies"
# ==============================================================================================
# ━━━ Scan Movies Root ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_CLEAN Scanning Movies Root ━━━"
info "Root: $RADARR_MOVIES_ROOT"
info "Orphan age: ${RADARR_ORPHAN_AGE} days"
info "Protected: ${RADARR_PROTECTED_PATTERNS[*]}"
log "Root: $RADARR_MOVIES_ROOT"
log "Orphan age: ${RADARR_ORPHAN_AGE} days"
log "Protected: ${RADARR_PROTECTED_PATTERNS[*]}"
echo ""
START=$(date +%s)
@@ -283,18 +385,19 @@ JUNK_BYTES=0
AGE_SECONDS=$(( RADARR_ORPHAN_AGE * 86400 ))
NOW=$(date +%s)
MAX_DELETE_BYTES=$(awk "BEGIN {printf \"%d\", $RADARR_MAX_DELETE_GB * 1073741824}")
while IFS= read -r filepath; do
[[ -z "$filepath" ]] && continue
if grep -qF "$filepath" "$TRACKED_FILE" 2>/dev/null; then
if [[ -n "${TRACKED_MAP[$filepath]:-}" ]]; then
log "TRACKED: $filepath"
continue
fi
if is_protected_file "$filepath"; then
log "$ICON_PROTECTED PROTECTED: $filepath"
((PROTECTED_COUNT++))
(( PROTECTED_COUNT++ ))
continue
fi
@@ -306,111 +409,114 @@ while IFS= read -r filepath; do
if [[ "$FILE_AGE" -lt "$AGE_SECONDS" ]] && [[ "$SKIP_STRIKES" != true ]]; then
log "RECENT (skipping): $filepath"
((RECENT_COUNT++))
(( RECENT_COUNT++ ))
continue
fi
warn "$ICON_TRASH ORPHAN: $filepath"
if [[ "$DRY_RUN" == false ]]; then
rm -f "$filepath" && {
((ORPHAN_COUNT++))
ORPHAN_BYTES=$((ORPHAN_BYTES + FILE_SIZE))
} || error "Failed to delete: $filepath"
else
((ORPHAN_COUNT++))
ORPHAN_BYTES=$((ORPHAN_BYTES + FILE_SIZE))
fi
(( ORPHAN_COUNT++ ))
ORPHAN_BYTES=$(( ORPHAN_BYTES + FILE_SIZE ))
else
log "JUNK: $filepath"
if [[ "$DRY_RUN" == false ]]; then
rm -f "$filepath" && {
((JUNK_COUNT++))
JUNK_BYTES=$((JUNK_BYTES + FILE_SIZE))
} || error "Failed to delete: $filepath"
else
((JUNK_COUNT++))
JUNK_BYTES=$((JUNK_BYTES + FILE_SIZE))
fi
(( JUNK_COUNT++ ))
JUNK_BYTES=$(( JUNK_BYTES + FILE_SIZE ))
fi
done < <(
# Scan all host paths defined in ARR_PATH_MAP — covers all root folders managed by Radarr
for host_path in "${ARR_PATH_MAP[@]}" "$RADARR_MOVIES_ROOT"; do
[[ -d "$host_path" ]] && find "$host_path" -type f 2>/dev/null
done | sort -u
)
if [[ "$DRY_RUN" == false ]]; then
echo ""
info "Cleaning up empty folders..."
for host_path in "${ARR_PATH_MAP[@]}" "$RADARR_MOVIES_ROOT"; do
[[ -d "$host_path" ]] && \
find "$host_path" -mindepth 1 -type d -empty -delete 2>/dev/null
done
success "Empty folders removed"
fi
END=$(date +%s)
format_bytes() {
local bytes=$1
if (( bytes > 1073741824 )); then
awk "BEGIN {printf \"%.1fGB\", $bytes / 1073741824}"
elif (( bytes > 1048576 )); then
awk "BEGIN {printf \"%.1fMB\", $bytes / 1048576}"
else
echo "${bytes}B"
fi
}
ORPHAN_HUMAN=$(format_bytes $ORPHAN_BYTES)
JUNK_HUMAN=$(format_bytes $JUNK_BYTES)
TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT ))
TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES ))
MAX_DELETE_BYTES=$(awk "BEGIN {printf \"%d\", ${RADARR_MAX_DELETE_GB:-1} * 1073741824}")
TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT ))
# Size threshold check
# ==============================================================================================
# ━━━ Safety Layer 6 — Deletion Size Threshold ━━━
# ==============================================================================================
if [[ "$TOTAL_DELETE_BYTES" -gt "$MAX_DELETE_BYTES" ]]; then
TOTAL_HUMAN=$(awk "BEGIN {printf \"%.1fGB\", $TOTAL_DELETE_BYTES / 1073741824}")
if [[ "$I_KNOW" != true ]]; then
echo ""
error "Deletion would exceed ${RADARR_MAX_DELETE_GB:-1}GB threshold — $TOTAL_HUMAN would be deleted"
error "Review the ORPHAN lines above carefully before proceeding"
error "If this is expected, rerun with: --i-know-what-im-doing"
error "To also bypass age check and delete on first pass: add --skip-strike-list"
notify "Radarr cleanup halted on $(hostname) — ${TOTAL_HUMAN} deletion requires --i-know-what-im-doing" "Radarr Cleanup" "warning"
error "Deletion would exceed ${RADARR_MAX_DELETE_GB}GB — $TOTAL_HUMAN would be deleted"
error "Review ORPHAN lines above carefully before proceeding"
error "Rerun with: --i-know-what-im-doing"
error "To also bypass age check: add --skip-strike-list"
notify "Radarr cleanup halted on $(hostname) — ${TOTAL_HUMAN} requires --i-know-what-im-doing" \
"Radarr Cleanup" "warning"
exit 1
else
warn "OVERRIDE — deletion is ${TOTAL_HUMAN} — proceeding because --i-know-what-im-doing"
warn "OVERRIDE — deletion is $TOTAL_HUMAN — proceeding with --i-know-what-im-doing"
fi
fi
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Summary ━━━
# -----------------------------------------------------------------------------------------------
# ── Execute Deletions ─────────────────────────────────────────────────────────────────────────
if [[ "$DRY_RUN" == false ]]; then
while IFS= read -r filepath; do
[[ -z "$filepath" ]] && continue
[[ -n "${TRACKED_MAP[$filepath]:-}" ]] && continue
is_protected_file "$filepath" && continue
FILE_MTIME=$(stat -c %Y "$filepath" 2>/dev/null || echo 0)
FILE_AGE=$(( NOW - FILE_MTIME ))
if is_video_file "$filepath"; then
[[ "$FILE_AGE" -lt "$AGE_SECONDS" ]] && \
[[ "$SKIP_STRIKES" != true ]] && continue
fi
rm -f "$filepath" 2>/dev/null || error "Failed to delete: $filepath"
done < <(
for host_path in "${ARR_PATH_MAP[@]}" "$RADARR_MOVIES_ROOT"; do
[[ -d "$host_path" ]] && find "$host_path" -type f 2>/dev/null
done | sort -u
)
log "Cleaning up empty folders..."
for host_path in "${ARR_PATH_MAP[@]}" "$RADARR_MOVIES_ROOT"; do
[[ -d "$host_path" ]] && \
find "$host_path" -mindepth 1 -type d -empty -delete 2>/dev/null
done
log "Empty folders removed"
fi
END=$(date +%s)
ORPHAN_HUMAN=$(format_bytes "$ORPHAN_BYTES")
JUNK_HUMAN=$(format_bytes "$JUNK_BYTES")
# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
echo ""
echo "━━━━━ $ICON_SUMMARY RADARR CLEANUP SUMMARY ━━━━━"
echo "$ICON_SYNC Tracked by Radarr: $TRACKED_COUNT files"
echo "$ICON_SHIELD Protected: $PROTECTED_COUNT files (artwork, subtitles, metadata)"
echo "$ICON_TRASH Orphans removed: $ORPHAN_COUNT files ($ORPHAN_HUMAN)"
echo "$ICON_TRASH Junk removed: $JUNK_COUNT files ($JUNK_HUMAN)"
echo "$ICON_TIME Recent skipped: $RECENT_COUNT files (under ${RADARR_ORPHAN_AGE} days)"
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_SYNC Tracked: $TRACKED_COUNT files ($MOVIE_COUNT movies)"
echo "$ICON_SHIELD Protected: $PROTECTED_COUNT files (artwork, subtitles, metadata)"
echo "$ICON_TRASH Orphans: $ORPHAN_COUNT files ($ORPHAN_HUMAN)"
echo "$ICON_TRASH Junk: $JUNK_COUNT files ($JUNK_HUMAN)"
echo "$ICON_SKIP Recent skipped: $RECENT_COUNT files (under ${RADARR_ORPHAN_AGE} days)"
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
echo ""
if [[ "$DRY_RUN" == true ]]; then
echo "$ICON_WARN Status: DRY RUN — no files deleted"
warn "DRY RUN — no files deleted"
elif [[ "$TOTAL_REMOVED" -eq 0 ]]; then
echo "$ICON_DONE Status: $ICON_SUCCESS CLEAN — nothing to remove"
notify "Radarr cleanup complete on $(hostname) — library is clean" "Radarr Cleanup" "normal"
log "$ICON_DONE Clean — nothing to remove"
else
echo "$ICON_DONE Status: $ICON_SUCCESS DONE — $TOTAL_REMOVED files removed"
notify "Radarr cleanup on $(hostname) — removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" "Radarr Cleanup" "normal"
warn "$ICON_DONE Removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)"
notify "Radarr cleanup on $(hostname) — removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" \
"Radarr Cleanup" "warning"
# Notify Emby to clean missing files — removes ghost entries immediately
notify_emby_scan
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
# Write stats for sunday_morning_coffee_report.sh
if [[ "$DRY_RUN" == false ]] && [[ -n "${ARR_CLEANUP_STATS:-}" ]]; then
DATE=$(date '+%Y-%m-%d')
echo "${DATE}|radarr|${ORPHAN_COUNT}|${ORPHAN_BYTES}|${JUNK_COUNT}|${JUNK_BYTES}|${RECENT_COUNT}|${TRACKED_COUNT}" \
echo "$(date '+%Y-%m-%d')|radarr|${ORPHAN_COUNT}|${ORPHAN_BYTES}|${JUNK_COUNT}|${JUNK_BYTES}|${RECENT_COUNT}|${TRACKED_COUNT}" \
>> "$ARR_CLEANUP_STATS" 2>/dev/null || true
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
Media/sonarr_cleanup.sh
+275 -178
View File
@@ -1,67 +1,109 @@
#!/bin/bash
# -----------------------------------------------------------------------------------------------
# --------------------------------- Sonarr Cleanup Script --------------------------------------
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ================================= Sonarr Cleanup =============================================
# ==============================================================================================
# Removes orphaned TV episode files from the library that Sonarr no longer tracks.
# Uses the Sonarr API to build a complete list of tracked episode file paths then compares
# against what exists on disk — anything not tracked and older than SONARR_ORPHAN_AGE
# against what exists on disk — anything untracked and older than SONARR_ORPHAN_AGE
# days is considered an orphan and deleted.
#
# File classification:
# ── FILE CLASSIFICATION ───────────────────────────────────────────────────────────────────────
# TRACKED — Sonarr API knows about this exact file path → leave it alone
# PROTECTED — matches SONARR_PROTECTED_PATTERNS → never delete (artwork, subtitles, .nfo)
# ORPHAN — video file, not tracked, older than SONARR_ORPHAN_AGE days → delete
# JUNK — not a video extension, not protected → delete regardless of age
# RECENT — not tracked, under SONARR_ORPHAN_AGE days old → skip (may be mid-import)
#
# Why protected patterns matter:
# Sonarr generates show artwork (*.jpg), metadata (*.nfo) and manages subtitles
# (*.srt, *.sub, *.ass) but does not include these in its tracked file API response.
# Without protection these would be classified as orphans and deleted.
# ── WHY PROTECTED PATTERNS MATTER ─────────────────────────────────────────────────────────────
# Sonarr generates show artwork (*.jpg), metadata (*.nfo) and manages subtitles (*.srt,
# *.sub, *.ass) but does NOT include these in its tracked file API response.
# Without protection these would be classified as orphans and deleted — breaking
# Sonarr and Emby metadata display.
#
# ── SAFETY LAYERS — ALL MUST PASS BEFORE ANY FILE IS TOUCHED ─────────────────────────────────
# 1. Container must be running and not starting/unhealthy
# 2. API must be reachable
# 3. API version must match tested major version in master.conf
# 4. Series count must be > 0
# 5. Tracked file count must be > 0
# 6. Deletion size must be < SONARR_MAX_DELETE_GB — or --i-know-what-im-doing required
#
# ── POST-DELETION ─────────────────────────────────────────────────────────────────────────────
# After files are deleted notify_emby_scan() triggers Emby "Clean Missing Files" task.
# Emby immediately removes ghost entries — no user-facing file-not-found errors.
#
# ── OVERRIDE FLAGS ────────────────────────────────────────────────────────────────────────────
# --i-know-what-im-doing required when deletion exceeds SONARR_MAX_DELETE_GB
# --skip-strike-list bypasses SONARR_ORPHAN_AGE age check
# NUCLEAR MODE — both active: age + size bypass, deletes on first pass
# ⚠️ User accepts full responsibility — no recovery possible after deletion
#
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
# HOST1 Sonarr manages Tv_Shows. HOST2 Sonarr manages Anime_Shows.
# detect_hosts() selects the correct URL, API key, and root path at runtime.
# All configuration in Master.conf under Arr Cleanup section.
# Supports --dry-run to preview what would be deleted without making changes.
# -----------------------------------------------------------------------------------------------
# detect_hosts() sets MY_ID and aliases SONARR_URL, SONARR_API_KEY, SONARR_TV_ROOT.
# No manual HOST1/HOST2 comparisons — MY_ID routes correctly on any server.
#
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
# acquire_lock "wait" — large scans take time, wait for previous run to finish
# jq + curl validation — exits if either tool missing
# DOCKER_TIMEOUT — container checks protected against daemon hangs
# 6 safety layers — all must pass before any file is touched
# notify_emby_scan() — triggers Emby clean after deletion
# validate_unraid_cmd — notify script validated before use
# Silent by default — orphans/junk warn(), clean library logs silently
#
# ── CONFIGURATION (master_host*.conf) ─────────────────────────────────────────────────────────
# HOST*_SONARR_URL / HOST*_SONARR_API_KEY / HOST*_SONARR_TV_ROOT
# HOST*_SONARR_PATH_MAP — container path → host path translation
# All aliased by detect_hosts() — script uses unprefixed names
#
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
# SONARR_ORPHAN_AGE — days before untracked file eligible for deletion
# SONARR_MAX_DELETE_GB — require --i-know-what-im-doing above this
# SONARR_EXTENSIONS — video file extensions considered for orphan classification
# SONARR_PROTECTED_PATTERNS — file patterns never deleted
# SONARR_VERSION_MAJOR — expected Sonarr major version for API safety check
# ARR_CLEANUP_STATS — stats file path (read by coffee report)
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
# sonarr_cleanup.sh — normal run
# sonarr_cleanup.sh --dry-run — preview, no deletions
# sonarr_cleanup.sh --log — verbose output
# sonarr_cleanup.sh --status — show config and exit
# sonarr_cleanup.sh --i-know-what-im-doing — bypass size threshold
# sonarr_cleanup.sh --i-know-what-im-doing --skip-strike-list — NUCLEAR MODE
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../Master.conf"
source "$SCRIPT_DIR/../common.sh"
source "$SCRIPT_DIR/../load_config.sh"
# Check for --i-know-what-im-doing and --skip-strike-list flags before parse_args
# ── Special flag pre-processing ───────────────────────────────────────────────────────────────
I_KNOW=false
SKIP_STRIKES=false
FILTERED_ARGS=()
for arg in "$@"; do
if [[ "$arg" == "--i-know-what-im-doing" ]]; then
I_KNOW=true
elif [[ "$arg" == "--skip-strike-list" ]]; then
SKIP_STRIKES=true
else
FILTERED_ARGS+=("$arg")
fi
case "$arg" in
--i-know-what-im-doing) I_KNOW=true ;;
--skip-strike-list) SKIP_STRIKES=true ;;
*) FILTERED_ARGS+=("$arg") ;;
esac
done
parse_args "${FILTERED_ARGS[@]}"
# Nuclear mode disclaimer
# ── Nuclear mode warning ──────────────────────────────────────────────────────────────────────
if [[ "$I_KNOW" == true ]] && [[ "$SKIP_STRIKES" == true ]] && [[ "$DRY_RUN" != true ]]; then
echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "⚠️ WARNING — NUCLEAR MODE ACTIVE"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo " Flags: --i-know-what-im-doing --skip-strike-list"
echo " Strike system: BYPASSED — deletes on first pass"
echo " Size threshold: BYPASSED — no GB limit"
echo " Data recovery: NOT POSSIBLE after deletion"
echo " Flags: --i-know-what-im-doing --skip-strike-list"
echo " Strike system: BYPASSED — deletes on first pass"
echo " Size threshold: BYPASSED — no GB limit"
echo " Data recovery: NOT POSSIBLE after deletion"
echo ""
echo " The script author takes no responsibility for data"
echo " loss when both flags are used together. This is a"
echo " 100% intentional action by the user."
echo ""
echo " Review the dry run output before proceeding."
echo " Review --dry-run output before proceeding."
echo " You have 10 seconds to cancel (Ctrl+C)..."
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
sleep 10
@@ -69,9 +111,9 @@ if [[ "$I_KNOW" == true ]] && [[ "$SKIP_STRIKES" == true ]] && [[ "$DRY_RUN" !=
echo ""
fi
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Setup ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_GEAR Setup ━━━"
@@ -80,8 +122,6 @@ if [[ "$EUID" -ne 0 ]]; then
exit 1
fi
success "Running as root"
if ! command -v curl >/dev/null 2>&1; then
error "curl not found — required for Sonarr API calls"
exit 1
@@ -93,33 +133,25 @@ if ! command -v jq >/dev/null 2>&1; then
exit 1
fi
# Select correct Sonarr instance based on which server is running this script
validate_unraid_cmd \
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
"" "" \
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
acquire_lock "wait"
# detect_hosts() sets MY_ID and aliases SONARR_URL, SONARR_API_KEY, SONARR_TV_ROOT
detect_hosts
if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then
SONARR_URL="$HOST1_SONARR_URL"
SONARR_API_KEY="$HOST1_SONARR_API_KEY"
SONARR_TV_ROOT="$HOST1_SONARR_TV_ROOT"
else
SONARR_URL="$HOST2_SONARR_URL"
SONARR_API_KEY="$HOST2_SONARR_API_KEY"
SONARR_TV_ROOT="$HOST2_SONARR_TV_ROOT"
fi
DOCKER_TIMEOUT=15
SONARR_CONTAINER="Sonarr"
# Load path map for this host
# Build path map from MY_ID's Sonarr path map
declare -A ARR_PATH_MAP
if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then
for key in "${!HOST1_SONARR_PATH_MAP[@]}"; do
ARR_PATH_MAP["$key"]="${HOST1_SONARR_PATH_MAP[$key]}"
done
else
for key in "${!HOST2_SONARR_PATH_MAP[@]}"; do
ARR_PATH_MAP["$key"]="${HOST2_SONARR_PATH_MAP[$key]}"
done
fi
info "Sonarr instance: $LOCAL_SERVER_NAME → $SONARR_URL"
info "TV root: $SONARR_TV_ROOT"
local_path_map_var="${MY_ID}_SONARR_PATH_MAP"
eval "for key in \"\${!${local_path_map_var}[@]}\"; do
ARR_PATH_MAP[\"\$key\"]=\"\${${local_path_map_var}[\$key]}\"
done"
require_var SONARR_URL
require_var SONARR_API_KEY
@@ -127,35 +159,77 @@ require_var SONARR_TV_ROOT
if [[ ! -d "$SONARR_TV_ROOT" ]]; then
error "TV root not found: $SONARR_TV_ROOT"
notify "Sonarr cleanup failed on $(hostname) — TV root not found: $SONARR_TV_ROOT" "Sonarr Cleanup" "warning"
notify "Sonarr cleanup failed on $(hostname) — TV root not found: $SONARR_TV_ROOT" \
"Sonarr Cleanup" "warning"
exit 1
fi
acquire_lock "wait"
log "Sonarr URL: $SONARR_URL"
log "TV root: $SONARR_TV_ROOT"
success "Config validated"
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be deleted"
[[ "$I_KNOW" == true ]] && warn "OVERRIDE — --i-know-what-im-doing active"
[[ "$SKIP_STRIKES" == true ]] && warn "OVERRIDE — --skip-strike-list active — age check bypassed"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Status ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
echo ""
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
echo "$ICON_GEAR Sonarr URL: $SONARR_URL"
echo "$ICON_GEAR TV root: $SONARR_TV_ROOT"
echo "$ICON_TIME Orphan age: ${SONARR_ORPHAN_AGE} days"
echo "$ICON_GEAR Extensions: ${SONARR_EXTENSIONS[*]}"
echo "$ICON_GEAR Protected patterns: ${SONARR_PROTECTED_PATTERNS[*]}"
echo "$ICON_GEAR Dry Run: $DRY_RUN"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_GEAR Sonarr URL: $SONARR_URL"
echo "$ICON_GEAR TV root: $SONARR_TV_ROOT"
echo "$ICON_TIME Orphan age: ${SONARR_ORPHAN_AGE} days"
echo "$ICON_GEAR Max delete: ${SONARR_MAX_DELETE_GB}GB (requires --i-know-what-im-doing)"
echo "$ICON_GEAR Sonarr ver: v${SONARR_VERSION_MAJOR} expected"
echo "$ICON_GEAR Extensions: ${SONARR_EXTENSIONS[*]}"
echo "$ICON_GEAR Protected patterns: ${SONARR_PROTECTED_PATTERNS[*]}"
echo "$ICON_GEAR Dry Run: $DRY_RUN"
echo "$ICON_GEAR I know: $I_KNOW"
echo "$ICON_GEAR Skip strikes: $SKIP_STRIKES"
echo "━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
fi
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be deleted"
# ==============================================================================================
# ━━━ Safety Layer 1 — Container Health ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_SHIELD Safety Checks ━━━"
# -----------------------------------------------------------------------------------------------
# HELPERS
# -----------------------------------------------------------------------------------------------
CONTAINER_RUNNING=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
'{{.State.Running}}' "$SONARR_CONTAINER" 2>/dev/null)
if [[ "$CONTAINER_RUNNING" != "true" ]]; then
error "$SONARR_CONTAINER is not running — aborting"
notify "Sonarr cleanup aborted on $(hostname) — container not running" \
"Sonarr Cleanup" "warning"
exit 1
fi
CONTAINER_HEALTH=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
'{{.State.Health.Status}}' "$SONARR_CONTAINER" 2>/dev/null)
case "$CONTAINER_HEALTH" in
healthy) log "$SONARR_CONTAINER is healthy" ;;
"") log "$SONARR_CONTAINER has no health check — proceeding" ;;
starting)
error "$SONARR_CONTAINER is still starting — aborting"
notify "Sonarr cleanup aborted on $(hostname) — container still starting" \
"Sonarr Cleanup" "warning"
exit 1 ;;
unhealthy)
error "$SONARR_CONTAINER is unhealthy — aborting"
notify "Sonarr cleanup aborted on $(hostname) — container unhealthy" \
"Sonarr Cleanup" "warning"
exit 1 ;;
*) warn "$SONARR_CONTAINER health: $CONTAINER_HEALTH — proceeding with caution" ;;
esac
log "Safety layer 1 passed — container healthy"
# ==============================================================================================
# ── HELPER FUNCTIONS ──────────────────────────────────────────────────────────────────────────
# ==============================================================================================
sonarr_api() {
local endpoint="$1"
@@ -171,10 +245,9 @@ sonarr_api() {
body=$(echo "$response" | head -n -1)
if [[ "$http_code" != "200" ]]; then
error "Sonarr API returned HTTP $http_code for endpoint: $endpoint"
error "Sonarr API HTTP $http_code for: $endpoint"
return 1
fi
echo "$body"
}
@@ -199,36 +272,55 @@ is_protected_file() {
return 1
}
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SYNC Fetching Sonarr Tracked Files ━━━
# -----------------------------------------------------------------------------------------------
format_bytes() {
local bytes=$1
if (( bytes > 1073741824 )); then
awk "BEGIN {printf \"%.1fGB\", $bytes / 1073741824}"
elif (( bytes > 1048576 )); then
awk "BEGIN {printf \"%.1fMB\", $bytes / 1048576}"
else
echo "${bytes}B"
fi
}
# ==============================================================================================
# ━━━ Fetch Sonarr Tracked Files ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_SYNC Fetching Sonarr Tracked Files ━━━"
check_api "$SONARR_URL" "Sonarr" || {
# Safety Layer 2 — API reachability
if ! check_api "$SONARR_URL" "Sonarr" 10; then
notify "Sonarr cleanup aborted on $(hostname) — API unreachable" "Sonarr Cleanup" "warning"
exit 1
}
fi
info "Querying Sonarr API: $SONARR_URL"
# Safety Layer 3 — API version check
check_arr_version "$SONARR_URL" "$SONARR_API_KEY" "v3" "$SONARR_VERSION_MAJOR" "Sonarr" || exit 1
# Step 1 — get all series to extract IDs
log "Querying Sonarr API: $SONARR_URL"
# Fetch all series
SERIES_RESPONSE=$(sonarr_api "series") || {
error "Failed to fetch series from Sonarr — check URL and API key"
notify "Sonarr cleanup failed on $(hostname) — API unreachable" "Sonarr Cleanup" "warning"
error "Failed to fetch series from Sonarr"
notify "Sonarr cleanup failed on $(hostname) — could not fetch series" \
"Sonarr Cleanup" "warning"
exit 1
}
SERIES_IDS=$(echo "$SERIES_RESPONSE" | jq -r '.[].id' 2>/dev/null)
SERIES_COUNT=$(echo "$SERIES_IDS" | grep -c . 2>/dev/null || echo 0)
info "Series in Sonarr: $SERIES_COUNT"
SERIES_COUNT=$(echo "$SERIES_IDS" | grep -c "." 2>/dev/null || echo 0)
# Safety Layer 4 — series count > 0
if [[ "$SERIES_COUNT" -eq 0 ]]; then
warn "No series returned from Sonarr — aborting to prevent mass deletion"
notify "Sonarr cleanup aborted on $(hostname) — no series returned from API" "Sonarr Cleanup" "warning"
error "API returned 0 series — aborting to prevent mass deletion"
notify "Sonarr cleanup aborted on $(hostname) — 0 series returned" \
"Sonarr Cleanup" "warning"
exit 1
fi
log "Found $SERIES_COUNT series — fetching episode files..."
TMP_DIR="/tmp/sonarr_cleanup_$$"
mkdir -p "$TMP_DIR"
trap "rm -rf $TMP_DIR" EXIT
@@ -236,12 +328,12 @@ trap "rm -rf $TMP_DIR" EXIT
TRACKED_FILE="$TMP_DIR/tracked_paths.txt"
> "$TRACKED_FILE"
# Step 2 — get episode files per series ID
SERIES_INDEX=0
while IFS= read -r series_id; do
[[ -z "$series_id" ]] && continue
((SERIES_INDEX++))
[[ $(( SERIES_INDEX % 50 )) -eq 0 ]] && info "Fetching files: $SERIES_INDEX/$SERIES_COUNT series..."
(( SERIES_INDEX++ ))
[[ $(( SERIES_INDEX % 50 )) -eq 0 ]] && \
log "Fetching files: $SERIES_INDEX/$SERIES_COUNT series..."
SERIES_FILES=$(sonarr_api "episodefile?seriesId=${series_id}" 2>/dev/null)
if [[ -n "$SERIES_FILES" ]]; then
while IFS= read -r api_path; do
@@ -252,25 +344,26 @@ while IFS= read -r series_id; do
done <<< "$SERIES_IDS"
sort -u "$TRACKED_FILE" -o "$TRACKED_FILE"
TRACKED_COUNT=$(wc -l < "$TRACKED_FILE")
success "Sonarr tracks $TRACKED_COUNT episode files"
# Safety Layer 5 — tracked count > 0
if [[ "$TRACKED_COUNT" -eq 0 ]]; then
warn "No tracked files returned — Sonarr may not have scanned yet or library is empty"
warn "Aborting to prevent mass deletion"
notify "Sonarr cleanup aborted on $(hostname) — no tracked files returned from API" "Sonarr Cleanup" "warning"
error "API returned 0 tracked files — aborting to prevent mass deletion"
notify "Sonarr cleanup aborted on $(hostname) — 0 tracked files returned" \
"Sonarr Cleanup" "warning"
exit 1
fi
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_CLEAN Scanning TV Root ━━━
# -----------------------------------------------------------------------------------------------
warn "Sonarr tracks $TRACKED_COUNT episode files across $SERIES_COUNT series"
# ==============================================================================================
# ━━━ Scan TV Root ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_CLEAN Scanning TV Root ━━━"
info "Root: $SONARR_TV_ROOT"
info "Orphan age: ${SONARR_ORPHAN_AGE} days"
info "Protected: ${SONARR_PROTECTED_PATTERNS[*]}"
log "Root: $SONARR_TV_ROOT"
log "Orphan age: ${SONARR_ORPHAN_AGE} days"
log "Protected: ${SONARR_PROTECTED_PATTERNS[*]}"
echo ""
START=$(date +%s)
@@ -283,6 +376,7 @@ JUNK_BYTES=0
AGE_SECONDS=$(( SONARR_ORPHAN_AGE * 86400 ))
NOW=$(date +%s)
MAX_DELETE_BYTES=$(awk "BEGIN {printf \"%d\", $SONARR_MAX_DELETE_GB * 1073741824}")
while IFS= read -r filepath; do
[[ -z "$filepath" ]] && continue
@@ -294,7 +388,7 @@ while IFS= read -r filepath; do
if is_protected_file "$filepath"; then
log "$ICON_PROTECTED PROTECTED: $filepath"
((PROTECTED_COUNT++))
(( PROTECTED_COUNT++ ))
continue
fi
@@ -306,111 +400,114 @@ while IFS= read -r filepath; do
if [[ "$FILE_AGE" -lt "$AGE_SECONDS" ]] && [[ "$SKIP_STRIKES" != true ]]; then
log "RECENT (skipping): $filepath"
((RECENT_COUNT++))
(( RECENT_COUNT++ ))
continue
fi
warn "$ICON_TRASH ORPHAN: $filepath"
if [[ "$DRY_RUN" == false ]]; then
rm -f "$filepath" && {
((ORPHAN_COUNT++))
ORPHAN_BYTES=$((ORPHAN_BYTES + FILE_SIZE))
} || error "Failed to delete: $filepath"
else
((ORPHAN_COUNT++))
ORPHAN_BYTES=$((ORPHAN_BYTES + FILE_SIZE))
fi
(( ORPHAN_COUNT++ ))
ORPHAN_BYTES=$(( ORPHAN_BYTES + FILE_SIZE ))
else
log "JUNK: $filepath"
if [[ "$DRY_RUN" == false ]]; then
rm -f "$filepath" && {
((JUNK_COUNT++))
JUNK_BYTES=$((JUNK_BYTES + FILE_SIZE))
} || error "Failed to delete: $filepath"
else
((JUNK_COUNT++))
JUNK_BYTES=$((JUNK_BYTES + FILE_SIZE))
fi
(( JUNK_COUNT++ ))
JUNK_BYTES=$(( JUNK_BYTES + FILE_SIZE ))
fi
done < <(
# Scan all host paths defined in ARR_PATH_MAP — covers all root folders managed by Sonarr
for host_path in "${ARR_PATH_MAP[@]}" "$SONARR_TV_ROOT"; do
[[ -d "$host_path" ]] && find "$host_path" -type f 2>/dev/null
done | sort -u
)
if [[ "$DRY_RUN" == false ]]; then
echo ""
info "Cleaning up empty folders..."
for host_path in "${ARR_PATH_MAP[@]}" "$SONARR_TV_ROOT"; do
[[ -d "$host_path" ]] && \
find "$host_path" -mindepth 1 -type d -empty -delete 2>/dev/null
done
success "Empty folders removed"
fi
END=$(date +%s)
format_bytes() {
local bytes=$1
if (( bytes > 1073741824 )); then
awk "BEGIN {printf \"%.1fGB\", $bytes / 1073741824}"
elif (( bytes > 1048576 )); then
awk "BEGIN {printf \"%.1fMB\", $bytes / 1048576}"
else
echo "${bytes}B"
fi
}
ORPHAN_HUMAN=$(format_bytes $ORPHAN_BYTES)
JUNK_HUMAN=$(format_bytes $JUNK_BYTES)
TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT ))
TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES ))
MAX_DELETE_BYTES=$(awk "BEGIN {printf \"%d\", ${SONARR_MAX_DELETE_GB:-1} * 1073741824}")
TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT ))
# Size threshold check
# ==============================================================================================
# ━━━ Safety Layer 6 — Deletion Size Threshold ━━━
# ==============================================================================================
if [[ "$TOTAL_DELETE_BYTES" -gt "$MAX_DELETE_BYTES" ]]; then
TOTAL_HUMAN=$(awk "BEGIN {printf \"%.1fGB\", $TOTAL_DELETE_BYTES / 1073741824}")
if [[ "$I_KNOW" != true ]]; then
echo ""
error "Deletion would exceed ${SONARR_MAX_DELETE_GB:-1}GB threshold — $TOTAL_HUMAN would be deleted"
error "Review the ORPHAN lines above carefully before proceeding"
error "If this is expected, rerun with: --i-know-what-im-doing"
error "To also bypass age check and delete on first pass: add --skip-strike-list"
notify "Sonarr cleanup halted on $(hostname) — ${TOTAL_HUMAN} deletion requires --i-know-what-im-doing" "Sonarr Cleanup" "warning"
error "Deletion would exceed ${SONARR_MAX_DELETE_GB}GB — $TOTAL_HUMAN would be deleted"
error "Review ORPHAN lines above carefully before proceeding"
error "Rerun with: --i-know-what-im-doing"
error "To also bypass age check: add --skip-strike-list"
notify "Sonarr cleanup halted on $(hostname) — ${TOTAL_HUMAN} requires --i-know-what-im-doing" \
"Sonarr Cleanup" "warning"
exit 1
else
warn "OVERRIDE — deletion is ${TOTAL_HUMAN} — proceeding because --i-know-what-im-doing"
warn "OVERRIDE — deletion is $TOTAL_HUMAN — proceeding with --i-know-what-im-doing"
fi
fi
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Summary ━━━
# -----------------------------------------------------------------------------------------------
# ── Execute Deletions ─────────────────────────────────────────────────────────────────────────
if [[ "$DRY_RUN" == false ]]; then
while IFS= read -r filepath; do
[[ -z "$filepath" ]] && continue
grep -qF "$filepath" "$TRACKED_FILE" 2>/dev/null && continue
is_protected_file "$filepath" && continue
FILE_MTIME=$(stat -c %Y "$filepath" 2>/dev/null || echo 0)
FILE_AGE=$(( NOW - FILE_MTIME ))
if is_video_file "$filepath"; then
[[ "$FILE_AGE" -lt "$AGE_SECONDS" ]] && \
[[ "$SKIP_STRIKES" != true ]] && continue
fi
rm -f "$filepath" 2>/dev/null || error "Failed to delete: $filepath"
done < <(
for host_path in "${ARR_PATH_MAP[@]}" "$SONARR_TV_ROOT"; do
[[ -d "$host_path" ]] && find "$host_path" -type f 2>/dev/null
done | sort -u
)
log "Cleaning up empty folders..."
for host_path in "${ARR_PATH_MAP[@]}" "$SONARR_TV_ROOT"; do
[[ -d "$host_path" ]] && \
find "$host_path" -mindepth 1 -type d -empty -delete 2>/dev/null
done
log "Empty folders removed"
fi
END=$(date +%s)
ORPHAN_HUMAN=$(format_bytes "$ORPHAN_BYTES")
JUNK_HUMAN=$(format_bytes "$JUNK_BYTES")
# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
echo ""
echo "━━━━━ $ICON_SUMMARY SONARR CLEANUP SUMMARY ━━━━━"
echo "$ICON_SYNC Tracked by Sonarr: $TRACKED_COUNT files"
echo "$ICON_SHIELD Protected: $PROTECTED_COUNT files (artwork, subtitles, metadata)"
echo "$ICON_TRASH Orphans removed: $ORPHAN_COUNT files ($ORPHAN_HUMAN)"
echo "$ICON_TRASH Junk removed: $JUNK_COUNT files ($JUNK_HUMAN)"
echo "$ICON_TIME Recent skipped: $RECENT_COUNT files (under ${SONARR_ORPHAN_AGE} days)"
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_SYNC Tracked: $TRACKED_COUNT files ($SERIES_COUNT series)"
echo "$ICON_SHIELD Protected: $PROTECTED_COUNT files (artwork, subtitles, metadata)"
echo "$ICON_TRASH Orphans: $ORPHAN_COUNT files ($ORPHAN_HUMAN)"
echo "$ICON_TRASH Junk: $JUNK_COUNT files ($JUNK_HUMAN)"
echo "$ICON_SKIP Recent skipped: $RECENT_COUNT files (under ${SONARR_ORPHAN_AGE} days)"
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
echo ""
if [[ "$DRY_RUN" == true ]]; then
echo "$ICON_WARN Status: DRY RUN — no files deleted"
warn "DRY RUN — no files deleted"
elif [[ "$TOTAL_REMOVED" -eq 0 ]]; then
echo "$ICON_DONE Status: $ICON_SUCCESS CLEAN — nothing to remove"
notify "Sonarr cleanup complete on $(hostname) — library is clean" "Sonarr Cleanup" "normal"
log "$ICON_DONE Clean — nothing to remove"
else
echo "$ICON_DONE Status: $ICON_SUCCESS DONE — $TOTAL_REMOVED files removed"
notify "Sonarr cleanup on $(hostname) — removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" "Sonarr Cleanup" "normal"
warn "$ICON_DONE Removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)"
notify "Sonarr cleanup on $(hostname) — removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" \
"Sonarr Cleanup" "warning"
# Notify Emby to clean missing files — removes ghost entries immediately
notify_emby_scan
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
# Write stats for sunday_morning_coffee_report.sh
if [[ "$DRY_RUN" == false ]] && [[ -n "${ARR_CLEANUP_STATS:-}" ]]; then
DATE=$(date '+%Y-%m-%d')
echo "${DATE}|sonarr|${ORPHAN_COUNT}|${ORPHAN_BYTES}|${JUNK_COUNT}|${JUNK_BYTES}|${RECENT_COUNT}|${TRACKED_COUNT}" \
echo "$(date '+%Y-%m-%d')|sonarr|${ORPHAN_COUNT}|${ORPHAN_BYTES}|${JUNK_COUNT}|${JUNK_BYTES}|${RECENT_COUNT}|${TRACKED_COUNT}" \
>> "$ARR_CLEANUP_STATS" 2>/dev/null || true
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
Monitors/README-Monitors.md
+890 -285
View File
File diff suppressed because it is too large Load Diff
Monitors/backup_verify.sh
+173 -88
View File
@@ -1,79 +1,152 @@
#!/bin/bash
# -----------------------------------------------------------------------------------------------
# --------------------------------- Backup Verify ----------------------------------------------
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ================================= Backup Verify ==============================================
# ==============================================================================================
# Verifies the rsync mirror is healthy by comparing random file samples between
# local and remote servers using MD5 checksums.
#
# Randomly samples BACKUP_VERIFY_SAMPLE files per share, computes checksums locally,
# then computes the same checksums on the remote via SSH and compares results.
# ── WHAT IT DOES ──────────────────────────────────────────────────────────────────────────────
# Randomly samples BACKUP_VERIFY_SAMPLE files per share above BACKUP_VERIFY_MIN_SIZE,
# computes MD5 checksums locally, then computes the same checksums on the remote via SSH
# and compares results. Catches silent corruption or incomplete syncs that rsync itself
# would not detect.
#
# Results per file:
# MATCH — checksums identical, file is correctly mirrored
# MISMATCH — file exists on both but checksums differ — sync may have failed
# MISSING — file exists locally but not on remote — not yet synced or deleted
# ── RESULTS PER FILE ──────────────────────────────────────────────────────────────────────────
# MATCH — checksums identical, file is correctly mirrored ✅
# MISMATCH — file exists on both but checksums differ — sync may have partially failed
# MISSING — file exists locally but not on remote — not yet synced or deleted on remote
#
# Silent when all files match. Notifies on any mismatch or missing file.
# Uses existing SSH keys — no additional configuration needed beyond share list.
# ── SHARE SELECTION ───────────────────────────────────────────────────────────────────────────
# Uses HOST*_BACKUP_VERIFY_SHARES if defined, falls back to HOST*_DAILY_SYNC_SHARES.
# Both aliased by detect_hosts() — no manual HOST1/HOST2 selection needed.
#
# All configuration in Master.conf under Backup Verify section.
# Supports --dry-run to show what would be checked without running checksums.
# -----------------------------------------------------------------------------------------------
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
# acquire_lock — prevents concurrent runs producing duplicate/conflicting results
# check_connectivity() — verifies remote reachable before attempting SSH calls
# check_remote_array() — verifies remote array mounted before checksums
# remote array down = all files "missing" = false alarm ✅
# version parity — verifies both servers on compatible unRAID before trusting results
# SSH_TIMEOUT — all SSH calls protected against hangs
# validate_unraid_cmd — notify script validated before use
# Silent by default — only issues produce output, all-match runs are silent
#
# ── CONFIGURATION (master_host*.conf) ─────────────────────────────────────────────────────────
# HOST*_BACKUP_VERIFY_SHARES — override share list (empty = use DAILY_SYNC_SHARES)
# HOST*_DAILY_SYNC_SHARES — fallback share list
# All aliased by detect_hosts()
#
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
# BACKUP_VERIFY_SAMPLE — random files to check per share (default 10)
# BACKUP_VERIFY_MIN_SIZE — minimum file size to include in sample (default 1M)
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
# backup_verify.sh — normal run
# backup_verify.sh --dry-run — show sample selection only, no checksums
# backup_verify.sh --log — verbose output
# backup_verify.sh --status — show config and exit
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../Master.conf"
source "$SCRIPT_DIR/../common.sh"
source "$SCRIPT_DIR/../load_config.sh"
parse_args "$@"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Setup ━━━
# -----------------------------------------------------------------------------------------------
SSH_TIMEOUT=15
# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_GEAR Setup ━━━"
if [[ "$EUID" -ne 0 ]]; then
error "Must be run as root"
exit 1
fi
validate_unraid_cmd \
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
"" "" \
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
acquire_lock
# detect_hosts() sets MY_ID and aliases BACKUP_VERIFY_SHARES + DAILY_SYNC_SHARES
detect_hosts
resolve_remote_ip
# Use BACKUP_VERIFY_SHARES if defined, fall back to DAILY_SYNC_SHARES
# Share selection — configured list or fallback to daily sync shares
if [[ ${#BACKUP_VERIFY_SHARES[@]} -gt 0 ]]; then
VERIFY_SHARES=("${BACKUP_VERIFY_SHARES[@]}")
info "Using BACKUP_VERIFY_SHARES (${#VERIFY_SHARES[@]} shares)"
log "Using BACKUP_VERIFY_SHARES (${#VERIFY_SHARES[@]} shares)"
else
VERIFY_SHARES=("${DAILY_SYNC_SHARES[@]}")
info "BACKUP_VERIFY_SHARES not set — using DAILY_SYNC_SHARES (${#VERIFY_SHARES[@]} shares)"
log "BACKUP_VERIFY_SHARES not set — using DAILY_SYNC_SHARES (${#VERIFY_SHARES[@]} shares)"
fi
if [[ ${#VERIFY_SHARES[@]} -eq 0 ]]; then
warn "No shares configured — nothing to verify"
exit 0
fi
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Status ━━━
# -----------------------------------------------------------------------------------------------
if [[ "$SHOW_STATUS" == true ]]; then
echo ""
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
echo "$ICON_VERIFY Shares: ${#VERIFY_SHARES[@]}"
echo "$ICON_VERIFY Sample: $BACKUP_VERIFY_SAMPLE files per share"
echo "$ICON_VERIFY Min size: $BACKUP_VERIFY_MIN_SIZE"
echo "$ICON_HOST Remote: $REMOTE_SERVER_NAME ($REMOTE_SERVER)"
echo "$ICON_GEAR Dry Run: $DRY_RUN"
echo "━━━━━━━━━━━━━━━━━━━━━━━"
warn "No shares configured for $MY_ID — nothing to verify"
warn "Check HOST*_BACKUP_VERIFY_SHARES or HOST*_DAILY_SYNC_SHARES in master_host*.conf"
exit 0
fi
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — showing sample selection only, no checksums computed"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_VERIFY Backup Verification ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
echo ""
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
echo "$ICON_HOST My ID: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_HOST Remote: $REMOTE_ID ($REMOTE_SERVER_NAME — $REMOTE_SERVER)"
echo "$ICON_VERIFY Shares: ${#VERIFY_SHARES[@]}"
echo "$ICON_VERIFY Sample: $BACKUP_VERIFY_SAMPLE files per share"
echo "$ICON_VERIFY Min size: $BACKUP_VERIFY_MIN_SIZE"
echo "$ICON_GEAR Dry Run: $DRY_RUN"
echo ""
echo " Shares to verify:"
for share in "${VERIFY_SHARES[@]}"; do
echo " $share"
done
echo "━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
fi
# ==============================================================================================
# ━━━ Pre-flight ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_SHIELD Pre-flight ━━━"
# Connectivity — no point making 100+ SSH calls if remote is unreachable
check_connectivity
# Version parity — mismatched unRAID could cause md5sum path differences
check_unraid_version_parity || {
warn "Version parity check failed — proceeding with caution"
warn "Checksum results may be unreliable if md5sum path changed between versions"
}
# Remote array — if array is down all files appear "missing" = false alarm
if ! check_remote_array; then
error "Remote array not mounted on $REMOTE_SERVER_NAME"
error "All files would appear as MISSING — aborting to prevent false alarm"
notify "Backup verify aborted on $(hostname) — remote array not mounted on $REMOTE_SERVER_NAME" \
"Backup Verify" "warning"
exit 1
fi
log "Pre-flight passed ✅"
# ==============================================================================================
# ━━━ Backup Verification ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_VERIFY Backup Verification — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
echo "$ICON_HOST Remote: $REMOTE_SERVER_NAME ($REMOTE_SERVER)"
echo "$ICON_VERIFY Sample: $BACKUP_VERIFY_SAMPLE files per share (min size: $BACKUP_VERIFY_MIN_SIZE)"
echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME) → $REMOTE_ID ($REMOTE_SERVER_NAME)"
echo "$ICON_VERIFY Sample: $BACKUP_VERIFY_SAMPLE files per share (min: $BACKUP_VERIFY_MIN_SIZE)"
echo ""
START=$(date +%s)
@@ -93,69 +166,74 @@ for share in "${VERIFY_SHARES[@]}"; do
continue
fi
# Find files above minimum size and randomly sample
SAMPLE_FILES=$(find "$share" -type f -size +"$BACKUP_VERIFY_MIN_SIZE" 2>/dev/null | \
shuf | head -n "$BACKUP_VERIFY_SAMPLE")
# Sample random files above minimum size
mapfile -t SAMPLE_FILES < <(
find "$share" -type f -size +"$BACKUP_VERIFY_MIN_SIZE" 2>/dev/null | \
shuf | head -n "$BACKUP_VERIFY_SAMPLE"
)
SAMPLE_COUNT=$(echo "$SAMPLE_FILES" | grep -c "." 2>/dev/null || echo 0)
if [[ "$SAMPLE_COUNT" -eq 0 ]]; then
info "No files found above $BACKUP_VERIFY_MIN_SIZE — skipping"
if [[ ${#SAMPLE_FILES[@]} -eq 0 ]]; then
log "$SHARE_NAME — no files found above $BACKUP_VERIFY_MIN_SIZE"
echo ""
continue
fi
info "Sampled $SAMPLE_COUNT files"
log "$SHARE_NAME — sampled ${#SAMPLE_FILES[@]} files"
if [[ "$DRY_RUN" == true ]]; then
echo "$SAMPLE_FILES" | while IFS= read -r f; do
for f in "${SAMPLE_FILES[@]}"; do
warn "DRY RUN — would check: $(basename "$f")"
done
echo ""
continue
fi
SHARE_MATCH=0
SHARE_MISMATCH=0
SHARE_MISSING=0
SHARE_MATCH=0
while IFS= read -r local_file; do
for local_file in "${SAMPLE_FILES[@]}"; do
[[ -z "$local_file" ]] && continue
# Compute local checksum
# Local checksum
local_md5=$(md5sum "$local_file" 2>/dev/null | awk '{print $1}')
if [[ -z "$local_md5" ]]; then
warn "Could not checksum: $local_file — skipping"
warn "Could not checksum locally: $(basename "$local_file") — skipping"
continue
fi
# Compute remote checksum via SSH
remote_md5=$(ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \
# Remote checksum via SSH — timeout protected
remote_md5=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
-o ConnectTimeout="$SSH_TIMEOUT" \
-o StrictHostKeyChecking=no \
root@"$REMOTE_SERVER" \
"md5sum '$local_file' 2>/dev/null | awk '{print \$1}'" 2>/dev/null)
((TOTAL_CHECKED++))
(( TOTAL_CHECKED++ ))
if [[ -z "$remote_md5" ]]; then
warn "$ICON_ERROR MISSING: $(basename "$local_file")"
((SHARE_MISSING++))
((TOTAL_MISSING++))
(( SHARE_MISSING++ ))
(( TOTAL_MISSING++ ))
elif [[ "$local_md5" == "$remote_md5" ]]; then
log "MATCH: $(basename "$local_file")"
((SHARE_MATCH++))
((TOTAL_MATCH++))
(( SHARE_MATCH++ ))
(( TOTAL_MATCH++ ))
else
error "$ICON_ERROR MISMATCH: $(basename "$local_file")"
((SHARE_MISMATCH++))
((TOTAL_MISMATCH++))
error "MISMATCH: $(basename "$local_file")"
error " local: $local_md5"
error " remote: $remote_md5"
(( SHARE_MISMATCH++ ))
(( TOTAL_MISMATCH++ ))
fi
done
done <<< "$SAMPLE_FILES"
echo " $ICON_SUCCESS Match: $SHARE_MATCH $ICON_WARN Missing: $SHARE_MISSING $ICON_ERROR Mismatch: $SHARE_MISMATCH"
# Per-share result — only visible if issues found
if [[ "$SHARE_MISMATCH" -gt 0 || "$SHARE_MISSING" -gt 0 ]]; then
warn "$SHARE_NAME — match: $SHARE_MATCH missing: $SHARE_MISSING mismatch: $SHARE_MISMATCH"
SHARES_WITH_ISSUES+=("$SHARE_NAME")
else
log "$SHARE_NAME — all $SHARE_MATCH files match ✅"
fi
echo ""
@@ -163,25 +241,32 @@ done
END=$(date +%s)
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Summary ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
echo "━━━━━ $ICON_SUMMARY BACKUP VERIFY SUMMARY ━━━━━"
echo "$ICON_HOST Remote: $REMOTE_SERVER_NAME"
echo "$ICON_HOST My ID: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_HOST Remote: $REMOTE_ID ($REMOTE_SERVER_NAME)"
echo "$ICON_VERIFY Checked: $TOTAL_CHECKED files"
echo "$ICON_SUCCESS Match: $TOTAL_MATCH"
echo "$ICON_WARN Missing: $TOTAL_MISSING"
echo "$ICON_ERROR Mismatch: $TOTAL_MISMATCH"
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
echo ""
if [[ "$DRY_RUN" == true ]]; then
echo "$ICON_WARN Status: DRY RUN — no checksums computed"
elif [[ "$TOTAL_MISMATCH" -gt 0 || "$TOTAL_MISSING" -gt 0 ]]; then
echo "$ICON_ERROR Status: ISSUES FOUND — ${#SHARES_WITH_ISSUES[@]} share(s) need attention"
notify "Backup verify failed on $(hostname) → $REMOTE_SERVER_NAME — mismatches: $TOTAL_MISMATCH missing: $TOTAL_MISSING — shares: ${SHARES_WITH_ISSUES[*]}" "Backup Verify" "warning"
else
echo "$ICON_DONE Status: $ICON_SUCCESS ALL FILES MATCH"
notify "Backup verify passed on $(hostname) → $REMOTE_SERVER_NAME — $TOTAL_CHECKED files checked across ${#VERIFY_SHARES[@]} shares" "Backup Verify" "normal"
if [[ "$TOTAL_MISMATCH" -gt 0 || "$TOTAL_MISSING" -gt 0 ]]; then
echo "$ICON_SUCCESS Match: $TOTAL_MATCH"
warn "Missing: $TOTAL_MISSING"
[[ "$TOTAL_MISMATCH" -gt 0 ]] && echo "$ICON_ERROR Mismatch: $TOTAL_MISMATCH"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — no checksums computed"
elif [[ "$TOTAL_MISMATCH" -gt 0 || "$TOTAL_MISSING" -gt 0 ]]; then
echo "$ICON_ERROR Status: ISSUES FOUND — ${#SHARES_WITH_ISSUES[@]} share(s) need attention: ${SHARES_WITH_ISSUES[*]}"
notify "Backup verify FAILED on $(hostname) → $REMOTE_SERVER_NAME — mismatches: $TOTAL_MISMATCH missing: $TOTAL_MISSING — shares: ${SHARES_WITH_ISSUES[*]}" \
"Backup Verify" "warning"
else
log "$ICON_DONE Status: all $TOTAL_CHECKED files match across ${#VERIFY_SHARES[@]} shares ✅"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
[[ "$TOTAL_MISMATCH" -gt 0 ]] && exit 1
exit 0
Monitors/bandwidth_monitor.sh
+149 -49
View File
@@ -1,38 +1,63 @@
#!/bin/bash
# -----------------------------------------------------------------------------------------------
# --------------------------------- Bandwidth Monitor ------------------------------------------
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ================================= Bandwidth Monitor ==========================================
# ==============================================================================================
# Logs rsync transfer history and generates weekly summary reports.
# Designed for minimal flash drive impact — one bounded write per rsync run.
#
# Two modes:
# --log-transfer "profile" duration status — called by rsync.sh after each sync
# appends one line, trims old entries
# --report (or no args) — generates summary from log
# ── TWO MODES ─────────────────────────────────────────────────────────────────────────────────
#
# Log format — one line per transfer, version-proof, never needs rsync output parsing:
# YYYY-MM-DD|HH:MM|profile|duration_seconds|status
# --log-transfer "profile" duration_seconds status
# Called automatically by rsync.sh after each sync completes.
# Appends one line to the log and trims entries older than BANDWIDTH_LOG_RETENTION.
# Flags syncs exceeding BANDWIDTH_WARN_GB in the log for weekly report highlighting.
#
# Log file stays bounded to BANDWIDTH_LOG_RETENTION days — old entries trimmed on every write.
# --report (or no args)
# Generates a summary from the accumulated log.
# Shows per-profile breakdown, last 7 days, and overall totals.
# This is a monitor script — SILENT_MODE=false — output is the point.
#
# ── LOG FORMAT ────────────────────────────────────────────────────────────────────────────────
# One line per transfer — version-proof, never needs rsync output parsing:
# YYYY-MM-DD|HH:MM|profile|duration_seconds|status|bytes_transferred
#
# Log file stays bounded to BANDWIDTH_LOG_RETENTION days — trimmed on every write.
# Minimal flash drive impact: one append + one trim per rsync run.
#
# All configuration in Master.conf under Bandwidth Monitor section.
# -----------------------------------------------------------------------------------------------
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
# acquire_lock "wait" — prevents log corruption from concurrent rsync completions
# validate_unraid_cmd — notify script validated before use
# Atomic log write — temp file + mv prevents partial writes on trim
# Log existence check — creates log directory if needed, exits cleanly if unwritable
#
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
# BANDWIDTH_LOG — log file path
# BANDWIDTH_LOG_RETENTION — days before old entries are purged (default 90)
# BANDWIDTH_WARN_GB — flag syncs larger than this in report (default 50)
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
# bandwidth_monitor.sh — generate report
# bandwidth_monitor.sh --report — generate report (explicit)
# bandwidth_monitor.sh --log-transfer profile secs ok — log a transfer (called by rsync.sh)
# bandwidth_monitor.sh --status — show config and exit
# bandwidth_monitor.sh --log — verbose output
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../Master.conf"
source "$SCRIPT_DIR/../common.sh"
source "$SCRIPT_DIR/../load_config.sh"
# Monitor script — output is the point
SILENT_MODE=false
parse_args "$@"
# -----------------------------------------------------------------------------------------------
# Parse mode from PARSED_ARGS
# -----------------------------------------------------------------------------------------------
# ── Parse mode from PARSED_ARGS ───────────────────────────────────────────────────────────────
LOG_TRANSFER_MODE=false
TRANSFER_PROFILE=""
TRANSFER_DURATION=0
TRANSFER_STATUS="success"
TRANSFER_BYTES=0
for arg in "${PARSED_ARGS[@]}"; do
case "$arg" in
@@ -42,79 +67,131 @@ for arg in "${PARSED_ARGS[@]}"; do
if [[ "$LOG_TRANSFER_MODE" == true ]]; then
if [[ -z "$TRANSFER_PROFILE" ]]; then
TRANSFER_PROFILE="$arg"
elif [[ "$TRANSFER_DURATION" -eq 0 ]]; then
elif [[ "$TRANSFER_DURATION" -eq 0 && "$arg" =~ ^[0-9]+$ ]]; then
TRANSFER_DURATION="$arg"
else
elif [[ "$arg" == "success" || "$arg" == "failed" ]]; then
TRANSFER_STATUS="$arg"
elif [[ "$arg" =~ ^[0-9]+$ ]]; then
TRANSFER_BYTES="$arg"
fi
fi
;;
esac
done
# Ensure log directory and file exist
# ── Ensure log file exists and is writable ────────────────────────────────────────────────────
mkdir -p "$(dirname "$BANDWIDTH_LOG")"
touch "$BANDWIDTH_LOG" 2>/dev/null || {
error "Cannot write to bandwidth log: $BANDWIDTH_LOG"
exit 1
}
# -----------------------------------------------------------------------------------------------
# LOG TRANSFER MODE
# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
error "Must be run as root"
exit 1
fi
validate_unraid_cmd \
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
"" "" \
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
# detect_hosts() sets MY_ID for report header
detect_hosts
# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
echo ""
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_BANDWIDTH Log file: $BANDWIDTH_LOG"
echo "$ICON_BANDWIDTH Retention: ${BANDWIDTH_LOG_RETENTION} days"
echo "$ICON_BANDWIDTH Warn GB: ${BANDWIDTH_WARN_GB}GB"
local entry_count=0
[[ -f "$BANDWIDTH_LOG" ]] && entry_count=$(wc -l < "$BANDWIDTH_LOG")
echo "$ICON_BANDWIDTH Log entries: $entry_count"
echo "━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
fi
# ==============================================================================================
# ━━━ Log Transfer Mode ━━━
# ==============================================================================================
# Called by rsync.sh after each sync — appends one line and trims old entries.
# Usage: bandwidth_monitor.sh --log-transfer "profile" duration_seconds status
# -----------------------------------------------------------------------------------------------
# Uses "wait" lock — if two rsync jobs finish simultaneously, wait and write in order.
# Usage: bandwidth_monitor.sh --log-transfer "profile" duration_seconds status [bytes]
if [[ "$LOG_TRANSFER_MODE" == true ]]; then
[[ -z "$TRANSFER_PROFILE" ]] && error "No profile specified for --log-transfer" && exit 1
[[ -z "$TRANSFER_PROFILE" ]] && { error "No profile specified for --log-transfer"; exit 1; }
acquire_lock "wait"
TODAY=$(date '+%Y-%m-%d')
NOW=$(date '+%H:%M')
DURATION_FMT=$(format_duration "$TRANSFER_DURATION")
# Append entry
echo "${TODAY}|${NOW}|${TRANSFER_PROFILE}|${TRANSFER_DURATION}|${TRANSFER_STATUS}" >> "$BANDWIDTH_LOG"
log "$ICON_BANDWIDTH Logged: $TRANSFER_PROFILE — ${DURATION_FMT} — $TRANSFER_STATUS"
# Check if transfer exceeds warn threshold
WARN_FLAG=""
if [[ -n "$TRANSFER_BYTES" && "$TRANSFER_BYTES" -gt 0 ]]; then
WARN_BYTES=$(awk "BEGIN {printf \"%d\", $BANDWIDTH_WARN_GB * 1073741824}")
[[ "$TRANSFER_BYTES" -gt "$WARN_BYTES" ]] && WARN_FLAG="LARGE"
fi
# Trim entries older than retention period — keeps file bounded
# Append entry — format: date|time|profile|duration|status|bytes|warn_flag
echo "${TODAY}|${NOW}|${TRANSFER_PROFILE}|${TRANSFER_DURATION}|${TRANSFER_STATUS}|${TRANSFER_BYTES}|${WARN_FLAG}" \
>> "$BANDWIDTH_LOG"
log "$ICON_BANDWIDTH Logged: $TRANSFER_PROFILE — ${DURATION_FMT} — $TRANSFER_STATUS${WARN_FLAG:+ [$WARN_FLAG]}"
# Trim entries older than retention — atomic write via temp file
CUTOFF=$(date -d "${BANDWIDTH_LOG_RETENTION} days ago" '+%Y-%m-%d')
TEMP_FILE="${BANDWIDTH_LOG}.tmp"
awk -F'|' -v cutoff="$CUTOFF" '$1 >= cutoff' "$BANDWIDTH_LOG" > "$TEMP_FILE" && \
mv "$TEMP_FILE" "$BANDWIDTH_LOG"
log "$ICON_BANDWIDTH Log trimmed — retaining entries from $CUTOFF onwards"
log "$ICON_BANDWIDTH Log trimmed — retaining entries from $CUTOFF"
exit 0
fi
# -----------------------------------------------------------------------------------------------
# REPORT MODE — generate summary from log
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Report Mode ━━━
# ==============================================================================================
acquire_lock "wait"
echo ""
echo "━━━ $ICON_BANDWIDTH Bandwidth Report — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME)"
echo ""
if [[ ! -s "$BANDWIDTH_LOG" ]]; then
warn "No bandwidth data yet — log is empty"
warn "Data accumulates as rsync jobs complete"
warn "Data accumulates as rsync jobs complete via rsync.sh"
exit 0
fi
START=$(date +%s)
# Date range
# ── Log overview ──────────────────────────────────────────────────────────────────────────────
OLDEST=$(awk -F'|' 'NR==1{print $1}' "$BANDWIDTH_LOG")
NEWEST=$(awk -F'|' 'END{print $1}' "$BANDWIDTH_LOG")
ENTRY_COUNT=$(wc -l < "$BANDWIDTH_LOG")
SUCCESS_COUNT=$(awk -F'|' '$5=="success"' "$BANDWIDTH_LOG" | wc -l)
FAILED_COUNT=$(awk -F'|' '$5=="failed"' "$BANDWIDTH_LOG" | wc -l)
LARGE_COUNT=$(awk -F'|' '$7=="LARGE"' "$BANDWIDTH_LOG" | wc -l)
info "Log covers: $OLDEST → $NEWEST ($ENTRY_COUNT runs)"
echo ""
# ── Per-profile breakdown ────────────────────────────────────────────────────────────────────
# ── Per-profile breakdown ─────────────────────────────────────────────────────────────────────
echo "━━━ $ICON_BANDWIDTH Per-Profile Summary ━━━"
awk -F'|' '{
runs[$3]++
duration[$3] += $4
if ($5 == "failed") fails[$3]++
if ($7 == "LARGE") large[$3]++
}
END {
for (profile in runs) {
@@ -122,21 +199,22 @@ END {
mins = int(avg / 60)
secs = int(avg % 60)
fail_count = (profile in fails) ? fails[profile] : 0
printf " %-20s %3d runs avg %dm%ds failed: %d\n", \
profile, runs[profile], mins, secs, fail_count
large_count = (profile in large) ? large[profile] : 0
large_str = (large_count > 0) ? " ⚠️ " large_count " large" : ""
printf " %-22s %3d runs avg %dm%ds failed: %d%s\n", \
profile, runs[profile], mins, secs, fail_count, large_str
}
}' "$BANDWIDTH_LOG" | sort
echo ""
# ── Last 7 days ──────────────────────────────────────────────────────────────────────────────
# ── Last 7 days ───────────────────────────────────────────────────────────────────────────────
echo "━━━ $ICON_BANDWIDTH Last 7 Days ━━━"
for i in 6 5 4 3 2 1 0; do
day=$(date -d "$i days ago" '+%Y-%m-%d')
day_name=$(date -d "$i days ago" '+%a')
day_runs=$(awk -F'|' -v d="$day" '$1==d' "$BANDWIDTH_LOG" | wc -l)
day_success=$(awk -F'|' -v d="$day" '$1==d && $5=="success"' "$BANDWIDTH_LOG" | wc -l)
day_failed=$(awk -F'|' -v d="$day" '$1==d && $5=="failed"' "$BANDWIDTH_LOG" | wc -l)
day_large=$(awk -F'|' -v d="$day" '$1==d && $7=="LARGE"' "$BANDWIDTH_LOG" | wc -l)
day_duration=$(awk -F'|' -v d="$day" '$1==d{sum+=$4} END{print sum+0}' "$BANDWIDTH_LOG")
day_duration_fmt=$(format_duration "$day_duration")
@@ -144,25 +222,47 @@ for i in 6 5 4 3 2 1 0; do
echo " $ICON_TIME $day ($day_name) — no syncs"
elif [[ "$day_failed" -gt 0 ]]; then
echo " $ICON_WARN $day ($day_name) — $day_runs runs / ${day_duration_fmt} total / $ICON_ERROR $day_failed failed"
elif [[ "$day_large" -gt 0 ]]; then
echo " $ICON_WARN $day ($day_name) — $day_runs runs / ${day_duration_fmt} total / ⚠️ $day_large large"
else
echo " $ICON_DONE $day ($day_name) — $day_runs runs / ${day_duration_fmt} total"
fi
done
echo ""
# ── Totals ───────────────────────────────────────────────────────────────────────────────────
# ── Large transfers ───────────────────────────────────────────────────────────────────────────
if [[ "$LARGE_COUNT" -gt 0 ]]; then
echo "━━━ $ICON_WARN Large Transfers (>${BANDWIDTH_WARN_GB}GB) ━━━"
awk -F'|' '$7=="LARGE" {
bytes=$6+0
gb=bytes/1073741824
printf " %s %s %-20s %.1fGB\n", $1, $2, $3, gb
}' "$BANDWIDTH_LOG" | tail -10
echo ""
fi
# ── Totals ────────────────────────────────────────────────────────────────────────────────────
TOTAL_DURATION=$(awk -F'|' '{sum+=$4} END{print sum+0}' "$BANDWIDTH_LOG")
TOTAL_DURATION_FMT=$(format_duration "$TOTAL_DURATION")
END=$(date +%s)
echo "━━━━━ $ICON_SUMMARY BANDWIDTH SUMMARY ━━━━━"
echo "$ICON_BANDWIDTH Total runs: $ENTRY_COUNT ($SUCCESS_COUNT success / $FAILED_COUNT failed)"
echo "$ICON_TIME Total time: $TOTAL_DURATION_FMT"
echo "$ICON_TIME Log period: $OLDEST → $NEWEST"
echo "$ICON_GEAR Retention: ${BANDWIDTH_LOG_RETENTION} days"
echo "$ICON_TIME Generated in: $(format_duration $((END - START)))"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_BANDWIDTH Runs: $ENTRY_COUNT ($SUCCESS_COUNT success / $FAILED_COUNT failed)"
[[ "$LARGE_COUNT" -gt 0 ]] && \
warn "Large: $LARGE_COUNT transfer(s) exceeded ${BANDWIDTH_WARN_GB}GB"
echo "$ICON_TIME Total: $TOTAL_DURATION_FMT"
echo "$ICON_TIME Period: $OLDEST → $NEWEST"
echo "$ICON_GEAR Retention: ${BANDWIDTH_LOG_RETENTION} days"
echo "$ICON_TIME Generated: $(format_duration $(( END - START )))"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
notify "Bandwidth report on $(hostname) — $ENTRY_COUNT rsync runs ($SUCCESS_COUNT success / $FAILED_COUNT failed) over ${BANDWIDTH_LOG_RETENTION} day window" "Bandwidth Monitor" "normal"
# Only notify if there are failures or large transfers worth flagging
if [[ "$FAILED_COUNT" -gt 0 ]]; then
notify "Bandwidth report on $(hostname) — $FAILED_COUNT failed sync(s) in ${BANDWIDTH_LOG_RETENTION} day window" \
"Bandwidth Monitor" "warning"
elif [[ "$LARGE_COUNT" -gt 0 ]]; then
notify "Bandwidth report on $(hostname) — $LARGE_COUNT large transfer(s) exceeded ${BANDWIDTH_WARN_GB}GB" \
"Bandwidth Monitor" "normal"
fi
Monitors/cert_monitor.sh
+129 -61
View File
@@ -1,73 +1,130 @@
#!/bin/bash
# -----------------------------------------------------------------------------------------------
# --------------------------------- Certificate Monitor ----------------------------------------
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ================================= Certificate Monitor ========================================
# ==============================================================================================
# Monitors SSL certificate expiry for all configured domains by connecting directly
# via openssl — no dependency on NPM or any other service. Reads the actual certificate
# the server is presenting to the outside world.
#
# This approach catches real-world cert issues that API-based checks miss:
# - Cert renewed but server not reloaded
# - Wrong cert being served
# - Cert chain issues
# ── WHY DIRECT OPENSSL ────────────────────────────────────────────────────────────────────────
# Catches real-world cert issues that API-based checks miss:
# - Cert renewed in NPM but server not reloaded (old cert still serving)
# - Wrong cert being served to external clients
# - Cert chain issues not visible from the internal network
# - NPM reporting healthy while the world sees an expired cert
#
# Each domain and subdomain is a separate entry — they have independent certs.
# Silent when all certs are healthy. Notifies when any approach warning threshold.
# Notifications batched per severity — one message for warnings, one for criticals.
# ── BEHAVIOUR ─────────────────────────────────────────────────────────────────────────────────
# Each domain is checked independently — they have independent certs.
# Results per domain:
# HEALTHY — > CERT_WARN_DAYS remaining — silent ✅
# WARNING — <= CERT_WARN_DAYS remaining — notifies
# CRITICAL — <= CERT_CRIT_DAYS remaining — notifies with urgency
# FAILED — could not connect or parse cert — notifies
#
# All configuration in Master.conf under Certificate Monitor section.
# Supports --dry-run to check certs and show results without sending notifications.
# -----------------------------------------------------------------------------------------------
# Notifications batched per severity — one message per severity level, not per domain.
# This is a monitor script — SILENT_MODE=false — output is the point.
#
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
# detect_hosts() sets MY_ID and aliases HOST*_CERT_MONITOR_DOMAINS → CERT_MONITOR_DOMAINS.
# Each server monitors its own domains — HOST1 monitors Gmer4Lfe.com etc.
#
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
# acquire_lock — prevents concurrent runs
# detect_hosts() — correct domain list per host via MY_ID aliases
# Empty array guard — warns and exits cleanly if no domains configured
# CERT_TIMEOUT — openssl connects are time-limited per domain
# validate_unraid_cmd — openssl and notify validated before use
# Silent healthy certs — only problems produce visible output
#
# ── CONFIGURATION (master_host*.conf) ─────────────────────────────────────────────────────────
# HOST*_CERT_MONITOR_DOMAINS — domains checked by this host
# Aliased by detect_hosts() — script uses CERT_MONITOR_DOMAINS
#
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
# CERT_WARN_DAYS — warn when cert expires within this many days (default 30)
# CERT_CRIT_DAYS — critical alert within this many days (default 7)
# CERT_TIMEOUT — seconds per domain before giving up (default 10)
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
# cert_monitor.sh — normal run
# cert_monitor.sh --dry-run — check certs and show results, no notifications
# cert_monitor.sh --log — verbose output
# cert_monitor.sh --status — show config and exit
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../Master.conf"
source "$SCRIPT_DIR/../common.sh"
source "$SCRIPT_DIR/../load_config.sh"
# Monitor script — output is the point
SILENT_MODE=false
parse_args "$@"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Setup ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_GEAR Setup ━━━"
if ! command -v openssl >/dev/null 2>&1; then
error "openssl not found — required for certificate checks"
if [[ "$EUID" -ne 0 ]]; then
error "Must be run as root"
exit 1
fi
success "openssl available"
# Validate openssl — required for all cert checks
validate_unraid_cmd \
"$(command -v openssl 2>/dev/null || echo /usr/bin/openssl)" \
"version" "OpenSSL" \
"openssl" || { error "openssl not found — required for certificate checks"; exit 1; }
validate_unraid_cmd \
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
"" "" \
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
acquire_lock
# detect_hosts() sets MY_ID and aliases HOST*_CERT_MONITOR_DOMAINS
detect_hosts
# Empty array guard
if [[ ${#CERT_MONITOR_DOMAINS[@]} -eq 0 ]]; then
warn "CERT_MONITOR_DOMAINS is empty in Master.conf — add your domains to enable monitoring"
warn "CERT_MONITOR_DOMAINS is empty for $MY_ID"
warn "Check HOST*_CERT_MONITOR_DOMAINS in master_host*.conf"
exit 0
fi
info "$ICON_CERT Domains to check: ${#CERT_MONITOR_DOMAINS[@]}"
log "Domains to check: ${#CERT_MONITOR_DOMAINS[@]}"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Status ━━━
# -----------------------------------------------------------------------------------------------
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — results shown but no notifications sent"
# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
echo ""
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
echo "$ICON_CERT Domains: ${CERT_MONITOR_DOMAINS[*]}"
echo "$ICON_WARN Warn at: ${CERT_WARN_DAYS} days remaining"
echo "$ICON_ERROR Crit at: ${CERT_CRIT_DAYS} days remaining"
echo "$ICON_TIME Timeout: ${CERT_TIMEOUT}s per domain"
echo "$ICON_GEAR Dry Run: $DRY_RUN"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_CERT Domains: ${CERT_MONITOR_DOMAINS[*]}"
echo "$ICON_WARN Warn at: ${CERT_WARN_DAYS} days remaining"
echo "$ICON_ERROR Crit at: ${CERT_CRIT_DAYS} days remaining"
echo "$ICON_TIME Timeout: ${CERT_TIMEOUT}s per domain"
echo "$ICON_GEAR Dry Run: $DRY_RUN"
echo "━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
fi
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — results shown but no notifications sent"
# -----------------------------------------------------------------------------------------------
# CERT CHECK FUNCTION
# ==============================================================================================
# ── CERT CHECK FUNCTION ───────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Connects to domain:443 via openssl, extracts expiry date, calculates days remaining.
# Returns 0=healthy 1=warning 2=critical 3=failed
# -----------------------------------------------------------------------------------------------
# Returns:
# 0 = healthy (> CERT_WARN_DAYS)
# 1 = warning (<= CERT_WARN_DAYS)
# 2 = critical (<= CERT_CRIT_DAYS)
# 3 = failed (could not connect or parse)
check_cert() {
local domain="$1"
local port="${2:-443}"
@@ -79,7 +136,7 @@ check_cert() {
2>/dev/null | openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2)
if [[ -z "$expiry_str" ]]; then
error "$ICON_CERT $domain — could not retrieve certificate"
error "$ICON_CERT $domain — could not retrieve certificate (unreachable or no TLS)"
return 3
fi
@@ -103,18 +160,19 @@ check_cert() {
warn "$ICON_CERT $domain — WARNING: ${days_remaining} days remaining (expires $expiry_display)"
return 1
else
success "$ICON_CERT $domain — OK: ${days_remaining} days remaining (expires $expiry_display)"
log "$ICON_CERT $domain — OK: ${days_remaining} days remaining (expires $expiry_display)"
return 0
fi
}
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_CERT Certificate Monitor ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Certificate Monitor ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_CERT Certificate Monitor — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
echo "$ICON_WARN Warn threshold: ${CERT_WARN_DAYS} days"
echo "$ICON_ERROR Crit threshold: ${CERT_CRIT_DAYS} days"
echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME)"
log "Warn threshold: ${CERT_WARN_DAYS} days"
log "Crit threshold: ${CERT_CRIT_DAYS} days"
echo ""
START=$(date +%s)
@@ -126,7 +184,6 @@ declare -A DOMAIN_STATUS
for domain in "${CERT_MONITOR_DOMAINS[@]}"; do
[[ -z "$domain" ]] && continue
echo "━━━ $ICON_CERT $domain ━━━"
check_cert "$domain"
result=$?
case $result in
@@ -135,46 +192,57 @@ for domain in "${CERT_MONITOR_DOMAINS[@]}"; do
2) CRITICAL+=("$domain"); DOMAIN_STATUS["$domain"]="CRIT" ;;
3) FAILED+=("$domain"); DOMAIN_STATUS["$domain"]="FAIL" ;;
esac
echo ""
done
END=$(date +%s)
# ── Send notifications — batched per severity ─────────────────────────────────────────────────
if [[ "$DRY_RUN" == false ]]; then
[[ ${#CRITICAL[@]} -gt 0 ]] && \
notify "Certificate CRITICAL on $(hostname) — expiring within ${CERT_CRIT_DAYS} days: ${CRITICAL[*]}" "Certificate Monitor" "warning"
notify "Certificate CRITICAL on $(hostname) — expiring within ${CERT_CRIT_DAYS} days: ${CRITICAL[*]}" \
"Certificate Monitor" "warning"
[[ ${#WARNING[@]} -gt 0 ]] && \
notify "Certificate WARNING on $(hostname) — expiring within ${CERT_WARN_DAYS} days: ${WARNING[*]}" "Certificate Monitor" "warning"
notify "Certificate WARNING on $(hostname) — expiring within ${CERT_WARN_DAYS} days: ${WARNING[*]}" \
"Certificate Monitor" "warning"
[[ ${#FAILED[@]} -gt 0 ]] && \
notify "Certificate check FAILED on $(hostname) — could not reach: ${FAILED[*]}" "Certificate Monitor" "warning"
notify "Certificate check FAILED on $(hostname) — could not reach: ${FAILED[*]}" \
"Certificate Monitor" "warning"
fi
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Summary ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
echo ""
echo "━━━━━ $ICON_SUMMARY CERTIFICATE MONITOR SUMMARY ━━━━━"
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
echo ""
echo " $ICON_SUCCESS Healthy: ${#HEALTHY[@]} $ICON_WARN Warning: ${#WARNING[@]} $ICON_ERROR Critical: ${#CRITICAL[@]} Failed: ${#FAILED[@]}"
echo " $ICON_SUCCESS Healthy: ${#HEALTHY[@]}"
[[ ${#WARNING[@]} -gt 0 ]] && warn "Warning: ${#WARNING[@]} — renewal recommended"
[[ ${#CRITICAL[@]} -gt 0 ]] && echo "$ICON_ERROR Critical: ${#CRITICAL[@]} — ACTION REQUIRED"
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${#FAILED[@]} — unreachable"
echo ""
# Per-domain results — only show problems, healthy ones stay in log()
for domain in "${CERT_MONITOR_DOMAINS[@]}"; do
[[ -z "$domain" ]] && continue
case "${DOMAIN_STATUS[$domain]:-UNKN}" in
OK) echo " $ICON_SUCCESS $domain" ;;
WARN) echo " $ICON_WARN $domain" ;;
CRIT) echo " $ICON_ERROR $domain" ;;
FAIL) echo " $ICON_ERROR $domain (unreachable)" ;;
OK) log " $ICON_SUCCESS $domain — healthy" ;;
WARN) warn " $ICON_WARN $domain — warning" ;;
CRIT) echo " $ICON_ERROR $domain — CRITICAL" ;;
FAIL) echo " $ICON_ERROR $domain — unreachable" ;;
esac
done
echo ""
if [[ "$DRY_RUN" == true ]]; then
echo "$ICON_WARN Status: DRY RUN — no notifications sent"
warn "DRY RUN — no notifications sent"
elif [[ ${#CRITICAL[@]} -gt 0 || ${#FAILED[@]} -gt 0 ]]; then
echo "$ICON_ERROR Status: ACTION REQUIRED"
elif [[ ${#WARNING[@]} -gt 0 ]]; then
echo "$ICON_WARN Status: WARNINGS — renewal recommended"
warn "Status: WARNINGS — renewal recommended"
else
echo "$ICON_DONE Status: $ICON_SUCCESS ALL CERTS HEALTHY"
log "$ICON_DONE Status: all ${#HEALTHY[@]} certs healthy ✅"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
Monitors/continuous_scripts_status.sh
+140 -140
View File
@@ -1,35 +1,58 @@
#!/bin/bash
# -----------------------------------------------------------------------------------------------
# ----------------------------- Continuous Scripts Status --------------------------------------
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ========================= Continuous Scripts Status ==========================================
# ==============================================================================================
# Live status dashboard for all continuously running scripts in the ecosystem.
# Run manually anytime — no schedule, no cron.
#
# Covers all scripts started by array_start.sh that run until array stops:
# system_watchdog.sh — system health monitor
# docker_watchdog.sh — container health monitor
# failover.sh — mutual failover monitor
# ── WHAT IT SHOWS ─────────────────────────────────────────────────────────────────────────────
# For each continuous script (system_watchdog, docker_watchdog, failover):
# Running state, PID, uptime, approximate cycle count
# Active strikes and skip list
# Recent restart history
# Live health snapshot
#
# Shows for each:
# Running state, PID, uptime, current cycle
# Active strikes, skip list, recent actions
# Live system/container health snapshot
# Failover state, tier status, Tailscale connectivity
# system_watchdog — rootfs, RAM, ZFS ARC, load, zombie count, CPU temp
# docker_watchdog — running/stopped/unhealthy containers, required containers,
# monitored memory containers, recent restart history
# failover — current state, tier status, remote Tailscale visibility
#
# If a script is mid-cycle state files are read as-is — reflects last completed cycle.
# Run: bash Monitors/continuous_scripts_status.sh
# -----------------------------------------------------------------------------------------------
# If a script is mid-cycle, state files are read as-is — reflects last completed cycle.
#
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
# detect_hosts() sets MY_ID and aliases all HOST*_WATCHDOG_* arrays.
# Required containers, tier delays, and Tailscale checks use MY_ID/REMOTE_ID correctly.
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
# continuous_scripts_status.sh — show dashboard
# continuous_scripts_status.sh --log — verbose output
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../Master.conf"
source "$SCRIPT_DIR/../common.sh"
source "$SCRIPT_DIR/../load_config.sh"
# Dashboard script — output is the point
SILENT_MODE=false
parse_args "$@"
# -----------------------------------------------------------------------------------------------
# HELPERS
# -----------------------------------------------------------------------------------------------
DOCKER_TIMEOUT=15
# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
error "Must be run as root"
exit 1
fi
# detect_hosts() sets MY_ID and aliases all HOST*_WATCHDOG_* arrays
detect_hosts
# ==============================================================================================
# ── HELPER FUNCTIONS ──────────────────────────────────────────────────────────────────────────
# ==============================================================================================
get_lock_pid() {
local script_name="$1"
@@ -51,11 +74,10 @@ get_lock_name() {
fi
}
is_watchdog_running() {
is_script_running() {
local script_name="$1"
local pid
local pid locked_name
pid=$(get_lock_pid "$script_name")
local locked_name
locked_name=$(get_lock_name "$script_name")
[[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null && [[ "$locked_name" == "$script_name" ]]
}
@@ -73,6 +95,7 @@ get_lock_age() {
fi
}
# Human readable uptime — days/hours/mins
format_uptime() {
local seconds=$1
local days=$(( seconds / 86400 ))
@@ -87,34 +110,29 @@ format_uptime() {
fi
}
get_strikes() {
local state_file="$1"
local key="$2"
grep -E "^${key}:" "$state_file" 2>/dev/null | cut -d: -f2
}
divider() { printf '%.0s─' {1..57}; echo; }
section() { echo ""; echo " $1"; divider; }
divider() { printf '%.0s─' {1..55}; echo; }
header() { echo ""; echo " $1"; divider; }
# -----------------------------------------------------------------------------------------------
# ━━━ HEADER ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Header ━━━
# ==============================================================================================
clear
echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo " 🛡️ WATCHDOG STATUS — $(date '+%A, %B %-d at %-I:%M%p')"
echo " 🖥️ $(hostname)"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo " $ICON_HOST $MY_ID — $LOCAL_SERVER_NAME"
echo " $ICON_HOST Remote: $REMOTE_ID — $REMOTE_SERVER_NAME"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ System Watchdog ━━━
# -----------------------------------------------------------------------------------------------
header "⚙️ SYSTEM WATCHDOG"
# ==============================================================================================
section "⚙️ SYSTEM WATCHDOG"
SYS_PID=$(get_lock_pid "system_watchdog")
SYS_RUNNING=false
if is_watchdog_running "system_watchdog"; then
if is_script_running "system_watchdog"; then
SYS_RUNNING=true
SYS_AGE=$(get_lock_age "system_watchdog")
SYS_UPTIME=$(format_uptime "$SYS_AGE")
@@ -128,7 +146,7 @@ fi
echo ""
# System watchdog strikes
# System strikes
if [[ -f "$SYS_WATCHDOG_STATE_FILE" ]]; then
ACTIVE_STRIKES=$(grep -v ":0$" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$")
if [[ -n "$ACTIVE_STRIKES" ]]; then
@@ -149,8 +167,8 @@ if [[ -f "$SYS_WATCHDOG_REBOOT_LOG" ]]; then
TOTAL_REBOOTS=$(grep -c "." "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null || echo 0)
TOTAL_REBOOTS="${TOTAL_REBOOTS//[^0-9]/}"
TOTAL_REBOOTS="${TOTAL_REBOOTS:-0}"
WEEK_EPOCH=$(date -d "7 days ago" +%s)
WEEK_REBOOTS=$(awk -v cutoff="$WEEK_EPOCH" '$1 >= cutoff' \
WEEK_CUTOFF=$(date -d "7 days ago" '+%Y-%m-%d %H:%M:%S')
WEEK_REBOOTS=$(awk -v cutoff="$WEEK_CUTOFF" '$0 >= cutoff' \
"$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null | wc -l)
echo " 🔄 Watchdog reboots: $WEEK_REBOOTS this week / $TOTAL_REBOOTS total"
fi
@@ -159,7 +177,7 @@ fi
if [[ -f "$SYS_WATCHDOG_FAILED_FILE" ]] && [[ -s "$SYS_WATCHDOG_FAILED_FILE" ]]; then
SKIP_COUNT=$(wc -l < "$SYS_WATCHDOG_FAILED_FILE")
echo ""
echo " ⛔ Skip list ($SKIP_COUNT containers — manual intervention needed):"
echo " ⛔ Skip list ($SKIP_COUNT — manual intervention needed):"
while IFS= read -r container; do
[[ -z "$container" ]] && continue
echo " → $container"
@@ -168,17 +186,15 @@ else
echo " ✅ Skip list: empty"
fi
# Current system health snapshot
# Live system health snapshot
echo ""
echo " 📊 Current system state:"
# rootfs
ROOTFS_PCT=$(df / --output=pcent 2>/dev/null | tail -1 | tr -d ' %')
[[ "${ROOTFS_PCT:-0}" -ge "${SYS_WATCHDOG_ROOTFS_PCT:-95}" ]] && \
ROOTFS_ICON="⚠️ " || ROOTFS_ICON="✅"
echo " ${ROOTFS_ICON} rootfs: ${ROOTFS_PCT}% (threshold: ${SYS_WATCHDOG_ROOTFS_PCT}%)"
# RAM
MEM_AVAIL_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo)
MEM_FREE_GB=$(awk "BEGIN {printf \"%.1f\", $MEM_AVAIL_KB / 1048576}")
MEM_TOTAL_GB=$(awk '/MemTotal/ {printf "%.0f", $2/1048576}' /proc/meminfo)
@@ -186,7 +202,6 @@ MEM_TOTAL_GB=$(awk '/MemTotal/ {printf "%.0f", $2/1048576}' /proc/meminfo)
MEM_ICON="⚠️ " || MEM_ICON="✅"
echo " ${MEM_ICON} RAM: ${MEM_FREE_GB}GB free / ${MEM_TOTAL_GB}GB total (threshold: ${SYS_WATCHDOG_MEM_GB}GB free)"
# ARC
if [[ -f /proc/spl/kstat/zfs/arcstats ]]; then
ARC_SIZE=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats)
ARC_MAX=$(awk '/^c_max / {print $3}' /proc/spl/kstat/zfs/arcstats)
@@ -197,7 +212,6 @@ if [[ -f /proc/spl/kstat/zfs/arcstats ]]; then
echo " ${ARC_ICON} ZFS ARC: ${ARC_GB}GB (${ARC_PCT}% of max, threshold: ${SYS_WATCHDOG_ARC_PINNED_PCT}%)"
fi
# Load
LOAD=$(awk '{print $1}' /proc/loadavg)
CORES=$(nproc)
LOAD_THRESH=$(( CORES * ${SYS_WATCHDOG_LOAD_MULTIPLIER:-3} ))
@@ -205,17 +219,13 @@ LOAD_INT=$(printf "%.0f" "$LOAD")
[[ "$LOAD_INT" -ge "$LOAD_THRESH" ]] && LOAD_ICON="⚠️ " || LOAD_ICON="✅"
echo " ${LOAD_ICON} Load avg: $LOAD (threshold: ${LOAD_THRESH} = ${SYS_WATCHDOG_LOAD_MULTIPLIER}x ${CORES} cores)"
# Zombies
ZOMBIE_COUNT=$(ps aux | awk '{print $8}' | grep -c "^Z$" 2>/dev/null || echo 0)
ZOMBIE_COUNT="${ZOMBIE_COUNT//[^0-9]/}"
ZOMBIE_COUNT="${ZOMBIE_COUNT:-0}"
ZOMBIE_COUNT="${ZOMBIE_COUNT//[^0-9]/}"
ZOMBIE_COUNT="${ZOMBIE_COUNT:-0}"
[[ "$ZOMBIE_COUNT" -ge "${SYS_WATCHDOG_ZOMBIE_LIMIT:-50}" ]] && \
ZOMBIE_ICON="⚠️ " || ZOMBIE_ICON="✅"
echo " ${ZOMBIE_ICON} Zombies: $ZOMBIE_COUNT (threshold: ${SYS_WATCHDOG_ZOMBIE_LIMIT})"
# CPU temp
if command -v sensors >/dev/null 2>&1; then
CPU_TEMP=$(sensors 2>/dev/null | \
grep -i "Package id 0\|Tctl\|CPU Temp" | \
@@ -228,15 +238,15 @@ if command -v sensors >/dev/null 2>&1; then
fi
fi
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Docker Watchdog ━━━
# -----------------------------------------------------------------------------------------------
header "🐳 DOCKER WATCHDOG"
# ==============================================================================================
section "🐳 DOCKER WATCHDOG"
DOCKER_PID=$(get_lock_pid "docker_watchdog")
DOCKER_RUNNING=false
if is_watchdog_running "docker_watchdog"; then
if is_script_running "docker_watchdog"; then
DOCKER_RUNNING=true
DOCKER_AGE=$(get_lock_age "docker_watchdog")
DOCKER_UPTIME=$(format_uptime "$DOCKER_AGE")
@@ -250,7 +260,7 @@ fi
echo ""
# Container watchdog strikes
# Container strikes
if [[ -f "$WATCHDOG_STATE_FILE" ]]; then
ACTIVE_CONTAINER_STRIKES=$(grep -v ":0$" "$WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$")
if [[ -n "$ACTIVE_CONTAINER_STRIKES" ]]; then
@@ -264,16 +274,15 @@ if [[ -f "$WATCHDOG_STATE_FILE" ]]; then
fi
fi
# Container restart history this week
# Container restart history
if [[ -f "$WATCHDOG_CONTAINER_RESTART_LOG" ]]; then
WEEK_EPOCH=$(date -d "7 days ago" +%s 2>/dev/null || date -v-7d +%s 2>/dev/null)
WEEK_RESTARTS=$(awk -F'|' -v cutoff="$WEEK_EPOCH" \
'NR>0 {if ($2 >= cutoff) count++} END {print count+0}' \
"$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null)
WEEK_CUTOFF=$(date -d "7 days ago" '+%Y-%m-%d %H:%M:%S')
WEEK_RESTARTS=$(awk -F'|' -v cutoff="$WEEK_CUTOFF" \
'$2 >= cutoff' "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | wc -l)
if [[ "${WEEK_RESTARTS:-0}" -gt 0 ]]; then
echo ""
echo " 🔄 Container restarts this week: $WEEK_RESTARTS"
awk -F'|' -v cutoff="$WEEK_EPOCH" \
awk -F'|' -v cutoff="$WEEK_CUTOFF" \
'$2 >= cutoff {print $1}' "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | \
sort | uniq -c | sort -rn | head -5 | \
while read -r count name; do
@@ -284,31 +293,34 @@ if [[ -f "$WATCHDOG_CONTAINER_RESTART_LOG" ]]; then
fi
fi
# Docker container overview
# Container overview
echo ""
echo " 📦 Container overview:"
if command -v docker >/dev/null 2>&1; then
RUNNING=$(docker ps -q 2>/dev/null | wc -l)
TOTAL=$(docker ps -aq 2>/dev/null | wc -l)
UNHEALTHY=$(docker ps --filter health=unhealthy -q 2>/dev/null | wc -l)
RUNNING=$(timeout "$DOCKER_TIMEOUT" docker ps -q 2>/dev/null | wc -l)
TOTAL=$(timeout "$DOCKER_TIMEOUT" docker ps -aq 2>/dev/null | wc -l)
UNHEALTHY=$(timeout "$DOCKER_TIMEOUT" docker ps \
--filter health=unhealthy -q 2>/dev/null | wc -l)
# Filter intentionally stopped containers from the stopped list
# Stopped containers — filter intentionally ignored ones
STOPPED_FILTERED=()
while IFS= read -r name; do
[[ -z "$name" ]] && continue
SKIP=false
local SKIP=false
for ignore in "${WATCHDOG_SCAN_IGNORE[@]:-}"; do
[[ "$name" == "$ignore" ]] && SKIP=true && break
done
[[ "$SKIP" == false ]] && STOPPED_FILTERED+=("$name")
done < <(docker ps -af "status=exited" --format "{{.Names}}" 2>/dev/null)
STOPPED_COUNT="${#STOPPED_FILTERED[@]}"
done < <(timeout "$DOCKER_TIMEOUT" docker ps -af "status=exited" \
--format "{{.Names}}" 2>/dev/null)
STOPPED_COUNT="${#STOPPED_FILTERED[@]}"
echo " Running: $RUNNING / $TOTAL total"
[[ "$UNHEALTHY" -gt 0 ]] && echo " ⚠️ Unhealthy: $UNHEALTHY"
if [[ "$STOPPED_COUNT" -gt 0 ]]; then
echo " ⚠️ Stopped containers (unexpected):"
echo " ⚠️ Stopped (unexpected):"
for name in "${STOPPED_FILTERED[@]}"; do
echo " → $name"
done
@@ -316,40 +328,35 @@ if command -v docker >/dev/null 2>&1; then
echo " ✅ All containers running"
fi
# Check required containers
detect_hosts 2>/dev/null
if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then
REQUIRED=("${HOST1_WATCHDOG_REQUIRED_CONTAINERS[@]}")
else
REQUIRED=("${HOST2_WATCHDOG_REQUIRED_CONTAINERS[@]}")
fi
# Required containers — aliased by detect_hosts() → WATCHDOG_REQUIRED_CONTAINERS
REQUIRED_ISSUES=0
if [[ ${#REQUIRED[@]} -gt 0 ]]; then
if [[ ${#WATCHDOG_REQUIRED_CONTAINERS[@]} -gt 0 ]]; then
echo ""
echo " 🔐 Required containers:"
for container in "${REQUIRED[@]}"; do
for container in "${WATCHDOG_REQUIRED_CONTAINERS[@]}"; do
[[ -z "$container" ]] && continue
STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null || echo "not found")
STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
'{{.State.Running}}' "$container" 2>/dev/null || echo "not found")
if [[ "$STATUS" == "true" ]]; then
echo " ✅ $container"
else
echo " ❌ $container — $STATUS"
((REQUIRED_ISSUES++))
(( REQUIRED_ISSUES++ ))
fi
done
fi
# Tier 1 monitored containers from WATCHDOG_CONTAINERS
# Memory-monitored containers — aliased by detect_hosts() → WATCHDOG_CONTAINERS
if [[ ${#WATCHDOG_CONTAINERS[@]} -gt 0 ]]; then
echo ""
echo " 📊 Monitored containers (memory):"
for container in "${!WATCHDOG_CONTAINERS[@]}"; do
LIMIT_MB="${WATCHDOG_CONTAINERS[$container]}"
LIMIT_GB=$(awk "BEGIN {printf \"%.0f\", $LIMIT_MB / 1024}")
USAGE=$(docker stats --no-stream --format "{{.MemUsage}}" "$container" \
2>/dev/null | awk '{print $1}')
STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null || echo "not found")
USAGE=$(timeout "$DOCKER_TIMEOUT" docker stats --no-stream \
--format "{{.MemUsage}}" "$container" 2>/dev/null | awk '{print $1}')
STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
'{{.State.Running}}' "$container" 2>/dev/null || echo "not found")
if [[ "$STATUS" == "true" ]]; then
echo " ✅ $container: ${USAGE:-?} (limit: ${LIMIT_GB}GB)"
else
@@ -361,22 +368,22 @@ else
echo " Docker not available"
fi
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Failover ━━━
# -----------------------------------------------------------------------------------------------
header "🔀 FAILOVER"
# ==============================================================================================
section "🔀 FAILOVER"
FAILOVER_PID=$(get_lock_pid "failover")
FAILOVER_RUNNING=false
if is_watchdog_running "failover"; then
if is_script_running "failover"; then
FAILOVER_RUNNING=true
FAILOVER_AGE=$(get_lock_age "failover")
FAILOVER_UPTIME=$(format_uptime "$FAILOVER_AGE")
echo " ✅ Running │ PID: $FAILOVER_PID │ Uptime: $FAILOVER_UPTIME"
else
if [[ "${FAILOVER_ENABLED:-true}" == false ]]; then
echo " ⏸️ Disabled — FAILOVER_ENABLED=false in Master.conf"
echo " ⏸️ Disabled — FAILOVER_ENABLED=false in master.conf"
else
echo " ❌ NOT RUNNING — failover.sh is not active"
echo " Start via: bash Orchestrators/array_start.sh"
@@ -387,60 +394,55 @@ echo ""
# Failover state
FAILOVER_STATE="UNKNOWN"
FAILOVER_LAST_CHANGE=""
FAILOVER_STATE_SECONDS=0
if [[ -f "$FAILOVER_STATE_FILE" ]]; then
FAILOVER_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
FAILOVER_LAST_CHANGE=$(grep "^last_change=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
FAILOVER_LAST_EPOCH=$(grep "^last_change_epoch=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
if [[ -n "$FAILOVER_LAST_EPOCH" ]]; then
FAILOVER_LAST_EPOCH=$(grep "^failover_start=" "$FAILOVER_STATE_FILE" \
2>/dev/null | cut -d= -f2)
if [[ -n "$FAILOVER_LAST_EPOCH" && "$FAILOVER_LAST_EPOCH" -gt 0 ]]; then
FAILOVER_STATE_SECONDS=$(( $(date +%s) - FAILOVER_LAST_EPOCH ))
fi
fi
STATE_DURATION=$(format_uptime "${FAILOVER_STATE_SECONDS:-0}")
# Tier delays via REMOTE_ID — same logic as failover.sh
REMOTE_TIER2_VAR="${REMOTE_ID}_TIER2_DELAY"
REMOTE_TIER3_VAR="${REMOTE_ID}_TIER3_DELAY"
REMOTE_TIER4_VAR="${REMOTE_ID}_TIER4_DELAY"
TIER2_DELAY="${!REMOTE_TIER2_VAR:-240}"
TIER3_DELAY="${!REMOTE_TIER3_VAR:-720}"
TIER4_DELAY="${!REMOTE_TIER4_VAR:-1440}"
case "$FAILOVER_STATE" in
NORMAL)
echo " ✅ State: NORMAL"
echo " 📅 In NORMAL state for: $STATE_DURATION"
;;
FAILOVER)
echo " ⚠️ State: FAILOVER — remote server down"
echo " ⚠️ State: FAILOVER — $REMOTE_SERVER_NAME is down"
echo " ⏱️ Duration: $STATE_DURATION"
# Show which tiers are active
TIER1_DELAY=0
if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then
TIER2_DELAY=$HOST2_TIER2_DELAY
TIER3_DELAY=$HOST2_TIER3_DELAY
TIER4_DELAY=$HOST2_TIER4_DELAY
else
TIER2_DELAY=$HOST1_TIER2_DELAY
TIER3_DELAY=$HOST1_TIER3_DELAY
TIER4_DELAY=$HOST1_TIER4_DELAY
fi
FAILOVER_MINS=$(( FAILOVER_STATE_SECONDS / 60 ))
echo ""
echo " 🔄 Tier status:"
echo " Tier 1 (immediate): ✅ active"
echo " Tier 1 (immediate): ✅ active"
if (( FAILOVER_MINS >= TIER2_DELAY )); then
echo " Tier 2 (${TIER2_DELAY}min): ✅ active"
echo " Tier 2 (${TIER2_DELAY}min): ✅ active"
else
REMAINING=$(( TIER2_DELAY - FAILOVER_MINS ))
echo " Tier 2 (${TIER2_DELAY}min): ⏳ activates in ${REMAINING}min"
echo " Tier 2 (${TIER2_DELAY}min): ⏳ in ${REMAINING}min"
fi
if (( FAILOVER_MINS >= TIER3_DELAY )); then
echo " Tier 3 (${TIER3_DELAY}min): ✅ active"
echo " Tier 3 (${TIER3_DELAY}min): ✅ active"
else
REMAINING=$(( TIER3_DELAY - FAILOVER_MINS ))
echo " Tier 3 (${TIER3_DELAY}min): ⏳ activates in ${REMAINING}min"
echo " Tier 3 (${TIER3_DELAY}min): ⏳ in ${REMAINING}min"
fi
if (( FAILOVER_MINS >= TIER4_DELAY )); then
echo " Tier 4 (${TIER4_DELAY}min): ✅ active"
echo " Tier 4 (${TIER4_DELAY}min): ✅ active"
else
REMAINING=$(( TIER4_DELAY - FAILOVER_MINS ))
echo " Tier 4 (${TIER4_DELAY}min): ⏳ activates in ${REMAINING}min"
echo " Tier 4 (${TIER4_DELAY}min): ⏳ in ${REMAINING}min"
fi
;;
NO_INTERNET)
@@ -448,7 +450,7 @@ case "$FAILOVER_STATE" in
echo " ⏱️ Down for: $STATE_DURATION"
;;
DARK)
echo " ❌ State: DARK — remote down AND no internet"
echo " ❌ State: DARK — $REMOTE_SERVER_NAME down AND no internet"
echo " ⏱️ Duration: $STATE_DURATION"
;;
*)
@@ -456,19 +458,18 @@ case "$FAILOVER_STATE" in
;;
esac
# Tailscale remote visibility
# Tailscale remote visibility — uses REMOTE_SERVER_NAME from detect_hosts()
echo ""
if command -v tailscale >/dev/null 2>&1; then
REMOTE_IP=$(tailscale ip -4 "$HOST2" 2>/dev/null)
REMOTE_IP=$(tailscale ip -4 "$REMOTE_SERVER_NAME" 2>/dev/null)
if [[ -n "$REMOTE_IP" ]]; then
# Try a quick ping to see last seen
if ping -c 1 -W 2 "$REMOTE_IP" >/dev/null 2>&1; then
echo " 🌐 Remote: $REMOTE_IP │ reachable ✅"
echo " 🌐 $REMOTE_ID ($REMOTE_SERVER_NAME): $REMOTE_IP — reachable ✅"
else
echo " 🌐 Remote: $REMOTE_IP │ not responding ⚠️"
echo " 🌐 $REMOTE_ID ($REMOTE_SERVER_NAME): $REMOTE_IP — not responding ⚠️"
fi
else
echo " 🌐 Remote: $HOST2 not visible on Tailscale ❌"
echo " 🌐 $REMOTE_ID ($REMOTE_SERVER_NAME) not visible on Tailscale ❌"
fi
else
echo " 🌐 Tailscale: not available"
@@ -476,28 +477,27 @@ fi
echo " 📡 Check interval: ${FAILOVER_CHECK_INTERVAL}s │ Handback strikes: ${FAILOVER_HANDBACK_STRIKES}"
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Footer ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
# Overall status
ISSUES=0
[[ "$SYS_RUNNING" == false ]] && ((ISSUES++))
[[ "$DOCKER_RUNNING" == false ]] && ((ISSUES++))
[[ "$FAILOVER_RUNNING" == false ]] && [[ "${FAILOVER_ENABLED:-true}" != false ]] && ((ISSUES++))
[[ -n "$ACTIVE_STRIKES" ]] && ((ISSUES++))
[[ -n "$ACTIVE_CONTAINER_STRIKES" ]] && ((ISSUES++))
[[ "${REQUIRED_ISSUES:-0}" -gt 0 ]] && ((ISSUES++))
[[ "$FAILOVER_STATE" != "NORMAL" ]] && [[ "$FAILOVER_STATE" != "UNKNOWN" ]] && ((ISSUES++))
[[ "$SYS_RUNNING" == false ]] && (( ISSUES++ ))
[[ "$DOCKER_RUNNING" == false ]] && (( ISSUES++ ))
[[ "$FAILOVER_RUNNING" == false && "${FAILOVER_ENABLED:-true}" != false ]] && (( ISSUES++ ))
[[ -n "$ACTIVE_STRIKES" ]] && (( ISSUES++ ))
[[ -n "$ACTIVE_CONTAINER_STRIKES" ]] && (( ISSUES++ ))
[[ "${REQUIRED_ISSUES:-0}" -gt 0 ]] && (( ISSUES++ ))
[[ "$FAILOVER_STATE" != "NORMAL" && "$FAILOVER_STATE" != "UNKNOWN" ]] && (( ISSUES++ ))
if [[ "$ISSUES" -eq 0 ]]; then
echo " ✅ All continuous scripts healthy — no issues detected"
echo " ✅ $MY_ID — all continuous scripts healthy"
else
echo " ⚠️ $ISSUES issue(s) detected — review above"
fi
echo " 🕐 Checked at: $(date '+%H:%M:%S')"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo " 🕐 Checked: $(date '+%H:%M:%S')"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo ""
Monitors/emby_session_report.sh
+214 -78
View File
@@ -1,37 +1,75 @@
#!/bin/bash
# -----------------------------------------------------------------------------------------------
# --------------------------------- Emby Session Report ----------------------------------------
# -----------------------------------------------------------------------------------------------
# Generates a weekly usage report from the Emby media server via its API.
# Queries activity logs, session history and library stats to produce a
# human-readable summary of what was watched, by whom and how.
# ==============================================================================================
# ================================= Emby Session Report ========================================
# ==============================================================================================
# Generates a usage report from the Emby media server via its API.
# Queries activity logs and session history to produce a summary of what was
# watched, by whom, and how over the configured report period.
#
# Report includes:
# Total streams during the report period
# Transcode vs direct play ratio
# Live TV usage
# Top N most watched content
# Most active users
# Peak concurrent streams
# ── REPORT INCLUDES ───────────────────────────────────────────────────────────────────────────
# Server info — name, version, uptime
# Active sessions — current streams, direct play vs transcode
# Library stats — movie, episode, song counts
# Activity history — play events from the last EMBY_REPORT_DAYS days
# Top content — most played items in the period (top EMBY_REPORT_TOP_N)
# Most active users — who watched the most in the period
# Transcode ratio — how often transcoding was needed vs direct play
# Ramdisk status — current transcode location and usage
#
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
# detect_hosts() sets MY_ID and aliases EMBY_URL and EMBY_API_KEY.
# Each server reports on its own Emby instance automatically.
#
# ── BEHAVIOUR ─────────────────────────────────────────────────────────────────────────────────
# No persistent writes — queries API fresh each run.
# All configuration in Master.conf under Emby Session Report section.
# Supports --dry-run to test API connectivity without sending notification.
# -----------------------------------------------------------------------------------------------
# This is a monitor/report script — SILENT_MODE=false — output is the point.
# Silent when healthy (no notification on clean run).
# Notifies only if transcoding is very high (>80% of streams) — may indicate config issue.
#
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
# acquire_lock — prevents duplicate reports running simultaneously
# check_api() — verifies Emby reachable before queries
# jq + curl validation — exits if either tool missing
# validate_unraid_cmd — notify script validated before use
# Per-section guards — API failure in one section does not abort others
#
# ── CONFIGURATION (master_host*.conf) ─────────────────────────────────────────────────────────
# HOST*_EMBY_URL / HOST*_EMBY_API_KEY
# Aliased by detect_hosts() — script uses EMBY_URL / EMBY_API_KEY
#
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
# EMBY_REPORT_DAYS — days to include in the report period (default 7)
# EMBY_REPORT_TOP_N — number of top content items to show (default 10)
# RAMDISK_PATH — ramdisk mount path (for transcode status)
# TRANSCODE_LINK — symlink path (for transcode location)
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
# emby_session_report.sh — generate report
# emby_session_report.sh --dry-run — test API connectivity only, no notification
# emby_session_report.sh --log — verbose output
# emby_session_report.sh --status — show config and exit
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../Master.conf"
source "$SCRIPT_DIR/../common.sh"
source "$SCRIPT_DIR/../load_config.sh"
# Monitor/report script — output is the point
SILENT_MODE=false
parse_args "$@"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Setup ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_GEAR Setup ━━━"
if [[ "$EUID" -ne 0 ]]; then
error "Must be run as root"
exit 1
fi
if ! command -v curl >/dev/null 2>&1; then
error "curl not found — required for Emby API calls"
exit 1
@@ -43,28 +81,40 @@ if ! command -v jq >/dev/null 2>&1; then
exit 1
fi
# Select correct Emby instance based on which server is running this script
validate_unraid_cmd \
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
"" "" \
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
acquire_lock
# detect_hosts() sets MY_ID and aliases EMBY_URL, EMBY_API_KEY
detect_hosts
if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then
EMBY_URL="$HOST1_EMBY_URL"
EMBY_API_KEY="$HOST1_EMBY_API_KEY"
else
EMBY_URL="$HOST2_EMBY_URL"
EMBY_API_KEY="$HOST2_EMBY_API_KEY"
fi
info "Emby instance: $LOCAL_SERVER_NAME → $EMBY_URL"
require_var EMBY_URL
require_var EMBY_API_KEY
success "Config validated"
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — API will be queried but no notification sent"
log "Emby: $EMBY_URL"
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — API queried but no notification sent"
# -----------------------------------------------------------------------------------------------
# API HELPER
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
echo ""
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_EMBY Emby URL: $EMBY_URL"
echo "$ICON_TIME Period: Last ${EMBY_REPORT_DAYS} days"
echo "$ICON_EMBY Top N: ${EMBY_REPORT_TOP_N} items"
echo "$ICON_GEAR Dry Run: $DRY_RUN"
echo "━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
fi
# ==============================================================================================
# ── API HELPER ────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
emby_api() {
local endpoint="$1"
local response http_code body
@@ -79,54 +129,66 @@ emby_api() {
body=$(echo "$response" | head -n -1)
if [[ "$http_code" != "200" ]]; then
error "Emby API returned HTTP $http_code for: $endpoint"
error "Emby API HTTP $http_code for: $endpoint"
return 1
fi
echo "$body"
}
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_EMBY Emby Session Report ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Emby Session Report ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_EMBY Emby Session Report — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
echo "$ICON_EMBY URL: $EMBY_URL"
echo "$ICON_TIME Period: Last ${EMBY_REPORT_DAYS} days"
echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_TIME Period: Last ${EMBY_REPORT_DAYS} days"
echo ""
START=$(date +%s)
# Test connectivity
info "Testing Emby API connectivity..."
# ── Connectivity and server info ──────────────────────────────────────────────────────────────
if ! check_api "$EMBY_URL" "Emby" 10; then
notify "Emby report failed on $(hostname) — cannot connect to Emby at $EMBY_URL" \
"Emby Report" "warning"
exit 1
fi
SYSTEM_INFO=$(emby_api "System/Info" 2>/dev/null) || {
error "Cannot connect to Emby at $EMBY_URL"
notify "Emby report failed on $(hostname) — cannot connect to Emby" "Emby Report" "warning"
exit 1
}
SERVER_NAME=$(echo "$SYSTEM_INFO" | jq -r '.ServerName // "Unknown"' 2>/dev/null)
SERVER_VERSION=$(echo "$SYSTEM_INFO" | jq -r '.Version // "Unknown"' 2>/dev/null)
success "Connected to: $SERVER_NAME (v$SERVER_VERSION)"
echo ""
log "Connected to: $SERVER_NAME (v$SERVER_VERSION)"
# Calculate date range
REPORT_START=$(date -d "${EMBY_REPORT_DAYS} days ago" '+%Y-%m-%dT00:00:00')
# ── Active Sessions ──────────────────────────────────────────────────────────────────────────
# ── Active Sessions ───────────────────────────────────────────────────────────────────────────
echo "━━━ $ICON_EMBY Active Sessions ━━━"
SESSIONS=$(emby_api "Sessions" 2>/dev/null) || { warn "Could not fetch sessions"; SESSIONS="[]"; }
ACTIVE_COUNT=$(echo "$SESSIONS" | jq '[.[] | select(.NowPlayingItem != null)] | length' 2>/dev/null || echo 0)
TRANSCODE_COUNT=$(echo "$SESSIONS" | jq '[.[] | select(.NowPlayingItem != null) | select(.TranscodingInfo != null)] | length' 2>/dev/null || echo 0)
DIRECT_COUNT=$(( ACTIVE_COUNT - TRANSCODE_COUNT ))
ACTIVE_COUNT=$(echo "$SESSIONS" | \
jq '[.[] | select(.NowPlayingItem != null)] | length' 2>/dev/null || echo 0)
TRANSCODE_NOW=$(echo "$SESSIONS" | \
jq '[.[] | select(.NowPlayingItem != null) | select(.TranscodingInfo != null)] | length' \
2>/dev/null || echo 0)
DIRECT_NOW=$(( ACTIVE_COUNT - TRANSCODE_NOW ))
echo " $ICON_EMBY Active streams: $ACTIVE_COUNT"
echo " $ICON_EMBY Direct play: $DIRECT_COUNT"
echo " $ICON_EMBY Transcoding: $TRANSCODE_COUNT"
echo " $ICON_EMBY Active streams: $ACTIVE_COUNT"
echo " $ICON_EMBY Direct play: $DIRECT_NOW"
echo " $ICON_EMBY Transcoding: $TRANSCODE_NOW"
if [[ "$ACTIVE_COUNT" -gt 0 ]]; then
echo ""
echo " Now playing:"
echo "$SESSIONS" | jq -r '
.[] |
select(.NowPlayingItem != null) |
" \(.UserName // "Unknown") → \(.NowPlayingItem.Name // "Unknown") [\(if .TranscodingInfo != null then "transcode" else "direct" end)]"
' 2>/dev/null || true
fi
echo ""
# ── Library Stats ────────────────────────────────────────────────────────────────────────────
# ── Library Stats ─────────────────────────────────────────────────────────────────────────────
echo "━━━ $ICON_EMBY Library ━━━"
ITEMS=$(emby_api "Items/Counts" 2>/dev/null) || { warn "Could not fetch library counts"; ITEMS="{}"; }
@@ -139,31 +201,105 @@ echo " $ICON_EMBY Episodes: $EPISODE_COUNT"
echo " $ICON_EMBY Songs: $SONG_COUNT"
echo ""
# ── Ramdisk Status (from state file) ────────────────────────────────────────────────────────
echo "━━━ $ICON_RAM Transcode Location ━━━"
if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
RAMDISK_USED_KB=$(df "$RAMDISK_PATH" --output=used | tail -1 | tr -d ' ')
RAMDISK_USED_GB=$(awk "BEGIN {printf \"%.2f\", $RAMDISK_USED_KB / 1048576}")
SYMLINK=$(readlink "$TRANSCODE_LINK" 2>/dev/null || echo "unknown")
echo " $ICON_RAM Ramdisk usage: ${RAMDISK_USED_GB}GB"
echo " $ICON_LINK Symlink target: $SYMLINK"
# ── Activity History ──────────────────────────────────────────────────────────────────────────
# Query activity log for the configured period
echo "━━━ $ICON_EMBY Activity — Last ${EMBY_REPORT_DAYS} Days ━━━"
REPORT_START=$(date -d "${EMBY_REPORT_DAYS} days ago" '+%Y-%m-%dT00:00:00.000Z')
ACTIVITY=$(emby_api "System/ActivityLog/Entries?MinDate=${REPORT_START}&Limit=1000" \
2>/dev/null) || { warn "Could not fetch activity log"; ACTIVITY="{}"; }
TOTAL_PLAYS=$(echo "$ACTIVITY" | \
jq '[.Items // [] | .[] | select(.Type == "VideoPlayback" or .Type == "AudioPlayback")] | length' \
2>/dev/null || echo 0)
TRANSCODE_PLAYS=$(echo "$ACTIVITY" | \
jq '[.Items // [] | .[] | select(.Type == "VideoPlaybackUnplugged" or
(.Type == "VideoPlayback" and (.Overview // "" | contains("Transcode"))))] | length' \
2>/dev/null || echo 0)
echo " $ICON_EMBY Total play events: $TOTAL_PLAYS"
if [[ "$TOTAL_PLAYS" -gt 0 ]]; then
TRANSCODE_PCT=$(awk "BEGIN {printf \"%.0f\", ($TRANSCODE_PLAYS / $TOTAL_PLAYS) * 100}")
DIRECT_PCT=$(( 100 - TRANSCODE_PCT ))
echo " $ICON_EMBY Direct play: ~${DIRECT_PCT}%"
echo " $ICON_EMBY Transcoded: ~${TRANSCODE_PCT}%"
fi
echo ""
# ── Top Content ───────────────────────────────────────────────────────────────────────────────
echo "━━━ $ICON_EMBY Top ${EMBY_REPORT_TOP_N} Content ━━━"
TOP_ITEMS=$(emby_api "Items?SortBy=DatePlayed&SortOrder=Descending&Limit=${EMBY_REPORT_TOP_N}&Recursive=true&Fields=Overview&IncludeItemTypes=Movie,Episode" \
2>/dev/null) || { warn "Could not fetch top content"; TOP_ITEMS="{}"; }
TOP_COUNT=$(echo "$TOP_ITEMS" | jq '.Items // [] | length' 2>/dev/null || echo 0)
if [[ "$TOP_COUNT" -gt 0 ]]; then
echo "$TOP_ITEMS" | jq -r '
.Items // [] |
to_entries[] |
" \(.key + 1). \(.value.Name // "Unknown") [\(.value.Type // "")]"
' 2>/dev/null || warn "Could not parse top content"
else
echo " $ICON_RAM Ramdisk: not mounted"
echo " No recent play history found"
fi
echo ""
# ── Most Active Users ─────────────────────────────────────────────────────────────────────────
echo "━━━ $ICON_EMBY Most Active Users ━━━"
USERS=$(emby_api "Users" 2>/dev/null) || { warn "Could not fetch users"; USERS="[]"; }
USER_COUNT=$(echo "$USERS" | jq 'length' 2>/dev/null || echo 0)
echo " $ICON_EMBY Total users: $USER_COUNT"
if [[ "$USER_COUNT" -gt 0 ]]; then
echo "$USERS" | jq -r '
sort_by(.LastActivityDate // "0") |
reverse |
.[:5][] |
" \(.Name // "Unknown") — last active: \(.LastActivityDate // "never" | split("T")[0])"
' 2>/dev/null || true
fi
echo ""
# ── Ramdisk / Transcode Status ────────────────────────────────────────────────────────────────
echo "━━━ $ICON_RAM Transcode Status ━━━"
if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
RAMDISK_USED_KB=$(df "$RAMDISK_PATH" --output=used 2>/dev/null | tail -1 | tr -d ' ')
RAMDISK_USED_GB=$(awk "BEGIN {printf \"%.2f\", ${RAMDISK_USED_KB:-0} / 1048576}")
SYMLINK=$(readlink "$TRANSCODE_LINK" 2>/dev/null || echo "unknown")
echo " $ICON_RAM Ramdisk usage: ${RAMDISK_USED_GB}GB / ${RAMDISK_SIZE:-8G}"
echo " $ICON_LINK Symlink target: $SYMLINK"
if [[ "$SYMLINK" == *"ssd"* ]] || [[ "$SYMLINK" == *"cache"* ]]; then
warn "Transcode link pointing at SSD — ramdisk may be full"
fi
else
warn "Ramdisk not mounted at $RAMDISK_PATH"
fi
echo ""
END=$(date +%s)
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Summary ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
echo "━━━━━ $ICON_SUMMARY EMBY REPORT SUMMARY ━━━━━"
echo "$ICON_EMBY Server: $SERVER_NAME (v$SERVER_VERSION)"
echo "$ICON_EMBY Active: $ACTIVE_COUNT streams ($DIRECT_COUNT direct / $TRANSCODE_COUNT transcode)"
echo "$ICON_EMBY Library: $MOVIE_COUNT movies $EPISODE_COUNT episodes $SONG_COUNT songs"
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_EMBY Server: $SERVER_NAME (v$SERVER_VERSION)"
echo "$ICON_EMBY Active: $ACTIVE_COUNT streams ($DIRECT_NOW direct / $TRANSCODE_NOW transcode)"
echo "$ICON_EMBY Library: $MOVIE_COUNT movies $EPISODE_COUNT episodes $SONG_COUNT songs"
echo "$ICON_EMBY Period: $TOTAL_PLAYS play events in last ${EMBY_REPORT_DAYS} days"
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
# Only notify on issues — high transcode rate may indicate config problem
if [[ "$DRY_RUN" == false ]]; then
notify "Emby report on $(hostname) — $ACTIVE_COUNT active streams ($DIRECT_COUNT direct / $TRANSCODE_COUNT transcode) — Library: $MOVIE_COUNT movies $EPISODE_COUNT episodes" "Emby Report" "normal"
fi
if [[ "$TOTAL_PLAYS" -gt 10 && "${TRANSCODE_PCT:-0}" -gt 80 ]]; then
notify "Emby report on $(hostname) — high transcode rate: ${TRANSCODE_PCT}% of $TOTAL_PLAYS plays — check direct play config" \
"Emby Report" "warning"
fi
fi
exit 0
Monitors/smart_health.sh
+195 -95
View File
@@ -1,35 +1,67 @@
#!/bin/bash
# -----------------------------------------------------------------------------------------------
# --------------------------------- SMART Health Monitor ---------------------------------------
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ================================= SMART Health Monitor =======================================
# ==============================================================================================
# Checks SMART health attributes for all drives on the system.
# Reads data live from each drive via smartctl — no persistent writes.
# Designed to run weekly as a scheduled report.
#
# Monitored attributes:
# Reallocated_Sector_Ct — bad sectors remapped — any > 0 is concerning
# Current_Pending_Sector — sectors waiting for reallocation — any > 0 is concerning
# Offline_Uncorrectable — sectors that could not be corrected — any > 0 is critical
# Temperature_Celsius — drive temperature vs SMART_TEMP_WARN / SMART_TEMP_CRIT
# Power_On_Hours — informational — drive age estimation
# SMART overall status — pass/fail per drive
# ── MONITORED ATTRIBUTES ──────────────────────────────────────────────────────────────────────
# Overall SMART status — PASSED/FAILED — immediate fail = drive is dying
# Reallocated_Sector_Ct — bad sectors remapped — any > 0 is concerning
# Current_Pending_Sector — sectors waiting for reallocation — any > 0 is concerning
# Offline_Uncorrectable — sectors that could not be corrected — any > 0 is critical
# Temperature_Celsius — vs thresholds from dynamix.cfg (or master.conf fallback)
# Power_On_Hours — informational — drive age in days
#
# Discovers drives automatically — no configuration needed for drive list.
# SMART_IGNORE_DRIVES allows skipping specific drives (e.g. USB flash drives).
# ── DRIVE DISCOVERY ───────────────────────────────────────────────────────────────────────────
# Discovers drives automatically via /dev/sd* and /dev/nvme* — no config needed.
# NVMe drives use different attribute names — detected and handled automatically.
# HOST*_SMART_IGNORE_DRIVES skips specific drives (e.g. boot USB flash drive).
#
# All configuration in Master.conf under SMART Health section.
# Supports --dry-run to show which drives would be checked without running smartctl.
# -----------------------------------------------------------------------------------------------
# ── TEMPERATURE THRESHOLDS ────────────────────────────────────────────────────────────────────
# Reads hot/max/hotssd/maxssd from /boot/config/plugins/dynamix/dynamix.cfg at runtime.
# Uses unRAID's own configured thresholds — no need to duplicate them here.
# Falls back to SMART_TEMP_WARN / SMART_TEMP_CRIT from master.conf if dynamix.cfg not found.
#
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
# detect_hosts() sets MY_ID and aliases HOST*_SMART_IGNORE_DRIVES → SMART_IGNORE_DRIVES.
# Each server monitors its own drives with its own ignore list.
#
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
# acquire_lock — smartctl calls are slow, prevent duplicate runs
# detect_hosts() — correct ignore list per host via MY_ID aliases
# validate_unraid_cmd — smartctl and notify validated before use
# Silent healthy drives — only problems produce output
# Silent healthy run — no notify when all drives pass
#
# ── CONFIGURATION (master_host*.conf) ─────────────────────────────────────────────────────────
# HOST*_SMART_IGNORE_DRIVES — drives skipped in SMART monitoring
# Aliased by detect_hosts() — script uses SMART_IGNORE_DRIVES
#
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
# SMART_TEMP_WARN — fallback warn threshold in °C (if dynamix.cfg not found)
# SMART_TEMP_CRIT — fallback crit threshold in °C (if dynamix.cfg not found)
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
# smart_health.sh — normal run
# smart_health.sh --dry-run — show which drives would be checked
# smart_health.sh --log — verbose output
# smart_health.sh --status — show config and exit
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../Master.conf"
source "$SCRIPT_DIR/../common.sh"
source "$SCRIPT_DIR/../load_config.sh"
# Monitor script — output is the point
SILENT_MODE=false
parse_args "$@"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Setup ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_GEAR Setup ━━━"
@@ -38,35 +70,56 @@ if [[ "$EUID" -ne 0 ]]; then
exit 1
fi
success "Running as root"
# Validate smartctl — required for all drive checks
validate_unraid_cmd \
"$(command -v smartctl 2>/dev/null || echo /usr/bin/smartctl)" \
"--version" "smartmontools" \
"smartctl" || {
error "smartctl not found — install smartmontools"
notify "SMART health check failed on $(hostname) — smartmontools not installed" \
"SMART Health" "warning"
exit 1
}
if ! command -v smartctl >/dev/null 2>&1; then
error "smartctl not found — install smartmontools"
notify "SMART health check failed on $(hostname) — smartmontools not installed" "SMART Health" "warning"
exit 1
fi
validate_unraid_cmd \
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
"" "" \
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
success "smartctl available"
acquire_lock
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Status ━━━
# -----------------------------------------------------------------------------------------------
# detect_hosts() sets MY_ID and aliases HOST*_SMART_IGNORE_DRIVES
detect_hosts
# Load temperature thresholds from dynamix.cfg — unRAID's own settings
get_unraid_temp_thresholds
log "HDD warn: ${UNRAID_DISK_HOT}°C crit: ${UNRAID_DISK_MAX}°C"
log "SSD warn: ${UNRAID_SSD_HOT}°C crit: ${UNRAID_SSD_MAX}°C"
log "Ignore: ${SMART_IGNORE_DRIVES[*]:-none}"
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — showing drive list only, no SMART data read"
# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
echo ""
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
echo "$ICON_SMART Temp warn: ${SMART_TEMP_WARN}°C"
echo "$ICON_SMART Temp crit: ${SMART_TEMP_CRIT}°C"
echo "$ICON_SMART Ignore drives: ${SMART_IGNORE_DRIVES[*]:-none}"
echo "$ICON_GEAR Dry Run: $DRY_RUN"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_SMART HDD warn: ${UNRAID_DISK_HOT}°C"
echo "$ICON_SMART HDD crit: ${UNRAID_DISK_MAX}°C"
echo "$ICON_SMART SSD warn: ${UNRAID_SSD_HOT}°C"
echo "$ICON_SMART SSD crit: ${UNRAID_SSD_MAX}°C"
echo "$ICON_SMART Ignore drives: ${SMART_IGNORE_DRIVES[*]:-none}"
echo "$ICON_GEAR Dry Run: $DRY_RUN"
echo ""
# Show drives that would be checked
echo "━━━ Discovered Drives ━━━"
for drive in /dev/sd? /dev/nvme?; do
[[ ! -e "$drive" ]] && continue
drive_name=$(basename "$drive")
ignored=false
for ignore in "${SMART_IGNORE_DRIVES[@]}"; do
for ignore in "${SMART_IGNORE_DRIVES[@]:-}"; do
[[ "$drive_name" == "$ignore" ]] && ignored=true && break
done
if [[ "$ignored" == true ]]; then
@@ -79,23 +132,55 @@ if [[ "$SHOW_STATUS" == true ]]; then
exit 0
fi
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — showing drive list only, no SMART data read"
# ==============================================================================================
# ── HELPER FUNCTIONS ──────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# -----------------------------------------------------------------------------------------------
# HELPER — extract SMART attribute value
# Usage: get_smart_attr "/dev/sda" "Reallocated_Sector_Ct"
# -----------------------------------------------------------------------------------------------
# Extract a named SMART attribute value (column 10 — raw value)
get_smart_attr() {
local drive="$1" attr="$2"
smartctl -A "$drive" 2>/dev/null | \
awk -v attr="$attr" '$2 == attr {print $10}'
}
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SMART SMART Health Check ━━━
# -----------------------------------------------------------------------------------------------
# Get drive temperature — handles HDD (attribute) and NVMe (different output format)
get_drive_temp() {
local drive="$1"
local temp
# Standard HDD SMART attribute
temp=$(get_smart_attr "$drive" "Temperature_Celsius")
[[ -n "$temp" ]] && echo "$temp" && return
# NVMe — temperature in different section
temp=$(smartctl -A "$drive" 2>/dev/null | \
awk '/Temperature:/{gsub(/[^0-9]/,"",$2); if($2>0) print $2; exit}')
[[ -n "$temp" ]] && echo "$temp" && return
# Fallback — any temperature line
temp=$(smartctl -A "$drive" 2>/dev/null | \
awk '/Temp/{gsub(/[^0-9]/,"",$NF); if($NF>0 && $NF<120) print $NF; exit}')
echo "${temp:-}"
}
# Detect if a drive is SSD/NVMe (rotational=0)
is_ssd() {
local drive="$1"
local dev_name
dev_name=$(basename "$drive" | sed 's/nvme[0-9]/nvme0/')
local rotational="/sys/block/$(basename "$drive")/queue/rotational"
[[ -f "$rotational" ]] && [[ "$(cat "$rotational" 2>/dev/null)" == "0" ]] && return 0
# NVMe is always SSD
[[ "$drive" == *nvme* ]] && return 0
return 1
}
# ==============================================================================================
# ━━━ SMART Health Check ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_SMART SMART Health Check — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME)"
echo ""
START=$(date +%s)
@@ -110,12 +195,12 @@ for drive in /dev/sd? /dev/nvme?; do
# Check ignore list
ignored=false
for ignore in "${SMART_IGNORE_DRIVES[@]}"; do
for ignore in "${SMART_IGNORE_DRIVES[@]:-}"; do
[[ "$drive_name" == "$ignore" ]] && ignored=true && break
done
if [[ "$ignored" == true ]]; then
info "$drive_name — ignored (in SMART_IGNORE_DRIVES)"
log "$drive_name — ignored (SMART_IGNORE_DRIVES)"
DRIVES_SKIP+=("$drive_name")
continue
fi
@@ -128,34 +213,40 @@ for drive in /dev/sd? /dev/nvme?; do
continue
fi
# Check if drive supports SMART
# Check SMART support
if ! smartctl -i "$drive" 2>/dev/null | grep -q "SMART support is: Enabled"; then
warn "$drive_name — SMART not enabled or not supported"
warn "$drive_name — SMART not enabled or not supported — skipping"
DRIVES_SKIP+=("$drive_name")
echo ""
continue
fi
# Overall SMART status
SMART_STATUS=$(smartctl -H "$drive" 2>/dev/null | grep "overall-health" | awk '{print $NF}')
if [[ "$SMART_STATUS" == "PASSED" ]]; then
success "Overall status: PASSED"
else
error "Overall status: $SMART_STATUS"
fi
# Key attributes
DRIVE_WARN=false
DRIVE_CRIT=false
# Overall SMART status
SMART_STATUS=$(smartctl -H "$drive" 2>/dev/null | \
grep "overall-health" | awk '{print $NF}')
case "${SMART_STATUS:-}" in
PASSED)
log "$drive_name overall status: PASSED" ;;
FAILED*)
error "$drive_name overall status: FAILED — drive may be failing"
DRIVE_CRIT=true ;;
"")
warn "$drive_name overall status: unknown — could not read SMART data" ;;
*)
warn "$drive_name overall status: $SMART_STATUS" ;;
esac
# Reallocated sectors
REALLOC=$(get_smart_attr "$drive" "Reallocated_Sector_Ct")
if [[ -n "$REALLOC" ]]; then
if [[ "$REALLOC" -gt 0 ]]; then
warn "$ICON_SMART Reallocated sectors: $REALLOC — drive showing wear"
warn "$ICON_SMART $drive_name — Reallocated sectors: $REALLOC (drive showing wear)"
DRIVE_WARN=true
else
success "$ICON_SMART Reallocated sectors: $REALLOC"
log "$drive_name reallocated sectors: 0 ✅"
fi
fi
@@ -163,47 +254,53 @@ for drive in /dev/sd? /dev/nvme?; do
PENDING=$(get_smart_attr "$drive" "Current_Pending_Sector")
if [[ -n "$PENDING" ]]; then
if [[ "$PENDING" -gt 0 ]]; then
warn "$ICON_SMART Pending sectors: $PENDING — sectors awaiting reallocation"
warn "$ICON_SMART $drive_name — Pending sectors: $PENDING (awaiting reallocation)"
DRIVE_WARN=true
else
success "$ICON_SMART Pending sectors: $PENDING"
log "$drive_name pending sectors: 0 ✅"
fi
fi
# Uncorrectable sectors
# Uncorrectable sectors — critical threshold
UNCORR=$(get_smart_attr "$drive" "Offline_Uncorrectable")
if [[ -n "$UNCORR" ]]; then
if [[ "$UNCORR" -gt 0 ]]; then
error "$ICON_SMART Uncorrectable sectors: $UNCORR — CRITICAL"
error "$ICON_SMART $drive_name — Uncorrectable sectors: $UNCORR — CRITICAL"
DRIVE_CRIT=true
else
success "$ICON_SMART Uncorrectable sectors: $UNCORR"
log "$drive_name uncorrectable sectors: 0 ✅"
fi
fi
# Temperature
TEMP=$(get_smart_attr "$drive" "Temperature_Celsius")
# NVMe uses different attribute name
[[ -z "$TEMP" ]] && TEMP=$(smartctl -A "$drive" 2>/dev/null | \
awk '/Temperature/{print $2}' | head -1)
# Temperature — use SSD/HDD thresholds from dynamix.cfg
TEMP=$(get_drive_temp "$drive")
if [[ -n "$TEMP" ]] && [[ "$TEMP" =~ ^[0-9]+$ ]]; then
if is_ssd "$drive"; then
WARN_THRESH="$UNRAID_SSD_HOT"
CRIT_THRESH="$UNRAID_SSD_MAX"
DRIVE_TYPE="SSD"
else
WARN_THRESH="$UNRAID_DISK_HOT"
CRIT_THRESH="$UNRAID_DISK_MAX"
DRIVE_TYPE="HDD"
fi
if [[ -n "$TEMP" ]]; then
if [[ "$TEMP" -ge "$SMART_TEMP_CRIT" ]]; then
error "$ICON_SMART Temperature: ${TEMP}°C — CRITICAL (threshold: ${SMART_TEMP_CRIT}°C)"
if [[ "$TEMP" -ge "$CRIT_THRESH" ]]; then
error "$ICON_SMART $drive_name — ${DRIVE_TYPE} temp: ${TEMP}°C — CRITICAL (threshold: ${CRIT_THRESH}°C)"
DRIVE_CRIT=true
elif [[ "$TEMP" -ge "$SMART_TEMP_WARN" ]]; then
warn "$ICON_SMART Temperature: ${TEMP}°C — warning (threshold: ${SMART_TEMP_WARN}°C)"
elif [[ "$TEMP" -ge "$WARN_THRESH" ]]; then
warn "$ICON_SMART $drive_name — ${DRIVE_TYPE} temp: ${TEMP}°C — warning (threshold: ${WARN_THRESH}°C)"
DRIVE_WARN=true
else
success "$ICON_SMART Temperature: ${TEMP}°C"
log "$drive_name temp: ${TEMP}°C ${DRIVE_TYPE} ✅"
fi
fi
# Power on hours — informational
# Power on hours — informational only
POH=$(get_smart_attr "$drive" "Power_On_Hours")
if [[ -n "$POH" ]]; then
POH_DAYS=$(( POH / 24 ))
info "$ICON_SMART Power on hours: $POH (${POH_DAYS} days)"
log "$drive_name power on hours: $POH (${POH_DAYS} days)"
fi
# Classify drive
@@ -220,30 +317,33 @@ done
END=$(date +%s)
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Summary ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
echo "━━━━━ $ICON_SUMMARY SMART HEALTH SUMMARY ━━━━━"
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
echo ""
echo " $ICON_SUCCESS Healthy: ${#DRIVES_OK[@]} $ICON_WARN Warning: ${#DRIVES_WARN[@]} $ICON_ERROR Critical: ${#DRIVES_CRIT[@]} skipped: ${#DRIVES_SKIP[@]}"
echo " $ICON_SUCCESS Healthy: ${#DRIVES_OK[@]}"
[[ ${#DRIVES_WARN[@]} -gt 0 ]] && warn "Warning: ${#DRIVES_WARN[@]} — ${DRIVES_WARN[*]}"
[[ ${#DRIVES_CRIT[@]} -gt 0 ]] && echo "$ICON_ERROR Critical: ${#DRIVES_CRIT[@]} — ${DRIVES_CRIT[*]}"
[[ ${#DRIVES_SKIP[@]} -gt 0 ]] && log "Skipped: ${#DRIVES_SKIP[@]} — ${DRIVES_SKIP[*]}"
echo ""
[[ ${#DRIVES_OK[@]} -gt 0 ]] && echo " $ICON_SUCCESS ${DRIVES_OK[*]}"
[[ ${#DRIVES_WARN[@]} -gt 0 ]] && echo " $ICON_WARN ${DRIVES_WARN[*]}"
[[ ${#DRIVES_CRIT[@]} -gt 0 ]] && echo " $ICON_ERROR ${DRIVES_CRIT[*]}"
echo ""
if [[ "$DRY_RUN" == true ]]; then
echo "$ICON_WARN Status: DRY RUN"
warn "DRY RUN — no SMART data read"
elif [[ ${#DRIVES_CRIT[@]} -gt 0 ]]; then
echo "$ICON_ERROR Status: CRITICAL — ${DRIVES_CRIT[*]}"
notify "SMART CRITICAL on $(hostname) — drives need immediate attention: ${DRIVES_CRIT[*]}" "SMART Health" "warning"
notify "SMART CRITICAL on $(hostname) — immediate attention needed: ${DRIVES_CRIT[*]}" \
"SMART Health" "warning"
elif [[ ${#DRIVES_WARN[@]} -gt 0 ]]; then
echo "$ICON_WARN Status: WARNING — ${DRIVES_WARN[*]}"
notify "SMART WARNING on $(hostname) — drives showing wear: ${DRIVES_WARN[*]}" "SMART Health" "warning"
warn "Status: WARNING — ${DRIVES_WARN[*]}"
notify "SMART WARNING on $(hostname) — drives showing wear: ${DRIVES_WARN[*]}" \
"SMART Health" "warning"
else
echo "$ICON_DONE Status: $ICON_SUCCESS ALL DRIVES HEALTHY"
notify "SMART health check passed on $(hostname) — ${#DRIVES_OK[@]} drives healthy" "SMART Health" "normal"
log "$ICON_DONE Status: all ${#DRIVES_OK[@]} drives healthy ✅"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
[[ ${#DRIVES_CRIT[@]} -gt 0 ]] && exit 1
exit 0
Monitors/system_tuning_monitor.sh
+205
View File
@@ -0,0 +1,205 @@
#!/bin/bash
# ==============================================================================================
# ============================= System Tuning Monitor ==========================================
# ==============================================================================================
# Tracks inotify and php-fpm usage over time.
# Snapshots written every 6 hours — read by sunday_morning_coffee_report.sh for weekly summary.
# Schedule: 0 */6 * * * (every 6 hours via User Scripts)
#
# ── WHAT IT TRACKS ────────────────────────────────────────────────────────────────────────────
# inotify instances:
# Current in use vs kernel limit
# % utilization — warns above INOTIFY_WARN_PCT (default 80%)
# Top 5 consumers by instance count
# Symptom of exhaustion: containers miss file events, downloads not detected,
# Live TV stutter, library not updated
#
# php-fpm workers:
# Active workers vs PHP_MAX_CHILDREN limit
# % utilization — warns above PHP_FPM_WARN_PCT (default 80%)
# Symptom: unRAID WebGUI slowdowns or timeouts under load
#
# ── LOG FORMAT ────────────────────────────────────────────────────────────────────────────────
# DATE|TIME|INOTIFY_USED|INOTIFY_LIMIT|INOTIFY_PCT|INOTIFY_WARN|PHPFPM_ACTIVE|PHPFPM_MAX|PHPFPM_PCT|PHPFPM_WARN
# Log trimmed to TUNING_LOG_RETENTION days on each write — bounded size.
#
# ── WHAT THE WEEKLY REPORT SHOWS ──────────────────────────────────────────────────────────────
# inotify: peak, average, warning count over the week
# php-fpm: peak workers, average workers, warning count over the week
#
# ── SILENT BY DEFAULT ─────────────────────────────────────────────────────────────────────────
# Background snapshot script — no output when healthy.
# Warns to stderr when thresholds exceeded — visible in User Scripts output log.
# Does NOT notify on every snapshot — only when threshold exceeded.
#
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
# Each server writes to its own DATA_DIR — no collision between servers.
# MY_ID included in warning output for clarity in shared notification channels.
#
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
# acquire_lock — prevents overlapping 6-hour snapshots
# root check — /proc/*/fd requires root access
# atomic log write — tmp file + mv prevents partial writes on trim
# validate_unraid — notify script validated before use
#
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
# INOTIFY_WARN_PCT — warn threshold % (default 80)
# PHP_FPM_WARN_PCT — warn threshold % (default 80)
# PHP_MAX_CHILDREN — max php-fpm workers (set by php_fpm_max_children.sh)
# TUNING_MONITOR_LOG — log file path
# TUNING_LOG_RETENTION — days before old entries purged (default 30)
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
# system_tuning_monitor.sh — normal snapshot run
# system_tuning_monitor.sh --dry-run — measure and show, no log write
# system_tuning_monitor.sh --log — verbose output
# system_tuning_monitor.sh --status — show config and exit
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../load_config.sh"
parse_args "$@"
# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
error "Must be run as root — /proc/*/fd requires root access"
exit 1
fi
validate_unraid_cmd \
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
"" "" \
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
acquire_lock
# detect_hosts() sets MY_ID — used in warning output
detect_hosts
DATE=$(date '+%Y-%m-%d')
TIME=$(date '+%H:%M')
INOTIFY_WARN=0
PHPFPM_WARN=0
# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
echo ""
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_GEAR inotify warn: ${INOTIFY_WARN_PCT:-80}%"
echo "$ICON_GEAR php-fpm warn: ${PHP_FPM_WARN_PCT:-80}%"
echo "$ICON_GEAR php-fpm max: ${PHP_MAX_CHILDREN:-250}"
echo "$ICON_GEAR Log file: ${TUNING_MONITOR_LOG:-not set}"
echo "$ICON_GEAR Retention: ${TUNING_LOG_RETENTION:-30} days"
echo "$ICON_GEAR Dry Run: $DRY_RUN"
if [[ -f "$TUNING_MONITOR_LOG" ]]; then
ENTRY_COUNT=$(wc -l < "$TUNING_MONITOR_LOG")
OLDEST=$(awk -F'|' 'NR==1{print $1}' "$TUNING_MONITOR_LOG")
NEWEST=$(awk -F'|' 'END{print $1}' "$TUNING_MONITOR_LOG")
echo "$ICON_MONITOR Log entries: $ENTRY_COUNT ($OLDEST → $NEWEST)"
else
echo "$ICON_MONITOR Log entries: none yet"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
fi
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — measuring only, no log write"
# ==============================================================================================
# ━━━ inotify ━━━
# ==============================================================================================
INOTIFY_LIMIT=$(sysctl -n fs.inotify.max_user_instances 2>/dev/null || echo 0)
INOTIFY_USED=$(find /proc/*/fd -lname 'anon_inode:inotify' 2>/dev/null | wc -l)
INOTIFY_USED="${INOTIFY_USED//[^0-9]/}"
INOTIFY_USED="${INOTIFY_USED:-0}"
if [[ "$INOTIFY_LIMIT" -gt 0 ]]; then
INOTIFY_PCT=$(( INOTIFY_USED * 100 / INOTIFY_LIMIT ))
else
INOTIFY_PCT=0
fi
[[ "$INOTIFY_PCT" -ge "${INOTIFY_WARN_PCT:-80}" ]] && INOTIFY_WARN=1
# Top 5 inotify consumers
INOTIFY_TOP=$(find /proc/*/fd -lname 'anon_inode:inotify' 2>/dev/null | \
awk -F/ '{print $3}' | sort | uniq -c | sort -rn | head -5 | \
while read -r count pid; do
comm=$(cat "/proc/$pid/comm" 2>/dev/null || echo "?")
echo "${count}×${comm}"
done | tr '\n' ',' | sed 's/,$//')
if [[ "$INOTIFY_WARN" -eq 1 ]]; then
warn "$MY_ID — inotify: ${INOTIFY_USED}/${INOTIFY_LIMIT} (${INOTIFY_PCT}%) — above ${INOTIFY_WARN_PCT}% threshold"
warn "Top consumers: ${INOTIFY_TOP:-unknown}"
warn "Symptoms: containers missing file events, library not updating, Live TV stutter"
notify "inotify at ${INOTIFY_PCT}% on $(hostname) — ${INOTIFY_USED}/${INOTIFY_LIMIT} in use — top: ${INOTIFY_TOP}" \
"System Tuning" "warning"
else
log "inotify: ${INOTIFY_USED}/${INOTIFY_LIMIT} (${INOTIFY_PCT}%) ✅"
log "inotify top consumers: ${INOTIFY_TOP:-none}"
fi
# ==============================================================================================
# ━━━ php-fpm ━━━
# ==============================================================================================
PHPFPM_MAX="${PHP_MAX_CHILDREN:-250}"
PHPFPM_ACTIVE=$(ps aux 2>/dev/null | grep -c "php-fpm: pool" || echo 0)
PHPFPM_ACTIVE="${PHPFPM_ACTIVE//[^0-9]/}"
PHPFPM_ACTIVE="${PHPFPM_ACTIVE:-0}"
if [[ "$PHPFPM_MAX" -gt 0 ]]; then
PHPFPM_PCT=$(( PHPFPM_ACTIVE * 100 / PHPFPM_MAX ))
else
PHPFPM_PCT=0
fi
[[ "$PHPFPM_PCT" -ge "${PHP_FPM_WARN_PCT:-80}" ]] && PHPFPM_WARN=1
if [[ "$PHPFPM_WARN" -eq 1 ]]; then
warn "$MY_ID — php-fpm: ${PHPFPM_ACTIVE}/${PHPFPM_MAX} workers (${PHPFPM_PCT}%) — above ${PHP_FPM_WARN_PCT}% threshold"
warn "Symptom: unRAID WebGUI slowdowns or timeouts under load"
notify "php-fpm at ${PHPFPM_PCT}% on $(hostname) — ${PHPFPM_ACTIVE}/${PHPFPM_MAX} workers active" \
"System Tuning" "warning"
else
log "php-fpm: ${PHPFPM_ACTIVE}/${PHPFPM_MAX} active workers (${PHPFPM_PCT}%) ✅"
fi
# ==============================================================================================
# ━━━ Write Snapshot ━━━
# ==============================================================================================
if [[ -z "${TUNING_MONITOR_LOG:-}" ]]; then
warn "TUNING_MONITOR_LOG not set — snapshot not written"
exit 0
fi
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — snapshot not written"
warn "Would write: ${DATE}|${TIME}|${INOTIFY_USED}|${INOTIFY_LIMIT}|${INOTIFY_PCT}|${INOTIFY_WARN}|${PHPFPM_ACTIVE}|${PHPFPM_MAX}|${PHPFPM_PCT}|${PHPFPM_WARN}"
exit 0
fi
mkdir -p "$(dirname "$TUNING_MONITOR_LOG")"
# Trim old entries — atomic write via temp file
if [[ -f "$TUNING_MONITOR_LOG" ]]; then
CUTOFF=$(date -d "${TUNING_LOG_RETENTION:-30} days ago" '+%Y-%m-%d')
awk -F'|' -v cutoff="$CUTOFF" '$1 >= cutoff' \
"$TUNING_MONITOR_LOG" > "${TUNING_MONITOR_LOG}.tmp" && \
mv "${TUNING_MONITOR_LOG}.tmp" "$TUNING_MONITOR_LOG"
fi
# Append snapshot
echo "${DATE}|${TIME}|${INOTIFY_USED}|${INOTIFY_LIMIT}|${INOTIFY_PCT}|${INOTIFY_WARN}|${PHPFPM_ACTIVE}|${PHPFPM_MAX}|${PHPFPM_PCT}|${PHPFPM_WARN}" \
>> "$TUNING_MONITOR_LOG"
log "Snapshot written: inotify ${INOTIFY_PCT}% php-fpm ${PHPFPM_PCT}%"
Monitors/weekly_health_digest.sh
+205 -161
View File
@@ -1,50 +1,117 @@
#!/bin/bash
# -----------------------------------------------------------------------------------------------
# --------------------------------- Health Digest ----------------------------------------------
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ================================= Health Digest ==============================================
# ==============================================================================================
# Aggregates system health data from across the ecosystem into a single digest report.
# Reads existing state files — no new writes to flash drive.
# Reads existing state files — no new writes.
#
# ── THREE PROFILES ────────────────────────────────────────────────────────────────────────────
# always — sends every run regardless of findings
# schedule daily for a daily digest
#
# smart — sends only if something worth reporting was found
# runs every run but stays silent when all healthy
# DIGEST_SMART_ON_* toggles control what triggers a send
#
# Three profiles controlled by DIGEST_PROFILE in Master.conf:
# always — sends every run regardless of findings (schedule daily for daily digest)
# smart — sends only if something worth reporting was found (intelligent filtering)
# weekly — sends once per week on DIGEST_DAY regardless of schedule frequency
# run daily, digest only fires on DIGEST_DAY (default Sunday)
#
# The cron schedule stays the same regardless of profile — just change DIGEST_PROFILE
# in Master.conf to switch behavior. Run daily, profile controls when it actually notifies.
# The cron schedule stays the same regardless of profile — change DIGEST_PROFILE in
# master.conf to switch behaviour. No cron changes needed.
#
# Data sources (reads only — no writes):
# /tmp/transcode_state.db — ramdisk symlink and usage
# /tmp/container_watchdog_state.db — active container strikes
# /tmp/system_watchdog_state.db — active system strikes
# /boot/config/failover_state.db — current failover state
# /boot/config/system_watchdog_failed.db — container skip list
# /boot/config/bandwidth_history.db — recent transfer totals
# SSL certs via openssl (live check) — days remaining per domain
# ── DATA SOURCES (reads only) ─────────────────────────────────────────────────────────────────
# FAILOVER_STATE_FILE — current failover state
# SYS_WATCHDOG_FAILED_FILE — container skip list (manual intervention needed)
# WATCHDOG_STATE_FILE — active container watchdog strikes
# SYS_WATCHDOG_STATE_FILE — active system watchdog strikes
# BANDWIDTH_LOG — yesterday's transfer totals
# TRANSCODE_DAILY_LOG — weekly transcode statistics
# CERT_MONITOR_DOMAINS — live SSL cert check via openssl
# RAMDISK_PATH / TRANSCODE_LINK — current transcode location and usage
#
# All configuration in Master.conf under Health Digest section.
# Supports --dry-run to generate report without sending notification.
# -----------------------------------------------------------------------------------------------
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
# detect_hosts() sets MY_ID and aliases CERT_MONITOR_DOMAINS, RAMDISK_WARN_GB,
# RAMDISK_SIZE, RAMDISK_LOW_GB and all other host-specific vars used in this report.
#
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
# acquire_lock — report takes time, prevent duplicate runs
# detect_hosts() — correct vars per host
# validate_unraid_cmd — notify and openssl validated before use
# Per-section guards — missing state file skipped cleanly
# Silent smart profile — completely silent when nothing to report
#
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
# DIGEST_PROFILE — always | smart | weekly
# DIGEST_DAY — day name for weekly profile (e.g. Sunday)
# DIGEST_SMART_ON_WATCHDOG — send on active watchdog strikes
# DIGEST_SMART_ON_FAILOVER — send on non-NORMAL failover state
# DIGEST_SMART_ON_CERT_WARN — send on cert warning
# DIGEST_SMART_ON_BANDWIDTH — send on high bandwidth day
# CERT_WARN_DAYS / CERT_CRIT_DAYS / CERT_TIMEOUT
# BANDWIDTH_WARN_GB
# TRANSCODE_DAILY_LOG
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
# weekly_health_digest.sh — normal run
# weekly_health_digest.sh --dry-run — generate report, no notification
# weekly_health_digest.sh --log — verbose output
# weekly_health_digest.sh --status — show config and exit
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../Master.conf"
source "$SCRIPT_DIR/../common.sh"
source "$SCRIPT_DIR/../load_config.sh"
# Report/monitor script — output is the point when sending
SILENT_MODE=false
parse_args "$@"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Setup ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_GEAR Setup ━━━"
# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
error "Must be run as root"
exit 1
fi
validate_unraid_cmd \
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
"" "" \
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
validate_unraid_cmd \
"$(command -v openssl 2>/dev/null || echo /usr/bin/openssl)" \
"version" "OpenSSL" \
"openssl" || warn "openssl not found — SSL cert checks will be skipped"
acquire_lock
# detect_hosts() sets MY_ID and aliases all host-specific vars used in this report
detect_hosts
success "Health Digest — profile: $DIGEST_PROFILE"
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — report generated but no notification sent"
# -----------------------------------------------------------------------------------------------
# Profile check — should we send today?
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
echo ""
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_DIGEST Profile: $DIGEST_PROFILE"
echo "$ICON_DIGEST Digest day: $DIGEST_DAY"
echo "$ICON_DIGEST Smart triggers: watchdog=$DIGEST_SMART_ON_WATCHDOG failover=$DIGEST_SMART_ON_FAILOVER cert=$DIGEST_SMART_ON_CERT_WARN bandwidth=$DIGEST_SMART_ON_BANDWIDTH"
echo "$ICON_CERT Cert domains: ${CERT_MONITOR_DOMAINS[*]:-none}"
echo "$ICON_BANDWIDTH Bandwidth warn: ${BANDWIDTH_WARN_GB}GB"
echo "$ICON_GEAR Dry Run: $DRY_RUN"
echo "━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
fi
# ==============================================================================================
# ── Profile gate — should we send today? ──────────────────────────────────────────────────────
# ==============================================================================================
SHOULD_SEND=false
case "$DIGEST_PROFILE" in
@@ -56,229 +123,206 @@ case "$DIGEST_PROFILE" in
TODAY_NAME=$(date '+%A')
if [[ "$TODAY_NAME" == "$DIGEST_DAY" ]]; then
SHOULD_SEND=true
log "Profile: weekly — today is $DIGEST_DAY, will send"
log "Profile: weekly — today is $DIGEST_DAY — will send"
else
info "Profile: weekly — today is $TODAY_NAME, digest day is $DIGEST_DAY — skipping"
log "Profile: weekly — today is $TODAY_NAME, digest day is $DIGEST_DAY — skipping"
exit 0
fi
;;
smart)
log "Profile: smart — will evaluate findings before deciding"
SHOULD_SEND=false # determined after gathering data
log "Profile: smart — evaluating findings before deciding"
SHOULD_SEND=false
;;
*)
warn "Unknown DIGEST_PROFILE: $DIGEST_PROFILE — defaulting to weekly behavior"
warn "Unknown DIGEST_PROFILE: $DIGEST_PROFILE — defaulting to weekly"
TODAY_NAME=$(date '+%A')
[[ "$TODAY_NAME" == "$DIGEST_DAY" ]] && SHOULD_SEND=true
;;
esac
# -----------------------------------------------------------------------------------------------
# DATA GATHERING
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_DIGEST Gathering System Data ━━━"
# ==============================================================================================
# ── Data Gathering ────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
FINDINGS=() # notable but not critical
ISSUES=() # need attention
DIGEST_LINES=() # full report lines
FINDINGS=() # things worth noting
ISSUES=() # things that need attention
DIGEST_LINES=() # full report lines
# ── Failover State ──────────────────────────────────────────────────────────────────────────
# ── Failover State ────────────────────────────────────────────────────────────────────────────
if [[ -f "$FAILOVER_STATE_FILE" ]]; then
FAILOVER_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
FAILOVER_CHANGE=$(grep "^last_change=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
DIGEST_LINES+=("$ICON_FAILOVER Failover: $FAILOVER_STATE (last change: ${FAILOVER_CHANGE:-unknown})")
if [[ "$FAILOVER_STATE" != "NORMAL" && -n "$FAILOVER_STATE" ]]; then
ISSUES+=("Failover state: $FAILOVER_STATE")
[[ "$DIGEST_SMART_ON_FAILOVER" == true ]] && SHOULD_SEND=true
if [[ -n "$FAILOVER_STATE" ]]; then
DIGEST_LINES+=("$ICON_FAILOVER Failover: $FAILOVER_STATE")
if [[ "$FAILOVER_STATE" != "NORMAL" ]]; then
ISSUES+=("Failover state: $FAILOVER_STATE")
[[ "$DIGEST_SMART_ON_FAILOVER" == true ]] && SHOULD_SEND=true
fi
fi
else
DIGEST_LINES+=("$ICON_FAILOVER Failover: state file not found")
fi
# ── Transcode Ramdisk ───────────────────────────────────────────────────────────────────────
if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
RAMDISK_USED_KB=$(df "$RAMDISK_PATH" --output=used | tail -1 | tr -d ' ')
RAMDISK_USED_GB=$(awk "BEGIN {printf \"%.2f\", $RAMDISK_USED_KB / 1048576}")
RAMDISK_AVAIL_KB=$(df "$RAMDISK_PATH" --output=avail | tail -1 | tr -d ' ')
RAMDISK_AVAIL_GB=$(awk "BEGIN {printf \"%.2f\", $RAMDISK_AVAIL_KB / 1048576}")
SYMLINK_TARGET=$(readlink "$TRANSCODE_LINK" 2>/dev/null || echo "unknown")
DIGEST_LINES+=("$ICON_RAM Transcodes: ${RAMDISK_USED_GB}GB used / ${RAMDISK_AVAIL_GB}GB free → $SYMLINK_TARGET")
# Read weekly transcode stats from daily log if available
TRANSCODE_DAILY_LOG="/boot/config/transcode_daily.db"
if [[ -f "$TRANSCODE_DAILY_LOG" ]]; then
WEEK_CUTOFF=$(date -d "7 days ago" '+%Y-%m-%d')
# Peak ramdisk usage this week
WEEK_PEAK=$(awk -F'|' -v cutoff="$WEEK_CUTOFF" \
'$1 >= cutoff {if ($2 > max) max=$2} END {printf "%.2f", max+0}' \
"$TRANSCODE_DAILY_LOG")
# Total flips this week
WEEK_FLIPS=$(awk -F'|' -v cutoff="$WEEK_CUTOFF" \
'$1 >= cutoff {sum+=$3} END {print sum+0}' \
"$TRANSCODE_DAILY_LOG")
# Total files cleaned this week
WEEK_FILES=$(awk -F'|' -v cutoff="$WEEK_CUTOFF" \
'$1 >= cutoff {sum+=$6} END {print sum+0}' \
"$TRANSCODE_DAILY_LOG")
# Ram vs SSD session ratio
WEEK_RAM=$(awk -F'|' -v cutoff="$WEEK_CUTOFF" \
'$1 >= cutoff {sum+=$4} END {print sum+0}' \
"$TRANSCODE_DAILY_LOG")
WEEK_SSD=$(awk -F'|' -v cutoff="$WEEK_CUTOFF" \
'$1 >= cutoff {sum+=$5} END {print sum+0}' \
"$TRANSCODE_DAILY_LOG")
DIGEST_LINES+=("$ICON_RAM Transcode week: peak ${WEEK_PEAK}GB | flips: ${WEEK_FLIPS} | cleaned: ${WEEK_FILES} files")
DIGEST_LINES+=("$ICON_RAM Session storage: ${WEEK_RAM} ramdisk / ${WEEK_SSD} SSD")
# Warn if peak is getting close to threshold
PEAK_INT=$(printf "%.0f" "$WEEK_PEAK")
WARN_INT=$(printf "%.0f" "$RAMDISK_WARN_GB")
if [[ "$PEAK_INT" -ge "$WARN_INT" ]]; then
DIGEST_LINES+=("$ICON_WARN Transcode peak ${WEEK_PEAK}GB near threshold ${RAMDISK_WARN_GB}GB — consider increasing RAMDISK_SIZE")
fi
fi
# ── Container Skip List ───────────────────────────────────────────────────────────────────────
if [[ -f "$SYS_WATCHDOG_FAILED_FILE" ]] && [[ -s "$SYS_WATCHDOG_FAILED_FILE" ]]; then
SKIP_COUNT=$(wc -l < "$SYS_WATCHDOG_FAILED_FILE")
SKIP_LIST=$(cat "$SYS_WATCHDOG_FAILED_FILE" | tr '\n' ' ')
DIGEST_LINES+=("$ICON_NOT_RUNNING Skip list: $SKIP_COUNT containers — $SKIP_LIST")
ISSUES+=("Containers on skip list (manual intervention needed): $SKIP_LIST")
SHOULD_SEND=true
else
DIGEST_LINES+=("$ICON_RAM Transcodes: ramdisk not mounted")
ISSUES+=("Ramdisk not mounted")
DIGEST_LINES+=("$ICON_RUNNING Skip list: empty ✅")
fi
# ── Container Watchdog Strikes ──────────────────────────────────────────────────────────────
# ── Container Watchdog Strikes ────────────────────────────────────────────────────────────────
if [[ -f "$WATCHDOG_STATE_FILE" ]]; then
ACTIVE_STRIKES=$(grep -v ":0$" "$WATCHDOG_STATE_FILE" 2>/dev/null | wc -l)
ACTIVE_STRIKES=$(grep -v ":0$" "$WATCHDOG_STATE_FILE" 2>/dev/null | grep -c "." || echo 0)
if [[ "$ACTIVE_STRIKES" -gt 0 ]]; then
STRIKE_LIST=$(grep -v ":0$" "$WATCHDOG_STATE_FILE" 2>/dev/null | tr '\n' ' ')
DIGEST_LINES+=("$ICON_WATCHDOG Container strikes: $ACTIVE_STRIKES active — $STRIKE_LIST")
FINDINGS+=("Container watchdog: $ACTIVE_STRIKES active strikes")
[[ "$DIGEST_SMART_ON_WATCHDOG" == true ]] && SHOULD_SEND=true
else
DIGEST_LINES+=("$ICON_WATCHDOG Container watchdog: no active strikes")
DIGEST_LINES+=("$ICON_WATCHDOG Container watchdog: no active strikes ✅")
fi
fi
# ── System Watchdog Strikes ─────────────────────────────────────────────────────────────────
# ── System Watchdog Strikes ───────────────────────────────────────────────────────────────────
if [[ -f "$SYS_WATCHDOG_STATE_FILE" ]]; then
SYS_STRIKES=$(grep -v ":0$" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | wc -l)
SYS_STRIKES=$(grep -v ":0$" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | grep -c "." || echo 0)
if [[ "$SYS_STRIKES" -gt 0 ]]; then
SYS_STRIKE_LIST=$(grep -v ":0$" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | tr '\n' ' ')
DIGEST_LINES+=("$ICON_REBOOT_SMART System strikes: $SYS_STRIKES active — $SYS_STRIKE_LIST")
FINDINGS+=("System watchdog: $SYS_STRIKES active strikes")
[[ "$DIGEST_SMART_ON_WATCHDOG" == true ]] && SHOULD_SEND=true
else
DIGEST_LINES+=("$ICON_REBOOT_SMART System watchdog: no active strikes")
DIGEST_LINES+=("$ICON_REBOOT_SMART System watchdog: no active strikes ✅")
fi
fi
# ── Container Skip List ─────────────────────────────────────────────────────────────────────
if [[ -f "$SYS_WATCHDOG_FAILED_FILE" ]] && [[ -s "$SYS_WATCHDOG_FAILED_FILE" ]]; then
SKIP_COUNT=$(wc -l < "$SYS_WATCHDOG_FAILED_FILE")
SKIP_LIST=$(cat "$SYS_WATCHDOG_FAILED_FILE" | tr '\n' ' ')
DIGEST_LINES+=("$ICON_NOT_RUNNING Skip list: $SKIP_COUNT containers — $SKIP_LIST")
ISSUES+=("Containers on skip list: $SKIP_LIST")
SHOULD_SEND=true
# ── Transcode Ramdisk ─────────────────────────────────────────────────────────────────────────
if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
RAMDISK_USED_KB=$(df "$RAMDISK_PATH" --output=used 2>/dev/null | tail -1 | tr -d ' ')
RAMDISK_USED_GB=$(awk "BEGIN {printf \"%.2f\", ${RAMDISK_USED_KB:-0} / 1048576}")
SYMLINK_TARGET=$(readlink "$TRANSCODE_LINK" 2>/dev/null || echo "unknown")
DIGEST_LINES+=("$ICON_RAM Transcodes: ${RAMDISK_USED_GB}GB used → $SYMLINK_TARGET")
# Weekly transcode stats from TRANSCODE_DAILY_LOG
if [[ -f "${TRANSCODE_DAILY_LOG:-}" ]]; then
WEEK_CUTOFF=$(date -d "7 days ago" '+%Y-%m-%d')
WEEK_PEAK=$(awk -F'|' -v c="$WEEK_CUTOFF" \
'$1>=c{if($2>max)max=$2} END{printf "%.2f",max+0}' "$TRANSCODE_DAILY_LOG")
WEEK_FLIPS=$(awk -F'|' -v c="$WEEK_CUTOFF" \
'$1>=c{sum+=$3} END{print sum+0}' "$TRANSCODE_DAILY_LOG")
WEEK_RAM=$(awk -F'|' -v c="$WEEK_CUTOFF" \
'$1>=c{sum+=$4} END{print sum+0}' "$TRANSCODE_DAILY_LOG")
WEEK_SSD=$(awk -F'|' -v c="$WEEK_CUTOFF" \
'$1>=c{sum+=$5} END{print sum+0}' "$TRANSCODE_DAILY_LOG")
WEEK_FILES=$(awk -F'|' -v c="$WEEK_CUTOFF" \
'$1>=c{sum+=$6} END{print sum+0}' "$TRANSCODE_DAILY_LOG")
DIGEST_LINES+=("$ICON_RAM Transcode week: peak ${WEEK_PEAK}GB | flips: $WEEK_FLIPS | sessions: ${WEEK_RAM} ramdisk / ${WEEK_SSD} SSD | cleaned: ${WEEK_FILES} files")
PEAK_INT=$(printf "%.0f" "$WEEK_PEAK")
WARN_INT=$(printf "%.0f" "${RAMDISK_WARN_GB:-6.8}")
if [[ "$PEAK_INT" -ge "$WARN_INT" ]]; then
DIGEST_LINES+=("$ICON_WARN Peak ${WEEK_PEAK}GB near threshold ${RAMDISK_WARN_GB}GB — consider increasing HOST*_RAMDISK_SIZE")
FINDINGS+=("Transcode ramdisk near threshold: ${WEEK_PEAK}GB / ${RAMDISK_WARN_GB}GB")
fi
fi
else
DIGEST_LINES+=("$ICON_RUNNING Skip list: empty — all containers healthy")
DIGEST_LINES+=("$ICON_RAM Transcodes: ramdisk not mounted")
ISSUES+=("Ramdisk not mounted at $RAMDISK_PATH")
SHOULD_SEND=true
fi
# ── Bandwidth (yesterday's total) ───────────────────────────────────────────────────────────
if [[ -f "$BANDWIDTH_LOG" ]] && [[ -s "$BANDWIDTH_LOG" ]]; then
# ── Bandwidth ─────────────────────────────────────────────────────────────────────────────────
# Updated for new log format: date|time|profile|duration|status|bytes|warn_flag
if [[ -f "${BANDWIDTH_LOG:-}" ]] && [[ -s "$BANDWIDTH_LOG" ]]; then
YESTERDAY=$(date -d "yesterday" '+%Y-%m-%d')
YESTERDAY_BYTES=$(awk -F'|' -v d="$YESTERDAY" '$1==d{sum+=$3} END{print sum+0}' "$BANDWIDTH_LOG")
YESTERDAY_GB=$(awk "BEGIN {printf \"%.2f\", $YESTERDAY_BYTES / 1073741824}")
OVER_WARN=$(awk "BEGIN {print ($YESTERDAY_BYTES > $BANDWIDTH_WARN_GB * 1073741824) ? 1 : 0}")
YESTERDAY_BYTES=$(awk -F'|' -v d="$YESTERDAY" '$1==d{sum+=$6} END{print sum+0}' \
"$BANDWIDTH_LOG")
YESTERDAY_GB=$(awk "BEGIN {printf \"%.2f\", ${YESTERDAY_BYTES:-0} / 1073741824}")
YESTERDAY_LARGE=$(awk -F'|' -v d="$YESTERDAY" '$1==d && $7=="LARGE"' \
"$BANDWIDTH_LOG" | wc -l)
if [[ "$OVER_WARN" == "1" ]]; then
DIGEST_LINES+=("$ICON_BANDWIDTH Yesterday's transfers: ${YESTERDAY_GB}GB ← exceeded ${BANDWIDTH_WARN_GB}GB threshold")
if [[ "${YESTERDAY_LARGE:-0}" -gt 0 ]]; then
DIGEST_LINES+=("$ICON_BANDWIDTH Yesterday: ${YESTERDAY_GB}GB — $YESTERDAY_LARGE large transfer(s) exceeded ${BANDWIDTH_WARN_GB}GB")
FINDINGS+=("High bandwidth day: ${YESTERDAY_GB}GB transferred")
[[ "$DIGEST_SMART_ON_BANDWIDTH" == true ]] && SHOULD_SEND=true
else
DIGEST_LINES+=("$ICON_BANDWIDTH Yesterday's transfers: ${YESTERDAY_GB}GB")
DIGEST_LINES+=("$ICON_BANDWIDTH Yesterday: ${YESTERDAY_GB}GB")
fi
else
DIGEST_LINES+=("$ICON_BANDWIDTH Bandwidth: no data yet")
fi
# ── SSL Certificates ────────────────────────────────────────────────────────────────────────
# ── SSL Certificates ──────────────────────────────────────────────────────────────────────────
if [[ ${#CERT_MONITOR_DOMAINS[@]} -gt 0 ]] && command -v openssl >/dev/null 2>&1; then
CERT_ISSUES=()
for domain in "${CERT_MONITOR_DOMAINS[@]}"; do
[[ -z "$domain" ]] && continue
expiry_str=$(echo | timeout "$CERT_TIMEOUT" openssl s_client \
expiry_str=$(echo | timeout "${CERT_TIMEOUT:-10}" openssl s_client \
-connect "${domain}:443" -servername "$domain" \
2>/dev/null | openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2)
if [[ -n "$expiry_str" ]]; then
expiry_epoch=$(date -d "$expiry_str" +%s 2>/dev/null)
days_remaining=$(( (expiry_epoch - $(date +%s)) / 86400 ))
if [[ "$days_remaining" -le "$CERT_CRIT_DAYS" ]]; then
if [[ "$days_remaining" -le "${CERT_CRIT_DAYS:-7}" ]]; then
CERT_ISSUES+=("$domain: ${days_remaining}d CRITICAL")
SHOULD_SEND=true
elif [[ "$days_remaining" -le "$CERT_WARN_DAYS" ]]; then
CERT_ISSUES+=("$domain: ${days_remaining}d WARNING")
elif [[ "$days_remaining" -le "${CERT_WARN_DAYS:-30}" ]]; then
CERT_ISSUES+=("$domain: ${days_remaining}d warning")
[[ "$DIGEST_SMART_ON_CERT_WARN" == true ]] && SHOULD_SEND=true
fi
fi
done
if [[ ${#CERT_ISSUES[@]} -gt 0 ]]; then
DIGEST_LINES+=("$ICON_CERT Certificates: ${CERT_ISSUES[*]}")
FINDINGS+=("Cert issues: ${CERT_ISSUES[*]}")
else
DIGEST_LINES+=("$ICON_CERT Certificates: all healthy")
DIGEST_LINES+=("$ICON_CERT Certificates: all healthy ✅")
fi
fi
# ── Smart profile final decision ────────────────────────────────────────────────────────────
# ==============================================================================================
# ── Smart profile — exit silently if nothing to report ────────────────────────────────────────
# ==============================================================================================
if [[ "$DIGEST_PROFILE" == "smart" && "$SHOULD_SEND" == false ]]; then
info "Profile: smart — no findings worth reporting — skipping notification"
echo ""
echo "━━━━━ $ICON_SUMMARY DIGEST SUMMARY ━━━━━"
echo "$ICON_SUCCESS Everything looks healthy — no digest sent (smart profile)"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
log "Profile: smart — no findings worth reporting — silent exit"
exit 0
fi
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_DIGEST Build and Send Digest ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Build and Send Digest ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_DIGEST Health Digest — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME)"
echo ""
echo "━━━ $ICON_DIGEST Health Digest ━━━"
DIGEST_DATE=$(date '+%Y-%m-%d %H:%M:%S')
DIGEST_HOST=$(hostname)
# Build notification message
NOTIFY_MSG="Health Digest — $DIGEST_HOST — $DIGEST_DATE"
if [[ ${#ISSUES[@]} -gt 0 ]]; then
NOTIFY_MSG+=" | Issues: ${ISSUES[*]}"
fi
if [[ ${#FINDINGS[@]} -gt 0 ]]; then
NOTIFY_MSG+=" | Findings: ${FINDINGS[*]}"
fi
# Print full digest to console
for line in "${DIGEST_LINES[@]}"; do
echo " $line"
done
echo ""
echo "━━━━━ $ICON_SUMMARY DIGEST SUMMARY ━━━━━"
echo "$ICON_DIGEST Profile: $DIGEST_PROFILE"
echo "$ICON_TIME Generated: $DIGEST_DATE"
echo "$ICON_ERROR Issues: ${#ISSUES[@]}"
echo "$ICON_WARN Findings: ${#FINDINGS[@]}"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_DIGEST Profile: $DIGEST_PROFILE"
echo "$ICON_ERROR Issues: ${#ISSUES[@]}"
echo "$ICON_WARN Findings: ${#FINDINGS[@]}"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
# Build notification message
NOTIFY_MSG="Health Digest — $MY_ID ($LOCAL_SERVER_NAME)"
[[ ${#ISSUES[@]} -gt 0 ]] && NOTIFY_MSG+=" | Issues: ${ISSUES[*]}"
[[ ${#FINDINGS[@]} -gt 0 ]] && NOTIFY_MSG+=" | Findings: ${FINDINGS[*]}"
[[ ${#ISSUES[@]} -eq 0 && ${#FINDINGS[@]} -eq 0 ]] && NOTIFY_MSG+=" | All systems healthy"
NOTIFY_SEV="normal"
[[ ${#ISSUES[@]} -gt 0 ]] && NOTIFY_SEV="warning"
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — digest generated but not sent"
elif [[ "$SHOULD_SEND" == true ]]; then
notify "$NOTIFY_MSG" "Health Digest" "$([[ ${#ISSUES[@]} -gt 0 ]] && echo "warning" || echo "normal")"
success "Digest sent"
notify "$NOTIFY_MSG" "Health Digest" "$NOTIFY_SEV"
log "Digest sent"
fi
Monitors/zfs_memory_snapshot.sh
+135 -102
View File
@@ -1,93 +1,133 @@
#!/bin/bash
# -----------------------------------------------------------------------------------------------
# --------------------------------- ZFS Memory Snapshot ----------------------------------------
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ================================= ZFS Memory Snapshot ========================================
# ==============================================================================================
# Weekly ZFS pool health and memory diagnostic report.
# Combines ZFS pool status, ARC statistics, memory summary, Docker memory usage
# and kernel pressure into a single report. Informational only — no action taken.
# system_watchdog.sh handles threshold-based intervention.
#
# ── WHAT IT REPORTS ───────────────────────────────────────────────────────────────────────────
# ZFS pool health — status, state, errors per pool (excluding ignored pools)
# ARC statistics — current size, max, utilization %, metadata pressure
# Memory status — total/free/available RAM vs thresholds
# Docker memory — top ZFS_REPORT_DOCKER_TOP containers by memory usage
# Kernel pressure — vmstat snapshot (3 samples)
#
# ── OUTPUT ────────────────────────────────────────────────────────────────────────────────────
# Output goes to both console and ZFS_REPORT_LOG for later review.
# In dry-run mode — console only, nothing written to log.
# Notifies if any warning thresholds are exceeded.
# Silent when all healthy — only problems produce output.
#
# Pools listed in ZFS_REPORT_IGNORE_POOLS are excluded from health reporting.
# Useful for pools expected to run at high usage (docker, cache etc.)
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
# detect_hosts() sets MY_ID and aliases HOST*_ZFS_REPORT_IGNORE_POOLS → ZFS_REPORT_IGNORE_POOLS.
# Each server ignores its own single-disk ZFS array pools — not the peer's.
#
# All configuration in Master.conf under ZFS Memory Snapshot section.
# Supports --dry-run (preview only, no log write) and --status.
# -----------------------------------------------------------------------------------------------
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
# acquire_lock — zpool + docker stats are slow, prevent duplicates
# detect_hosts() — correct pool ignore list per host
# validate_unraid_cmd — notify validated before use
# DOCKER_TIMEOUT — docker stats protected against hung daemon
# ZFS not available — skips pool and ARC sections gracefully
# Docker not available — skips container section gracefully
#
# ── CONFIGURATION (master_host*.conf) ─────────────────────────────────────────────────────────
# HOST*_ZFS_REPORT_IGNORE_POOLS — pools excluded from health reporting
# Aliased by detect_hosts() — script uses ZFS_REPORT_IGNORE_POOLS
#
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
# ZFS_REPORT_LOG — log file path for weekly reports
# ZFS_REPORT_ARC_WARN_PCT — warn if ARC using more than this % of max
# ZFS_REPORT_FREE_WARN_GB — warn if less than this GB free RAM
# ZFS_REPORT_AVAIL_WARN_GB — warn if less than this GB available RAM
# ZFS_REPORT_DOCKER_TOP — how many top Docker containers to show
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
# zfs_memory_snapshot.sh — normal report (writes to log)
# zfs_memory_snapshot.sh --dry-run — console only, no log write
# zfs_memory_snapshot.sh --log — verbose output
# zfs_memory_snapshot.sh --status — show config and exit
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../Master.conf"
source "$SCRIPT_DIR/../common.sh"
source "$SCRIPT_DIR/../load_config.sh"
# Monitor/report script — output is the point
SILENT_MODE=false
parse_args "$@"
DOCKER_TIMEOUT=15
# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
error "Must be run as root"
exit 1
fi
validate_unraid_cmd \
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
"" "" \
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
acquire_lock
# detect_hosts() sets MY_ID and aliases HOST*_ZFS_REPORT_IGNORE_POOLS
detect_hosts
# Build ignore pool lookup map — O(1) check per pool
declare -A IGNORE_POOL_MAP
for pool in "${ZFS_REPORT_IGNORE_POOLS[@]:-}"; do
[[ -n "$pool" ]] && IGNORE_POOL_MAP["$pool"]=1
done
log "Identity: $MY_ID ($LOCAL_SERVER_NAME)"
log "Ignoring pools: ${ZFS_REPORT_IGNORE_POOLS[*]:-none}"
# Tee output to log file unless dry run
if [[ "$DRY_RUN" == false ]]; then
mkdir -p "$(dirname "$ZFS_REPORT_LOG")"
exec > >(tee -a "$ZFS_REPORT_LOG") 2>&1
fi
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Setup ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_GEAR Setup ━━━"
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — output will not be written to log"
if [[ "$EUID" -ne 0 ]]; then
error "Must be run as root"
exit 1
fi
success "Running as root"
# Build ignore list for quick lookup
declare -A IGNORE_POOL_MAP
for pool in "${ZFS_REPORT_IGNORE_POOLS[@]}"; do
[[ -n "$pool" ]] && IGNORE_POOL_MAP["$pool"]=1
done
if [[ ${#ZFS_REPORT_IGNORE_POOLS[@]} -gt 0 ]]; then
info "Ignoring pools: ${ZFS_REPORT_IGNORE_POOLS[*]}"
fi
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Status ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
echo ""
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
echo "$ICON_ZFS Log file: $ZFS_REPORT_LOG"
echo "$ICON_ZFS ARC warn: ${ZFS_REPORT_ARC_WARN_PCT}%"
echo "$ICON_MEM Free RAM warn: ${ZFS_REPORT_FREE_WARN_GB}GB"
echo "$ICON_MEM Avail RAM warn: ${ZFS_REPORT_AVAIL_WARN_GB}GB"
echo "$ICON_CONTAINERS Docker top: $ZFS_REPORT_DOCKER_TOP"
echo "$ICON_ZFS Ignore pools: ${ZFS_REPORT_IGNORE_POOLS[*]:-none}"
echo "$ICON_GEAR Dry Run: $DRY_RUN"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_ZFS Log file: $ZFS_REPORT_LOG"
echo "$ICON_ZFS ARC warn: ${ZFS_REPORT_ARC_WARN_PCT}%"
echo "$ICON_MEM Free RAM warn: ${ZFS_REPORT_FREE_WARN_GB}GB"
echo "$ICON_MEM Avail warn: ${ZFS_REPORT_AVAIL_WARN_GB}GB"
echo "$ICON_CONTAINERS Docker top: $ZFS_REPORT_DOCKER_TOP"
echo "$ICON_ZFS Ignore pools: ${ZFS_REPORT_IGNORE_POOLS[*]:-none}"
echo "$ICON_GEAR Dry Run: $DRY_RUN"
echo "━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
fi
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — output will not be written to log"
# -----------------------------------------------------------------------------------------------
# Tracking
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Report ━━━
# ==============================================================================================
WARNINGS=()
START=$(date +%s)
DATE=$(date +"%Y-%m-%d %H:%M:%S")
DATE=$(date '+%Y-%m-%d %H:%M:%S')
echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo " $ICON_ZFS ZFS WEEKLY HEALTH REPORT — $DATE"
echo " $ICON_HOST Host: $(hostname)"
echo " $ICON_HOST $MY_ID — $LOCAL_SERVER_NAME"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_ZFS ZFS Pool Health ━━━
# -----------------------------------------------------------------------------------------------
# ── ZFS Pool Health ───────────────────────────────────────────────────────────────────────────
echo ""
echo "━━━ $ICON_ZFS ZFS Pool Health ━━━"
@@ -95,13 +135,11 @@ if ! command -v zpool >/dev/null 2>&1; then
warn "ZFS not available on this system — skipping pool checks"
else
# Pool status — filtered to key lines, ignoring specified pools
info "Pool status:"
CURRENT_POOL=""
while IFS= read -r line; do
# Extract pool name from "pool: poolname" lines
if [[ "$line" =~ ^[[:space:]]*pool:[[:space:]]*(.+) ]]; then
CURRENT_POOL="${BASH_REMATCH[1]// /}"
fi
# Skip lines belonging to ignored pools
[[ -n "${IGNORE_POOL_MAP[$CURRENT_POOL]:-}" ]] && continue
echo " $line"
done < <(zpool status 2>/dev/null | grep -E "pool:|state:|status:|errors:|scan:")
@@ -109,41 +147,36 @@ else
echo ""
# Pool list — filter out ignored pools
info "Pool overview:"
zpool list 2>/dev/null | while IFS= read -r line; do
# Always show header line
if [[ "$line" == NAME* ]]; then
echo " $line"
continue
fi
# Extract pool name (first field)
pool_name=$(echo "$line" | awk '{print $1}')
[[ -n "${IGNORE_POOL_MAP[$pool_name]:-}" ]] && continue
echo " $line"
done
# Check for unhealthy pools — excluding ignored ones
UNHEALTHY=$(zpool list -H -o name,health 2>/dev/null | while IFS=$'\t' read -r name health; do
[[ -n "${IGNORE_POOL_MAP[$name]:-}" ]] && continue
[[ "$health" != "ONLINE" ]] && echo "$name: $health"
done)
# Check for unhealthy non-ignored pools
UNHEALTHY=$(zpool list -H -o name,health 2>/dev/null | \
while IFS=$'\t' read -r name health; do
[[ -n "${IGNORE_POOL_MAP[$name]:-}" ]] && continue
[[ "$health" != "ONLINE" ]] && echo "$name: $health"
done)
if [[ -n "$UNHEALTHY" ]]; then
error "One or more ZFS pools are NOT ONLINE: $UNHEALTHY"
WARNINGS+=("ZFS pool unhealthy: $UNHEALTHY")
else
success "All monitored ZFS pools are ONLINE"
log "All monitored ZFS pools are ONLINE ✅"
fi
# Show ignored pools
if [[ ${#ZFS_REPORT_IGNORE_POOLS[@]} -gt 0 ]]; then
info "Ignored pools (not reported): ${ZFS_REPORT_IGNORE_POOLS[*]}"
log "Ignored pools: ${ZFS_REPORT_IGNORE_POOLS[*]}"
fi
fi
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_ZFS ARC Statistics ━━━
# -----------------------------------------------------------------------------------------------
# ── ARC Statistics ────────────────────────────────────────────────────────────────────────────
echo ""
echo "━━━ $ICON_ZFS ARC Statistics ━━━"
@@ -170,14 +203,16 @@ else
warn "ARC utilization ${ARC_PCT}% — above ${ZFS_REPORT_ARC_WARN_PCT}% threshold"
WARNINGS+=("ARC high: ${ARC_PCT}%")
else
success "ARC utilization ${ARC_PCT}% — within threshold (${ZFS_REPORT_ARC_WARN_PCT}%)"
log "ARC utilization ${ARC_PCT}% — within threshold ✅"
fi
echo ""
info "Metadata pressure:"
META_MRU_GHOST=$(awk '/^mru_ghost_metadata / {print $3}' /proc/spl/kstat/zfs/arcstats 2>/dev/null || echo 0)
META_MFU_GHOST=$(awk '/^mfu_ghost_metadata / {print $3}' /proc/spl/kstat/zfs/arcstats 2>/dev/null || echo 0)
META_MISSES=$(awk '/^demand_metadata_misses / {print $3}' /proc/spl/kstat/zfs/arcstats 2>/dev/null || echo 0)
META_MRU_GHOST=$(awk '/^mru_ghost_metadata / {print $3}' \
/proc/spl/kstat/zfs/arcstats 2>/dev/null || echo 0)
META_MFU_GHOST=$(awk '/^mfu_ghost_metadata / {print $3}' \
/proc/spl/kstat/zfs/arcstats 2>/dev/null || echo 0)
META_MISSES=$(awk '/^demand_metadata_misses / {print $3}' \
/proc/spl/kstat/zfs/arcstats 2>/dev/null || echo 0)
MRU_GB=$(awk "BEGIN {printf \"%.2f\", $META_MRU_GHOST / 1073741824}")
MFU_GB=$(awk "BEGIN {printf \"%.2f\", $META_MFU_GHOST / 1073741824}")
@@ -187,17 +222,15 @@ else
echo " $ICON_ZFS Metadata Misses: ${META_MISSES}"
fi
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_MEM Memory Status ━━━
# -----------------------------------------------------------------------------------------------
# ── Memory Status ─────────────────────────────────────────────────────────────────────────────
echo ""
echo "━━━ $ICON_MEM Memory Status ━━━"
FREE_HUMAN=$(free -h | awk '/Mem:/ {print $4}')
FREE_HUMAN=$(free -h | awk '/Mem:/ {print $4}')
AVAIL_HUMAN=$(free -h | awk '/Mem:/ {print $7}')
TOTAL_HUMAN=$(free -h | awk '/Mem:/ {print $2}')
FREE_GB=$(free -g | awk '/Mem:/ {print $4}')
AVAIL_GB=$(free -g | awk '/Mem:/ {print $7}')
FREE_GB=$(free -g | awk '/Mem:/ {print $4}')
AVAIL_GB=$(free -g | awk '/Mem:/ {print $7}')
echo " $ICON_MEM Total RAM: $TOTAL_HUMAN"
echo " $ICON_MEM Free RAM: $FREE_HUMAN"
@@ -207,42 +240,38 @@ if [[ "$FREE_GB" -lt "$ZFS_REPORT_FREE_WARN_GB" ]]; then
warn "Free RAM ${FREE_HUMAN} — below ${ZFS_REPORT_FREE_WARN_GB}GB threshold"
WARNINGS+=("Low free RAM: ${FREE_HUMAN}")
else
success "Free RAM ${FREE_HUMAN} — within threshold (${ZFS_REPORT_FREE_WARN_GB}GB)"
log "Free RAM ${FREE_HUMAN} — within threshold ✅"
fi
if [[ "$AVAIL_GB" -lt "$ZFS_REPORT_AVAIL_WARN_GB" ]]; then
warn "Available RAM ${AVAIL_HUMAN} — below ${ZFS_REPORT_AVAIL_WARN_GB}GB threshold"
WARNINGS+=("Low available RAM: ${AVAIL_HUMAN}")
else
success "Available RAM ${AVAIL_HUMAN} — within threshold (${ZFS_REPORT_AVAIL_WARN_GB}GB)"
log "Available RAM ${AVAIL_HUMAN} — within threshold ✅"
fi
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_CONTAINERS Docker Memory ━━━
# -----------------------------------------------------------------------------------------------
# ── Docker Memory ─────────────────────────────────────────────────────────────────────────────
echo ""
echo "━━━ $ICON_CONTAINERS Top $ZFS_REPORT_DOCKER_TOP Docker Memory Users ━━━"
if ! command -v docker >/dev/null 2>&1; then
warn "Docker not available — skipping container memory section"
else
docker stats --no-stream \
timeout "$DOCKER_TIMEOUT" docker stats --no-stream \
--format "table {{.Name}}\t{{.MemUsage}}\t{{.MemPerc}}" \
2>/dev/null | head -n $(( ZFS_REPORT_DOCKER_TOP + 1 )) | while IFS= read -r line; do
echo " $line"
done
2>/dev/null | head -n $(( ZFS_REPORT_DOCKER_TOP + 1 )) | \
while IFS= read -r line; do
echo " $line"
done
fi
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Kernel Pressure ━━━
# -----------------------------------------------------------------------------------------------
# ── Kernel Pressure ───────────────────────────────────────────────────────────────────────────
echo ""
echo "━━━ $ICON_GEAR Kernel Pressure ━━━"
if ! command -v vmstat >/dev/null 2>&1; then
warn "vmstat not available — skipping kernel pressure section"
else
info "vmstat snapshot (3 samples):"
vmstat 1 3 2>/dev/null | while IFS= read -r line; do
echo " $line"
done
@@ -250,23 +279,27 @@ fi
END=$(date +%s)
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Summary ━━━
# -----------------------------------------------------------------------------------------------
# ── Summary ───────────────────────────────────────────────────────────────────────────────────
echo ""
echo "━━━━━ $ICON_SUMMARY ZFS REPORT SUMMARY ━━━━━"
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
echo "$ICON_ZFS Log: $ZFS_REPORT_LOG"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
echo "$ICON_ZFS Log: $ZFS_REPORT_LOG"
[[ ${#ZFS_REPORT_IGNORE_POOLS[@]} -gt 0 ]] && \
echo "$ICON_ZFS Ignored: ${ZFS_REPORT_IGNORE_POOLS[*]}"
log "Ignored: ${ZFS_REPORT_IGNORE_POOLS[*]}"
echo ""
if [[ ${#WARNINGS[@]} -eq 0 ]]; then
success "Report complete — no warnings"
log "$ICON_DONE All checks within thresholds ✅"
else
echo "$ICON_WARN Warnings: ${#WARNINGS[@]}"
for w in "${WARNINGS[@]}"; do
echo " $ICON_WARN $w"
done
notify "ZFS weekly report on $(hostname) — ${#WARNINGS[@]} warning(s): ${WARNINGS[*]}" "ZFS Report" "warning"
notify "ZFS weekly report on $(hostname) — ${#WARNINGS[@]} warning(s): ${WARNINGS[*]}" \
"ZFS Report" "warning"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
[[ ${#WARNINGS[@]} -gt 0 ]] && exit 1
exit 0
Notes_To-Do.md
+40
View File
@@ -0,0 +1,40 @@
echo 'export PATH="$HOME/.local/bin:$PATH"' >> ~/.bashrc
source ~/.bashrc
claude
arrs system error, can we delete movies from lidar when dropped from tmdb, 99% of the time its future movies that get dropped
movie Silent Hill 2: The Movie (tmdbid 466226) was removed from TMDb
later.
. fix failover strike list timing
. verify silent toggle switches back on good notifications
. add to partnership, on offboard, remove all of containers that belonged to rmote, example remotes vaultwarden-jayred from my machine and leave my vaultwarden-Gmer4Lfe alone. and it does nothing to remote, thier side will hadle thier pc and remove my stuff from thier pc.
. add updater to update containers while daily runs, along with a toggle to dissable in master.
. add a script to check all docker containers and update any that still need it to run after the containers that get synced and updated.
error starting emby
--mount type=bind,source=/mnt/ram-transcode,target=/ext-ram-transcode,bind-propagation=shared 'lscr.io/linuxserver/emby'
0e616c065aebad2604d2120fbf9c6bb449cdfc4021d68f167e4b5bc01741c133
docker: Error response from daemon: failed to create task for container: failed to create shim task: OCI runtime create failed: runc create failed: unable to start container process: error during container init: error jailing process inside rootfs: open /proc/self/mountinfo: no such file or directory
Run 'docker run --help' for more information
The command failed.
had to use --mount type=bind,source=/mnt/ram-transcode,target=/ext-ram-transcode instead, for now, just to get it back online
Orchestrators/README-Orchestrators.md
+834 -332
View File
File diff suppressed because it is too large Load Diff
Orchestrators/array_start.sh
+157 -51
View File
@@ -1,53 +1,136 @@
#!/bin/bash
# -----------------------------------------------------------------------------------------------
# --------------------------------- Array Start Orchestrator -----------------------------------
# -----------------------------------------------------------------------------------------------
# Single entry point for "At Startup of Array" in User Scripts plugin.
# Launches everything configured in ARRAY_START_SCRIPTS in Master.conf.
# ==============================================================================================
# ================================= Array Start Orchestrator ===================================
# ==============================================================================================
# Single entry point for "At Startup of Array" in the User Scripts plugin.
# Launches everything configured in ARRAY_START_SCRIPTS in master.conf.
# This script exits after launching all scripts — unRAID sees it complete normally.
#
# What it launches (configured in Master.conf ARRAY_START_SCRIPTS):
# Transcodes/ramdisk_setup.sh — creates tmpfs + symlink before Emby starts (one-shot)
# unRAID_Essentials/docker_syslog_filter.sh — suppress veth log noise (one-shot)
# unRAID_Essentials/php_fpm_max_children.sh — WebGUI performance tuning (one-shot)
# Docker_Essentials/docker_network_connect.sh — ensure networks exist + connect containers (one-shot)
# unRAID_Essentials/system_watchdog.sh — system health monitor (continuous loop)
# Docker_Essentials/docker_watchdog.sh — container health monitor (continuous loop)
# Failover/failover.sh — mutual failover monitor (continuous loop)
# ── WHAT IT LAUNCHES ──────────────────────────────────────────────────────────────────────────
# Configured in master.conf ARRAY_START_SCRIPTS — no changes to this script ever needed.
# Current order (order matters — see below):
#
# One-shot scripts run and exit naturally — array_start.sh confirms completion.
# Continuous scripts run until array stops or SIGTERM received.
# This orchestrator exits after launching all scripts — unRAID sees it complete normally.
# ONE-SHOT (run and exit naturally):
# unRAID_Essentials/inotify_tuning.sh — raise inotify limits before containers start
# unRAID_Essentials/docker_syslog_filter.sh — suppress veth log noise before logs fill
# unRAID_Essentials/php_fpm_max_children.sh — WebGUI performance tuning
# Transcodes/ramdisk_setup.sh — create tmpfs + symlink before Emby starts
# Docker_Essentials/docker_network_connect.sh — ensure networks + container connections
#
# Add or remove scripts: edit ARRAY_START_SCRIPTS in Master.conf.
# Order matters — ramdisk first, network before watchdogs, watchdogs before failover.
# No changes to this script ever needed.
# -----------------------------------------------------------------------------------------------
# CONTINUOUS (run until array stops):
# unRAID_Essentials/system_watchdog.sh — system health monitor (last line of defense)
# Docker_Essentials/docker_watchdog.sh — container health monitor
# Failover/failover.sh — mutual failover monitor
#
# ── WHY ORDER MATTERS ─────────────────────────────────────────────────────────────────────────
# inotify_tuning.sh — must run BEFORE Code-Server and other containers start
# containers that start with low inotify limits keep them ✅
# docker_syslog_filter — must run BEFORE any container starts creating veth interfaces
# ramdisk_setup.sh — must run BEFORE Emby starts transcoding
# docker_network_connect — must run BEFORE watchdogs check container states
# system_watchdog.sh — before docker_watchdog (system > container priority)
# docker_watchdog.sh — before failover (containers must be healthy for failover)
# failover.sh — last — needs everything else stable to make decisions
#
# ── ONE-SHOT vs CONTINUOUS DETECTION ─────────────────────────────────────────────────────────
# Script is launched in background with bash script.sh &
# After 1 second: if PID still alive → continuous (running in background)
# if PID dead + exit 0 → one-shot completed successfully
# if PID dead + exit N → failure
#
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
# Root check — all launched scripts require root
# acquire_lock — prevents duplicate array start launches
# detect_hosts() — MY_ID in notifications
# validate_unraid_cmd — notify validated before use
# chmod +x auto-fix — non-executable scripts fixed before launch
# Full path on failure — shows exact path for debugging
# notify on failures — alert if any script fails to launch
#
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
# ARRAY_START_SCRIPTS — ordered list of scripts to launch at array start
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
# array_start.sh — normal launch (called by User Scripts at array start)
# array_start.sh --dry-run — show what would be launched without launching
# array_start.sh --status — show configured scripts and their current state
# array_start.sh --log — verbose output per script
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
source "$ECOSYSTEM_ROOT/Master.conf"
source "$ECOSYSTEM_ROOT/common.sh"
source "$ECOSYSTEM_ROOT/load_config.sh"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Setup ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_GEAR Array Start — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
parse_args "$@"
# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
error "Must be run as root"
exit 1
fi
success "Running as root"
validate_unraid_cmd \
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
"" "" \
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
SCRIPT_COUNT=${#ARRAY_START_SCRIPTS[@]}
info "Launching $SCRIPT_COUNT script(s)..."
acquire_lock
detect_hosts
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — scripts will not be launched"
# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
echo ""
echo "━━━━━ $ICON_SUMMARY ARRAY START STATUS ━━━━━"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_GEAR Scripts: ${#ARRAY_START_SCRIPTS[@]} configured"
echo ""
for relative_path in "${ARRAY_START_SCRIPTS[@]}"; do
[[ -z "$relative_path" ]] && continue
script_path="$ECOSYSTEM_ROOT/$relative_path"
script_name=$(basename "$script_path")
if [[ ! -f "$script_path" ]]; then
echo " $ICON_ERROR $script_name — FILE NOT FOUND"
echo " $script_path"
continue
fi
[[ ! -x "$script_path" ]] && flag=" (not executable — will auto-fix)" || flag=""
# Check if currently running
if pgrep -f "$script_path" >/dev/null 2>&1; then
RUN_PID=$(pgrep -f "$script_path" | head -1)
echo " $ICON_RUNNING $script_name — RUNNING (PID $RUN_PID)${flag}"
else
echo " $ICON_NOT_RUNNING $script_name — not running${flag}"
fi
done
echo "━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
fi
# ==============================================================================================
# ━━━ Launch Scripts ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_GEAR Array Start — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
log "Ecosystem root: $ECOSYSTEM_ROOT"
log "Launching ${#ARRAY_START_SCRIPTS[@]} script(s)..."
echo ""
START=$(date +%s)
LAUNCHED=0
FAILED=0
FAILED_SCRIPTS=()
for relative_path in "${ARRAY_START_SCRIPTS[@]}"; do
[[ -z "$relative_path" ]] && continue
@@ -55,57 +138,80 @@ for relative_path in "${ARRAY_START_SCRIPTS[@]}"; do
SCRIPT_PATH="$ECOSYSTEM_ROOT/$relative_path"
SCRIPT_NAME=$(basename "$SCRIPT_PATH")
# File existence check
if [[ ! -f "$SCRIPT_PATH" ]]; then
error "$SCRIPT_NAME — not found at $SCRIPT_PATH"
((FAILED++))
error "$SCRIPT_NAME — not found"
error " Expected: $SCRIPT_PATH"
(( FAILED++ ))
FAILED_SCRIPTS+=("$SCRIPT_NAME")
continue
fi
# Auto-fix permissions — chmod +x if needed
if [[ ! -x "$SCRIPT_PATH" ]]; then
error "$SCRIPT_NAME — not executable"
((FAILED++))
warn "$SCRIPT_NAME — not executable, fixing..."
chmod +x "$SCRIPT_PATH" || {
error "$SCRIPT_NAME — chmod +x failed"
(( FAILED++ ))
FAILED_SCRIPTS+=("$SCRIPT_NAME")
continue
}
fi
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would launch: $SCRIPT_NAME"
(( LAUNCHED++ ))
continue
fi
info "$ICON_START Launching $SCRIPT_NAME..."
log "$ICON_START Launching $SCRIPT_NAME..."
bash "$SCRIPT_PATH" &
PID=$!
# Brief pause to let script initialize and catch immediate failures
# Brief settle — 1s enough to detect immediate failures
sleep 1
if kill -0 "$PID" 2>/dev/null; then
success "$SCRIPT_NAME — running (PID $PID)"
((LAUNCHED++))
# Still running → continuous script
warn "$SCRIPT_NAME — running (PID $PID) ✅"
(( LAUNCHED++ ))
else
# Script exited — check if it was a one-shot (exit 0) or a failure
# Exited — check if one-shot success or failure
wait "$PID"
EXIT_CODE=$?
if [[ "$EXIT_CODE" -eq 0 ]]; then
success "$SCRIPT_NAME — completed (one-shot)"
((LAUNCHED++))
log "$SCRIPT_NAME — completed (one-shot) ✅"
(( LAUNCHED++ ))
else
error "$SCRIPT_NAME — exited with code $EXIT_CODE"
((FAILED++))
error " Path: $SCRIPT_PATH"
(( FAILED++ ))
FAILED_SCRIPTS+=("$SCRIPT_NAME")
fi
fi
done
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Summary ━━━
# -----------------------------------------------------------------------------------------------
END=$(date +%s)
# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
echo ""
echo "━━━━━ $ICON_SUMMARY ARRAY START SUMMARY ━━━━━"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_SUCCESS Launched: $LAUNCHED"
echo "$ICON_ERROR Failed: $FAILED"
echo "$ICON_TIME Time: $(date '+%H:%M:%S')"
[[ "$FAILED" -gt 0 ]] && echo "$ICON_ERROR Failed: $FAILED — ${FAILED_SCRIPTS[*]}"
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
echo ""
if [[ "$FAILED" -gt 0 ]]; then
echo "$ICON_WARN Status: $FAILED script(s) failed to launch — check logs"
notify "Array start on $(hostname) — $FAILED script(s) failed to launch" "Array Start" "warning"
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — no scripts launched"
elif [[ "$FAILED" -gt 0 ]]; then
warn "Status: $FAILED script(s) failed — ${FAILED_SCRIPTS[*]}"
notify "Array start on $(hostname) ($MY_ID) — $FAILED script(s) failed: ${FAILED_SCRIPTS[*]}" \
"Array Start" "warning"
else
echo "$ICON_DONE Status: $ICON_SUCCESS All scripts launched"
log "$ICON_DONE Status: all $LAUNCHED script(s) launched ✅"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
Orchestrators/critical_sync_maintenance.sh
+145 -85
View File
@@ -1,69 +1,136 @@
#!/bin/bash
# -----------------------------------------------------------------------------------------------
# ----------------------------- Critical Sync Maintenance --------------------------------------
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ============================= Critical Sync Maintenance ======================================
# ==============================================================================================
# Orchestrator for time-sensitive syncs that run every 15 minutes.
# Keeps the mirror current between the less frequent daily and weekly windows.
# Schedule: */15 * * * * (every 15 minutes)
# Schedule: */15 * * * * (every 15 minutes via User Scripts plugin)
#
# Execution order:
# ── EXECUTION ORDER ───────────────────────────────────────────────────────────────────────────
# 1. Critical-Data rsync — auth stack, NPM config, certs (containers stopped both sides)
# 2. emby-failover rsync — dirty Emby sync (watch states, library — Emby stays running)
# 3. partnership --check — read both state files, detect changes, act accordingly
# 3. CRITICAL_MAINTENANCE_SCRIPTS — any scripts configured for critical window
# 4. partnership --check — read both state files, detect changes, act accordingly
#
# Why every 15 minutes:
# ── WHY EVERY 15 MINUTES ──────────────────────────────────────────────────────────────────────
# Auth stack changes (new users, proxy rules, certs) propagate within 15min ✅
# Emby watch states stay in sync — mirror users see correct playback position ✅
# Partnership state changes detected and acted on quickly ✅
# Lock prevents: daily rsync doing Critical-Data mid-critical window ✅
#
# Rsync gate:
# RSYNC_ENABLED=false → skips all syncs (global gate)
# CRITICAL_RSYNC_ENABLED=false → skips critical syncs (per-orchestrator)
# partnership --check still runs regardless of rsync gate
# (state check doesn't need rsync to work)
# ── RSYNC GATE ────────────────────────────────────────────────────────────────────────────────
# RSYNC_ENABLED=false → skips all syncs (global gate)
# CRITICAL_RSYNC_ENABLED=false → skips critical syncs only (per-orchestrator gate)
# partnership --check always runs regardless — state check doesn't need rsync
#
# Lock behavior:
# acquire_lock "strict" — if previous 15min run still going, skip this cycle
# ── LOCK BEHAVIOUR ────────────────────────────────────────────────────────────────────────────
# acquire_lock "strict" — if previous 15min run still going, skip this cycle entirely
# Critical-Data taking > 15min is a problem worth knowing about
# Lock prevents pile-up ✅
# Strict mode prevents pile-up without waiting — log and move on ✅
#
# Configuration in Master.conf:
# CRITICAL_RSYNC_ENABLED — enable/disable rsync section
# PARTNERSHIP_ENABLED — enable/disable partnership check
# CRITICAL_SYNC_SHARES — shares synced every 15min
# -----------------------------------------------------------------------------------------------
# ── SILENT WHEN HEALTHY ───────────────────────────────────────────────────────────────────────
# Runs 96 times per day — clean runs must produce zero output ✅
# Only failures and notable events produce visible output
#
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
# CRITICAL_RSYNC_ENABLED — enable/disable rsync section
# CRITICAL_SYNC_SHARES — shares synced every 15min (HOST*_CRITICAL_SYNC_SHARES)
# CRITICAL_MAINTENANCE_SCRIPTS — scripts run in critical window (optional)
# PARTNERSHIP_ENABLED — enable/disable partnership check
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
# critical_sync_maintenance.sh — normal run
# critical_sync_maintenance.sh --dry-run — preview syncs without transferring
# critical_sync_maintenance.sh --log — verbose per-share output
# critical_sync_maintenance.sh --status — show configuration and exit
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../Master.conf"
source "$SCRIPT_DIR/../common.sh"
source "$SCRIPT_DIR/../load_config.sh"
parse_args "$@"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Setup ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
error "Must be run as root"
exit 1
fi
validate_unraid_cmd \
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
"" "" \
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
acquire_lock "strict"
detect_hosts
resolve_remote_ip
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
echo ""
echo "━━━━━ $ICON_SUMMARY CRITICAL SYNC STATUS ━━━━━"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_NET Remote: $REMOTE_ID ($REMOTE_SERVER_NAME)"
echo "$ICON_SYNC Rsync enabled: ${RSYNC_ENABLED:-true}"
echo "$ICON_SYNC Critical enabled: ${CRITICAL_RSYNC_ENABLED:-false}"
echo "$ICON_SHIELD Partnership: ${PARTNERSHIP_ENABLED:-false}"
echo ""
echo "━━━ Critical Sync Shares ━━━"
if [[ ${#CRITICAL_SYNC_SHARES[@]} -eq 0 ]]; then
warn " No CRITICAL_SYNC_SHARES configured"
else
for share in "${CRITICAL_SYNC_SHARES[@]:-}"; do
[[ -z "$share" ]] && continue
SHARE_PATH="${share%%|*}"
SHARE_PROFILE="${share##*|}"
SHARE_NAME=$(basename "$SHARE_PATH")
[[ "$SHARE_PATH" == "$SHARE_PROFILE" ]] && \
echo " $ICON_SYNC $SHARE_NAME — no profile" || \
echo " $ICON_SYNC $SHARE_NAME — profile: $SHARE_PROFILE"
done
fi
echo ""
echo "━━━ Critical Maintenance Scripts ━━━"
if [[ ${#CRITICAL_MAINTENANCE_SCRIPTS[@]} -eq 0 ]]; then
echo " None configured"
else
for entry in "${CRITICAL_MAINTENANCE_SCRIPTS[@]:-}"; do
[[ -z "$entry" || "$entry" == \#* ]] && continue
echo " $ICON_GEAR $(basename "${entry%% *}")"
done
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
fi
# ==============================================================================================
# ━━━ Critical Shares Sync ━━━
# ==============================================================================================
START=$(date +%s)
RSYNC_OK=false
PASS=()
FAIL=()
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SYNC Critical Shares Sync ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_SYNC Critical Sync — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
if ! check_rsync_enabled "CRITICAL"; then
warn "Critical rsync disabled — skipping sync, running partnership check only"
log "Critical rsync disabled — skipping sync, running partnership check only"
elif [[ ${#CRITICAL_SYNC_SHARES[@]} -eq 0 ]]; then
warn "CRITICAL_RSYNC_ENABLED=true but CRITICAL_SYNC_SHARES is empty for $MY_ID"
warn "Check HOST*_CRITICAL_SYNC_SHARES in master_host*.conf"
else
log "Critical sync — $MY_ID → $REMOTE_ID — $(date '+%H:%M:%S')"
# Build dry-run flag to pass through
RSYNC_DRY=""
[[ "$DRY_RUN" == true ]] && RSYNC_DRY="--dry-run"
for share in "${CRITICAL_SYNC_SHARES[@]:-}"; do
[[ -z "$share" ]] && continue
@@ -72,101 +139,94 @@ else
SHARE_PROFILE="${share##*|}"
SHARE_NAME=$(basename "$SHARE_PATH")
echo ""
echo "━━━ $ICON_SYNC $SHARE_NAME ━━━"
SHARE_START=$(date +%s)
if [[ "$SHARE_PATH" == "$SHARE_PROFILE" ]]; then
# No profile specified
bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$SHARE_PATH"
bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$SHARE_PATH" $RSYNC_DRY
else
bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$SHARE_PATH" --profile="$SHARE_PROFILE"
bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$SHARE_PATH" \
--profile="$SHARE_PROFILE" $RSYNC_DRY
fi
RSYNC_EXIT=$?
SHARE_END=$(date +%s)
SHARE_DUR=$(format_duration $(( SHARE_END - SHARE_START )))
SHARE_DUR=$(format_duration $(( $(date +%s) - SHARE_START )))
if [[ "$RSYNC_EXIT" -eq 0 ]]; then
PASS+=("$SHARE_NAME")
success "$SHARE_NAME — done in $SHARE_DUR ✅"
log "$SHARE_NAME — done in $SHARE_DUR ✅"
RSYNC_OK=true
else
FAIL+=("$SHARE_NAME")
error "$SHARE_NAME — failed after $SHARE_DUR"
error "$SHARE_NAME — failed after $SHARE_DUR (exit $RSYNC_EXIT)"
fi
done
fi
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_CLEAN Critical Maintenance Scripts ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_CLEAN Critical Maintenance — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
if [[ ${#CRITICAL_MAINTENANCE_SCRIPTS[@]} -eq 0 ]]; then
log "No CRITICAL_MAINTENANCE_SCRIPTS defined — skipping"
else
# ==============================================================================================
# ━━━ Critical Maintenance Scripts ━━━
# ==============================================================================================
if [[ ${#CRITICAL_MAINTENANCE_SCRIPTS[@]} -gt 0 ]]; then
for script_entry in "${CRITICAL_MAINTENANCE_SCRIPTS[@]}"; do
[[ -z "$script_entry" ]] && continue
# Strip leading comment lines
[[ "$script_entry" == \#* ]] && continue
[[ -z "$script_entry" || "$script_entry" == \#* ]] && continue
SCRIPT_PATH="$SCRIPT_DIR/../${script_entry%% *}"
SCRIPT_ARGS="${script_entry#* }"
[[ "$SCRIPT_ARGS" == "$script_entry" ]] && SCRIPT_ARGS=""
[[ "$DRY_RUN" == true ]] && SCRIPT_ARGS="$SCRIPT_ARGS --dry-run"
SCRIPT_NAME=$(basename "$SCRIPT_PATH")
echo " → $SCRIPT_NAME"
if [[ ! -f "$SCRIPT_PATH" ]]; then
warn "$SCRIPT_NAME not found at $SCRIPT_PATH — skipping"
continue
fi
log "Running: $SCRIPT_NAME"
bash "$SCRIPT_PATH" $SCRIPT_ARGS
EXIT_CODE=$?
if [[ "$EXIT_CODE" -ne 0 ]]; then
[[ "$EXIT_CODE" -ne 0 ]] && \
warn "$SCRIPT_NAME exited with code $EXIT_CODE"
fi
done
fi
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SHIELD Partnership Check ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_SHIELD Partnership Check ━━━"
# ==============================================================================================
# ━━━ Partnership Check ━━━
# ==============================================================================================
if [[ "${PARTNERSHIP_ENABLED:-false}" == true ]]; then
PARTNER_DRY=""
[[ "$DRY_RUN" == true ]] && PARTNER_DRY="--dry-run"
if [[ "${PARTNERSHIP_ENABLED:-false}" == false ]]; then
log "Partnership disabled — skipping check"
else
# Pass rsync outcome to --check so it can update last_seen_remote
if [[ "$RSYNC_OK" == true ]]; then
bash "$SCRIPT_DIR/partnership_manage.sh" --check --remote-seen
bash "$SCRIPT_DIR/partnership_manage.sh" \
--check --remote-seen $PARTNER_DRY
else
bash "$SCRIPT_DIR/partnership_manage.sh" --check --remote-unseen
bash "$SCRIPT_DIR/partnership_manage.sh" \
--check --remote-unseen $PARTNER_DRY
fi
else
log "Partnership disabled — skipping check"
fi
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Summary ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
END=$(date +%s)
DURATION=$(format_duration $(( END - START )))
echo ""
echo "━━━━━ $ICON_SUMMARY CRITICAL SYNC SUMMARY ━━━━━"
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
if [[ ${#PASS[@]} -gt 0 ]]; then
echo "$ICON_SUCCESS Synced: ${PASS[*]}"
fi
# Silent when healthy — only show summary if there were failures or notable events
if [[ ${#FAIL[@]} -gt 0 ]]; then
echo ""
echo "━━━━━ $ICON_SUMMARY CRITICAL SYNC SUMMARY ━━━━━"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_TIME Duration: $DURATION"
[[ ${#PASS[@]} -gt 0 ]] && log "Synced: ${PASS[*]}"
echo "$ICON_ERROR Failed: ${FAIL[*]}"
notify "Critical sync failed on $(hostname) — ${FAIL[*]}" "Critical Sync" "warning"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
notify "Critical sync failed on $(hostname) ($MY_ID) — ${FAIL[*]}" \
"Critical Sync" "warning"
exit 1
else
log "Critical sync complete — $MY_ID — ${DURATION} — ${#PASS[@]} share(s)"
fi
if [[ ${#PASS[@]} -eq 0 ]] && [[ ${#FAIL[@]} -eq 0 ]]; then
echo "$ICON_SKIP Rsync: disabled"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
Orchestrators/daily_sync_maintenance.sh
+229 -226
View File
@@ -1,81 +1,139 @@
#!/bin/bash
# -----------------------------------------------------------------------------------------------
# --------------------------------- Daily Sync Maintenance ------------------------------------
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ============================= Daily Sync Maintenance =========================================
# ==============================================================================================
# Daily orchestrator — runs the full daily maintenance window in the correct order.
# Schedule: 0 1 * * * (1am daily)
# Schedule: 0 1 * * * (1am daily via User Scripts plugin)
#
# Execution order:
# 1. git_pull_execute.sh — pull latest scripts first, always
# 2. Media share sync — HOST*_DAILY_SYNC_SHARES pushed to remote
# 3. HOST*_PERSONAL_SHARES — personal encrypted shares after media
# 4. media_shares_permissions.sh — fix ownership before arr cleanup
# 5. media_cleaner.sh anime — remove junk from anime shares
# 6. media_cleaner.sh media — remove junk from media shares
# 7. lidarr_cleanup.sh — remove orphaned music files
# 8. sonarr_cleanup.sh — remove orphaned TV files
# 9. radarr_cleanup.sh — remove orphaned movie files
# 10. docker_daily_restart.sh — restart containers that need daily restart
# ── EXECUTION ORDER ───────────────────────────────────────────────────────────────────────────
# Pre-sync:
# git_pull_execute.sh — pull latest scripts first, always
#
# What triggers weekly_health_digest.sh:
# NOT this script — weekly_health_digest.sh runs on its own schedule (Saturday)
# This script writes no stats — it just syncs and maintains
# Rsync window (DAILY_SYNC_SHARES per host):
# HOST*_DAILY_SYNC_SHARES — media shares pushed to mirror
# HOST*_PERSONAL_SHARES — encrypted personal shares
#
# Configuration in Master.conf:
# DAILY_MAINTENANCE_SCRIPTS — pre/post-sync scripts (git pull, docker restart)
# DAILY_MAINTENANCE_SCRIPTS — media maintenance jobs run after sync
# HOST1_DAILY_SYNC_SHARES — shares HOST1 pushes to HOST2
# HOST2_DAILY_SYNC_SHARES — shares HOST2 pushes to HOST1
# HOST1/2_PERSONAL_SHARES — encrypted personal shares
# Post-sync maintenance (DAILY_MAINTENANCE_SCRIPTS):
# media_shares_permissions.sh — fix ownership before arr cleanup
# media_cleaner.sh anime — remove junk from anime shares
# media_cleaner.sh media — remove junk from media shares
# lidarr_cleanup.sh — remove orphaned music files
# sonarr_cleanup.sh — remove orphaned TV files
# radarr_cleanup.sh — remove orphaned movie files
# docker_daily_restart.sh — restart containers needing daily restart
#
# ── WHY ORDER MATTERS ─────────────────────────────────────────────────────────────────────────
# git pull first — maintenance runs on latest code, not yesterday's
# Rsync before cleanup — cleanup sees the post-sync state
# Permissions before arr cleanup — arrs need correct ownership to delete/rename
# Arr cleanup after permissions — clean ownership = successful orphan deletion
# Docker restart last — containers already processed by cleanup
#
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
# Bidirectional — same script runs on both servers, correct direction automatic.
# Per-share rsync handled by rsync.sh — this script tracks pass/fail and total time.
# -----------------------------------------------------------------------------------------------
# detect_hosts() aliases DAILY_SYNC_SHARES and PERSONAL_SHARES from HOST*_ vars.
# No manual HOST1/HOST2 comparisons — MY_ID routes correctly on any server.
#
# ── DRIVE TEMP HANDLING ───────────────────────────────────────────────────────────────────────
# rsync.sh returns exit codes for temperature issues:
# exit 1 = temp WARN — skip this share, continue to next
# exit 2 = temp CRITICAL — abort ALL remaining syncs in this window
# All other failures — skip share, continue to next
#
# ── SILENT WHEN HEALTHY ───────────────────────────────────────────────────────────────────────
# Runs daily at 1am — clean run should produce minimal output.
# Each job reports log() on success (silent), warn()/error() on failure (visible).
# Summary always shown — gives window timing and share/job counts.
# Notify only on failure — successful daily maintenance doesn't need notification.
#
# ── CONFIGURATION (master.conf + master_host*.conf) ───────────────────────────────────────────
# HOST*_DAILY_SYNC_SHARES — shares pushed to mirror each day
# HOST*_PERSONAL_SHARES — encrypted personal shares
# DAILY_MAINTENANCE_SCRIPTS — maintenance jobs (permissions, cleanup, restart)
# DAILY_RSYNC_ENABLED — enable/disable rsync section
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
# daily_sync_maintenance.sh — normal run
# daily_sync_maintenance.sh --dry-run — preview without syncing or changing
# daily_sync_maintenance.sh --log — verbose per-share/per-job output
# daily_sync_maintenance.sh --status — show configured shares and jobs
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../Master.conf"
source "$SCRIPT_DIR/../common.sh"
source "$SCRIPT_DIR/../load_config.sh"
RSYNC_SCRIPT="$SCRIPT_DIR/../Rsync/rsync.sh"
SCRIPTS_ROOT="$SCRIPT_DIR/.."
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Setup ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_GEAR Setup ━━━"
parse_args "$@"
# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
error "Must be run as root"
exit 1
fi
validate_unraid_cmd \
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
"" "" \
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
detect_hosts
resolve_remote_ip
acquire_lock
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SHIELD Pre-flight Checks ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_SHIELD Pre-flight Checks ━━━"
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
check_connectivity
check_remote_rootfs
# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
echo ""
echo "━━━━━ $ICON_SUMMARY DAILY SYNC STATUS ━━━━━"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_NET Remote: $REMOTE_ID ($REMOTE_SERVER_NAME)"
echo "$ICON_SYNC Rsync enabled: ${RSYNC_ENABLED:-true}"
echo "$ICON_SYNC Daily enabled: ${DAILY_RSYNC_ENABLED:-false}"
echo ""
echo "━━━ Daily Sync Shares ━━━"
for share in "${DAILY_SYNC_SHARES[@]:-}"; do
[[ -n "$share" ]] && echo " $ICON_SYNC $share"
done
for share in "${PERSONAL_SHARES[@]:-}"; do
[[ -n "$share" ]] && echo " $ICON_SYNC $share (personal)"
done
echo ""
echo "━━━ Daily Maintenance Scripts ━━━"
for entry in "${DAILY_MAINTENANCE_SCRIPTS[@]:-}"; do
[[ -z "$entry" ]] && continue
echo " $ICON_GEAR $(basename "${entry%% *}") ${entry#* }"
done
echo "━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
fi
WINDOW_START=$(date +%s)
JOB_PASS=()
JOB_FAIL=()
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GIT Pre-sync Jobs ━━━
# git_pull_execute.sh runs first — pulls latest scripts before anything else runs
# Identified by script name — all other DAILY_MAINTENANCE_SCRIPTS run after sync
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_SYNC Pre-sync Jobs ━━━"
# ==============================================================================================
# ── BUILD SHARE LIST via detect_hosts aliases ─────────────────────────────────────────────────
# detect_hosts() sets DAILY_SYNC_SHARES and PERSONAL_SHARES from HOST*_ vars
# No manual HOST1/HOST2 comparison needed — aliased automatically per server
# ==============================================================================================
ALL_SHARES=()
for share in "${DAILY_SYNC_SHARES[@]:-}"; do
[[ -n "$share" ]] && ALL_SHARES+=("$share")
done
for share in "${PERSONAL_SHARES[@]:-}"; do
[[ -n "$share" ]] && ALL_SHARES+=("$share")
done
SHARE_COUNT=${#ALL_SHARES[@]}
# ── Split maintenance scripts: git pull runs pre-sync, rest run post-sync ─────────────────────
PRE_SYNC_SCRIPTS=()
POST_SYNC_SCRIPTS=()
for script_entry in "${DAILY_MAINTENANCE_SCRIPTS[@]}"; do
for script_entry in "${DAILY_MAINTENANCE_SCRIPTS[@]:-}"; do
[[ -z "$script_entry" ]] && continue
script_name=$(basename "${script_entry%% *}")
if [[ "$script_name" == "git_pull_execute.sh" ]]; then
@@ -85,236 +143,181 @@ for script_entry in "${DAILY_MAINTENANCE_SCRIPTS[@]}"; do
fi
done
for script_entry in "${PRE_SYNC_SCRIPTS[@]}"; do
[[ -z "$script_entry" ]] && continue
script_args=($script_entry)
script_path="$SCRIPTS_ROOT/${script_args[0]}"
script_name=$(basename "${script_args[0]}")
extra_args=("${script_args[@]:1}")
# Helper — run a maintenance script, track pass/fail
run_job() {
local script_entry="$1"
local extra_dry=""
[[ "$DRY_RUN" == true ]] && extra_dry="--dry-run"
echo ""
info "$ICON_START Running: $script_name"
read -r -a script_args <<< "$script_entry"
local script_path="$SCRIPTS_ROOT/${script_args[0]}"
local script_name
script_name=$(basename "${script_args[0]}")
local extra_args=("${script_args[@]:1}")
if [[ ! -f "$script_path" ]]; then
error "$script_name — not found at $script_path"
JOB_FAIL+=("$script_name")
continue
return 1
fi
if bash "$script_path" "${extra_args[@]}"; then
success "$script_name — done"
JOB_PASS+=("$script_name")
log "Running: $script_name ${extra_args[*]}"
# shellcheck disable=SC2086
if bash "$script_path" "${extra_args[@]}" $extra_dry; then
log "$script_name — done ✅"
JOB_PASS+=("$script_name ${extra_args[*]}")
else
error "$script_name — failed"
JOB_FAIL+=("$script_name")
error "$script_name — failed (exit $?)"
JOB_FAIL+=("$script_name ${extra_args[*]}")
fi
done
}
# -----------------------------------------------------------------------------------------------
# Build share list — host-specific truth shares + personal shares
# -----------------------------------------------------------------------------------------------
WINDOW_START=$(date +%s)
JOB_PASS=()
JOB_FAIL=()
PASS=()
FAIL=()
SHARE_TIMES=()
TOTAL_START=$(date +%s)
ALL_SHARES=()
echo ""
echo "━━━ $ICON_GEAR Daily Maintenance — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then
for share in "${HOST1_DAILY_SYNC_SHARES[@]}"; do
[[ -n "$share" ]] && ALL_SHARES+=("$share")
done
for share in "${HOST1_PERSONAL_SHARES[@]}"; do
[[ -n "$share" ]] && ALL_SHARES+=("$share")
done
elif [[ "$LOCAL_SERVER_NAME" == "$HOST2" ]]; then
for share in "${HOST2_DAILY_SYNC_SHARES[@]}"; do
[[ -n "$share" ]] && ALL_SHARES+=("$share")
done
for share in "${HOST2_PERSONAL_SHARES[@]}"; do
[[ -n "$share" ]] && ALL_SHARES+=("$share")
# ==============================================================================================
# ━━━ Pre-sync — git pull ━━━
# ==============================================================================================
if [[ ${#PRE_SYNC_SCRIPTS[@]} -gt 0 ]]; then
echo ""
echo "━━━ $ICON_GIT Pre-sync ━━━"
for script_entry in "${PRE_SYNC_SCRIPTS[@]}"; do
run_job "$script_entry"
done
fi
SHARE_COUNT=${#ALL_SHARES[@]}
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SYNC Media Share Sync ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_SYNC Media Share Sync — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
echo "$ICON_SUMMARY Shares: $SHARE_COUNT"
# ==============================================================================================
# ━━━ Media Share Sync ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_SYNC Media Share Sync — $SHARE_COUNT share(s) ━━━"
TOTAL_START=$(date +%s)
SHARE_INDEX=0
ABORT_ALL_SYNCS=false
# Tier 1 + Tier 2 rsync gate check
if ! check_rsync_enabled "DAILY"; then
warn "Rsync disabled — skipping all $SHARE_COUNT share syncs"
warn "Proceeding to media management jobs..."
warn "Daily rsync disabled — skipping all $SHARE_COUNT share syncs"
warn "Proceeding to maintenance jobs..."
elif [[ "$SHARE_COUNT" -eq 0 ]]; then
warn "No shares configured for $MY_ID — check HOST*_DAILY_SYNC_SHARES in master_host*.conf"
else
# Pre-flight — connectivity then remote rootfs
check_connectivity
check_remote_rootfs
RSYNC_DRY=""
[[ "$DRY_RUN" == true ]] && RSYNC_DRY="--dry-run"
for SHARE in "${ALL_SHARES[@]}"; do
SHARE_INDEX=$((SHARE_INDEX + 1))
SHARE_NAME=$(basename "$SHARE")
SHARE_START=$(date +%s)
echo "━━━ $ICON_SYNC Share $SHARE_INDEX of $SHARE_COUNT: $SHARE_NAME ━━━"
if [[ "$ABORT_ALL_SYNCS" == true ]]; then
warn "$SHARE_NAME — skipped (drive temps CRITICAL earlier in window)"
FAIL+=("$SHARE_NAME:temp-critical")
echo ""
continue
fi
bash "$RSYNC_SCRIPT" "$SHARE"
RSYNC_EXIT=$?
SHARE_END=$(date +%s)
SHARE_TIMES+=("$SHARE_NAME:$((SHARE_END - SHARE_START))")
if [[ "$RSYNC_EXIT" -eq 0 ]]; then
PASS+=("$SHARE_NAME")
echo "$ICON_DONE $SHARE_NAME complete"
elif [[ "$RSYNC_EXIT" -eq 1 ]]; then
# Temp warning — skip this profile, continue to next
FAIL+=("$SHARE_NAME:temp-warn")
warn "$SHARE_NAME skipped — drive temps too high"
elif [[ "$RSYNC_EXIT" -eq 2 ]]; then
# Temp critical — abort all remaining syncs
FAIL+=("$SHARE_NAME:temp-critical")
ABORT_ALL_SYNCS=true
error "$SHARE_NAME aborted — drive temps CRITICAL, stopping all remaining syncs"
notify "Daily sync aborted on $(hostname) — drive temps CRITICAL during $SHARE_NAME sync" "Daily Sync" "warning"
else
FAIL+=("$SHARE_NAME")
error "$SHARE_NAME failed — continuing to next share"
fi
echo ""
done
fi # end check_rsync_enabled "DAILY"
TOTAL_END=$(date +%s)
TOTAL_DURATION=$((TOTAL_END - TOTAL_START))
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_CLEAN Post-sync Media Jobs ━━━
# Reads DAILY_MAINTENANCE_SCRIPTS from Master.conf — permissions, cleaners, arr cleanup
# Runs after sync completes — correct ownership available, clean folders guaranteed
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_CLEAN Post-sync Media Jobs ━━━"
if [[ ${#DAILY_MAINTENANCE_SCRIPTS[@]} -gt 0 ]]; then
for script_entry in "${DAILY_MAINTENANCE_SCRIPTS[@]}"; do
[[ -z "$script_entry" ]] && continue
script_args=($script_entry)
script_path="$SCRIPTS_ROOT/${script_args[0]}"
script_name=$(basename "${script_args[0]}")
extra_args=("${script_args[@]:1}")
(( SHARE_INDEX++ ))
SHARE_NAME=$(basename "$SHARE")
SHARE_START=$(date +%s)
echo ""
info "$ICON_START Running: $script_name ${extra_args[*]}"
echo "━━━ $ICON_SYNC Share $SHARE_INDEX/$SHARE_COUNT: $SHARE_NAME ━━━"
if [[ ! -f "$script_path" ]]; then
error "$script_name — not found at $script_path"
JOB_FAIL+=("$script_name ${extra_args[*]}")
if [[ "$ABORT_ALL_SYNCS" == true ]]; then
warn "$SHARE_NAME — skipped (drive temps CRITICAL earlier in window)"
FAIL+=("$SHARE_NAME:temp-critical")
continue
fi
if [[ "$DRY_RUN" == true ]]; then
if bash "$script_path" "${extra_args[@]}" --dry-run; then
success "$script_name — done (dry run)"
JOB_PASS+=("$script_name ${extra_args[*]}")
else
error "$script_name — failed"
JOB_FAIL+=("$script_name ${extra_args[*]}")
fi
else
if bash "$script_path" "${extra_args[@]}"; then
success "$script_name — done"
JOB_PASS+=("$script_name ${extra_args[*]}")
else
error "$script_name — failed"
JOB_FAIL+=("$script_name ${extra_args[*]}")
fi
fi
bash "$RSYNC_SCRIPT" "$SHARE" $RSYNC_DRY
RSYNC_EXIT=$?
SHARE_TIMES+=("$SHARE_NAME:$(( $(date +%s) - SHARE_START ))")
case "$RSYNC_EXIT" in
0)
PASS+=("$SHARE_NAME")
log "$SHARE_NAME — done ✅"
;;
1)
FAIL+=("$SHARE_NAME:temp-warn")
warn "$SHARE_NAME skipped — drive temps too high"
;;
2)
FAIL+=("$SHARE_NAME:temp-critical")
ABORT_ALL_SYNCS=true
error "$SHARE_NAME aborted — drive temps CRITICAL, stopping all remaining syncs"
notify "Daily sync aborted on $(hostname) ($MY_ID) — drive temps CRITICAL during $SHARE_NAME" \
"Daily Sync" "warning"
;;
*)
FAIL+=("$SHARE_NAME")
error "$SHARE_NAME failed (exit $RSYNC_EXIT) — continuing to next share"
;;
esac
done
fi
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Post-sync System Jobs ━━━
# Reads remaining DAILY_MAINTENANCE_SCRIPTS — docker restart etc.
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_GEAR Post-sync Jobs ━━━"
for script_entry in "${POST_SYNC_SCRIPTS[@]}"; do
[[ -z "$script_entry" ]] && continue
script_args=($script_entry)
script_path="$SCRIPTS_ROOT/${script_args[0]}"
script_name=$(basename "${script_args[0]}")
extra_args=("${script_args[@]:1}")
TOTAL_END=$(date +%s)
# ==============================================================================================
# ━━━ Post-sync Maintenance Jobs ━━━
# ==============================================================================================
if [[ ${#POST_SYNC_SCRIPTS[@]} -gt 0 ]]; then
echo ""
info "$ICON_START Running: $script_name"
if [[ ! -f "$script_path" ]]; then
error "$script_name — not found at $script_path"
JOB_FAIL+=("$script_name")
continue
fi
if bash "$script_path" "${extra_args[@]}"; then
success "$script_name — done"
JOB_PASS+=("$script_name")
else
error "$script_name — failed"
JOB_FAIL+=("$script_name")
fi
done
echo "━━━ $ICON_CLEAN Post-sync Maintenance ━━━"
for script_entry in "${POST_SYNC_SCRIPTS[@]}"; do
echo ""
run_job "$script_entry"
done
fi
WINDOW_END=$(date +%s)
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Summary ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
echo ""
echo "━━━━━ $ICON_SUMMARY DAILY SYNC MAINTENANCE SUMMARY ━━━━━"
echo "$ICON_TIME Window: $(date -d @$WINDOW_START '+%Y-%m-%d %H:%M:%S') → $(date -d @$WINDOW_END '+%H:%M:%S')"
echo "$ICON_TIME Duration: $(format_duration $(( WINDOW_END - WINDOW_START )))"
echo "━━━━━ $ICON_SUMMARY DAILY MAINTENANCE SUMMARY ━━━━━"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_TIME Window: $(date -d @"$WINDOW_START" '+%Y-%m-%d %H:%M:%S') → $(date -d @"$WINDOW_END" '+%H:%M:%S')"
echo "$ICON_TIME Duration: $(format_duration $(( WINDOW_END - WINDOW_START )))"
echo ""
echo "$ICON_SYNC Media shares:"
echo "$ICON_SYNC Shares ($SHARE_COUNT):"
for entry in "${SHARE_TIMES[@]}"; do
SHARE_NAME="${entry%%:*}"
DURATION="${entry##*:}"
if printf '%s\n' "${FAIL[@]}" | grep -qx "$SHARE_NAME"; then
echo " $ICON_ERROR $SHARE_NAME — $(format_duration $DURATION)"
sname="${entry%%:*}"
sdur="${entry##*:}"
if printf '%s\n' "${FAIL[@]}" | grep -q "^${sname}"; then
echo " $ICON_ERROR $sname — $(format_duration "$sdur")"
else
echo " $ICON_DONE $SHARE_NAME — $(format_duration $DURATION)"
echo " $ICON_DONE $sname — $(format_duration "$sdur")"
fi
done
echo " Passed: ${#PASS[@]}/$SHARE_COUNT Failed: ${#FAIL[@]}/$SHARE_COUNT"
[[ "$SHARE_COUNT" -eq 0 || "${DAILY_RSYNC_ENABLED:-false}" == "false" ]] && \
echo " (rsync disabled)"
echo " Passed: ${#PASS[@]}/$SHARE_COUNT Failed: ${#FAIL[@]}/$SHARE_COUNT"
echo ""
if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then
echo "$ICON_GEAR Jobs (media + system):"
for job in "${JOB_PASS[@]}"; do echo " $ICON_DONE $job"; done
for job in "${JOB_FAIL[@]}"; do echo " $ICON_ERROR $job"; done
echo "$ICON_GEAR Jobs:"
for job in "${JOB_PASS[@]}"; do echo " $ICON_DONE $job"; done
for job in "${JOB_FAIL[@]}"; do echo " $ICON_ERROR $job"; done
echo ""
fi
TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} ))
if [[ "$TOTAL_FAIL" -gt 0 ]]; then
echo "$ICON_WARN Status: $TOTAL_FAIL failure(s) — check logs"
notify "Daily sync maintenance completed with failures on $(hostname) — shares: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" "Daily Maintenance" "warning"
warn "Status: $TOTAL_FAIL failure(s)"
notify "Daily maintenance completed with failures on $(hostname) ($MY_ID) — shares: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \
"Daily Maintenance" "warning"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
exit 1
else
echo "$ICON_DONE Status: $ICON_SUCCESS ALL COMPLETE"
notify "Daily sync maintenance complete on $(hostname) — ${#PASS[@]} shares synced, ${#JOB_PASS[@]} jobs run in $(format_duration $(( WINDOW_END - WINDOW_START )))" "Daily Maintenance" "normal"
log "$ICON_DONE Status: all complete — ${#PASS[@]} share(s) synced, ${#JOB_PASS[@]} job(s) run"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
fi
Orchestrators/sunday_morning_coffee_report.sh
+434 -511
View File
File diff suppressed because it is too large Load Diff
Orchestrators/transcode_management.sh
+121 -150
View File
@@ -1,132 +1,127 @@
#!/bin/bash
# -----------------------------------------------------------------------------------------------
# --------------------------------- Transcode Management ---------------------------------------
# -----------------------------------------------------------------------------------------------
# Runs transcode cleanup then transcode manager in the correct order every cycle.
# Cleanup runs first — clears stale files so manager sees accurate usage.
# Manager runs after — threshold decisions based on real current usage post-cleanup.
#
# Running cleanup before manager prevents unnecessary SSD flips caused by stale
# segment files from ended sessions inflating the ramdisk usage reading.
#
# Also tracks daily transcode statistics to a bounded log for weekly_health_digest.sh:
# Peak ramdisk usage per day
# Total flip count per day
# Ramdisk vs SSD session ratio
# Files cleaned per day
#
# Scheduled as: */3 * * * * (every 3 minutes)
# ==============================================================================================
# ============================= Transcode Management ===========================================
# ==============================================================================================
# Orchestrator — runs transcode_cleanup.sh then transcode_manager.sh in the correct order.
# Replace individual transcode_manager and transcode_cleanup cron entries with this.
# Schedule: */3 * * * * (every 3 minutes via User Scripts plugin)
#
# All configuration in Master.conf under Transcode Manager section.
# Supports --dry-run — passes through to both child scripts.
# -----------------------------------------------------------------------------------------------
# ── WHY CLEANUP BEFORE MANAGER ────────────────────────────────────────────────────────────────
# Cleanup runs first — removes stale segment files from ended sessions.
# Manager runs after — threshold decisions based on real current usage post-cleanup.
# Without this order, stale files inflate the ramdisk usage reading and trigger
# unnecessary SSD flips even when active sessions would fit on the ramdisk.
#
# ── WHAT EACH SCRIPT DOES ─────────────────────────────────────────────────────────────────────
# transcode_cleanup.sh — removes aged segment files not open by any process
# uses lsof for O(1) per-file active check (never per-file lsof)
# also triggers flip-back to ramdisk after cleanup if recovered ✅
#
# transcode_manager.sh — checks ramdisk usage against thresholds
# flips symlink between ramdisk and SSD as needed
# writes one entry to TRANSCODE_DAILY_LOG after each run
# shows active Emby sessions with play method
#
# ── DAILY LOG ─────────────────────────────────────────────────────────────────────────────────
# transcode_manager.sh writes to TRANSCODE_DAILY_LOG after each run:
# Format: DATE|RAMDISK_USED_GB|FLIP_COUNT|RAM_SESSIONS|SSD_SESSIONS
# This orchestrator does NOT write its own log — manager handles it ✅
# Log trimmed to TRANSCODE_LOG_RETENTION days by manager on each write.
# Read by sunday_morning_coffee_report.sh and weekly_health_digest.sh.
#
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
# detect_hosts() sets MY_ID and aliases RAMDISK_PATH, TRANSCODE_SSD, RAMDISK_WARN_GB etc.
# Each server manages its own transcode location independently.
#
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
# Root check — mount and docker operations require root
# acquire_lock — prevents concurrent 3-minute cycles overlapping
# detect_hosts() — correct paths per host
# --dry-run — passed through to both child scripts
# Exit code — worst exit code of both scripts returned
#
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
# TRANSCODE_DAILY_LOG — daily stats log (written by transcode_manager.sh)
# TRANSCODE_LOG_RETENTION — days to keep (trimmed by manager)
# TRANSCODE_STATE_FILE — current state (ramdisk_used, flip_count etc.)
# All TRANSCODE_* threshold vars — see master.conf Transcode Manager section
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
# transcode_management.sh — normal run (every 3 minutes via cron)
# transcode_management.sh --dry-run — preview without changes (passed to children)
# transcode_management.sh --status — show configuration and current state
# transcode_management.sh --log — verbose output from both child scripts
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../Master.conf"
source "$SCRIPT_DIR/../common.sh"
source "$SCRIPT_DIR/../load_config.sh"
parse_args "$@"
# -----------------------------------------------------------------------------------------------
# State and log files
# -----------------------------------------------------------------------------------------------
TRANSCODE_DAILY_LOG="/boot/config/transcode_daily.db"
TRANSCODE_STATE_FILE="/tmp/transcode_state.db"
CLEANUP_SCRIPT="$SCRIPT_DIR/../Transcodes/transcode_cleanup.sh"
MANAGER_SCRIPT="$SCRIPT_DIR/../Transcodes/transcode_manager.sh"
# Bounded log — keeps last 90 days
TRANSCODE_LOG_RETENTION=90
touch "$TRANSCODE_DAILY_LOG" 2>/dev/null
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Setup ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
error "Must be run as root"
exit 1
fi
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — passing to child scripts"
acquire_lock "wait"
acquire_lock
# detect_hosts() sets MY_ID and aliases all HOST*_TRANSCODE_* vars
detect_hosts
# -----------------------------------------------------------------------------------------------
# HELPERS
# -----------------------------------------------------------------------------------------------
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — passing through to child scripts"
# Read a value from transcode state file
state_get() {
grep "^${1}=" "$TRANSCODE_STATE_FILE" 2>/dev/null | cut -d= -f2
}
# Get today's date key
today() {
date '+%Y-%m-%d'
}
# Update daily log entry for today
# Format: YYYY-MM-DD|peak_gb|flip_count|ram_sessions|ssd_sessions|files_cleaned
update_daily_log() {
local peak_gb="$1"
local flips="$2"
local ram_sessions="$3"
local ssd_sessions="$4"
local files_cleaned="$5"
local today_key
today_key=$(today)
local existing
existing=$(grep "^${today_key}|" "$TRANSCODE_DAILY_LOG" 2>/dev/null)
if [[ -z "$existing" ]]; then
# New entry for today
echo "${today_key}|${peak_gb}|${flips}|${ram_sessions}|${ssd_sessions}|${files_cleaned}" \
>> "$TRANSCODE_DAILY_LOG"
# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
echo ""
echo "━━━━━ $ICON_SUMMARY TRANSCODE MANAGEMENT STATUS ━━━━━"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_RAM Ramdisk: $RAMDISK_PATH ($RAMDISK_SIZE)"
echo "$ICON_DISK SSD fallback: $TRANSCODE_SSD"
echo "$ICON_LINK Symlink: $TRANSCODE_LINK"
echo "$ICON_TIME Schedule: every 3 minutes"
echo ""
echo "━━━ Child Scripts ━━━"
[[ -f "$CLEANUP_SCRIPT" ]] && \
echo " $ICON_SUCCESS transcode_cleanup.sh — found" || \
echo " $ICON_ERROR transcode_cleanup.sh — NOT FOUND at $CLEANUP_SCRIPT"
[[ -f "$MANAGER_SCRIPT" ]] && \
echo " $ICON_SUCCESS transcode_manager.sh — found" || \
echo " $ICON_ERROR transcode_manager.sh — NOT FOUND at $MANAGER_SCRIPT"
echo ""
echo "━━━ Daily Log ━━━"
if [[ -f "${TRANSCODE_DAILY_LOG:-}" ]] && [[ -s "$TRANSCODE_DAILY_LOG" ]]; then
ENTRY_COUNT=$(wc -l < "$TRANSCODE_DAILY_LOG")
OLDEST=$(awk -F'|' 'NR==1{print $1}' "$TRANSCODE_DAILY_LOG")
NEWEST=$(awk -F'|' 'END{print $1}' "$TRANSCODE_DAILY_LOG")
echo " $ICON_SUCCESS $TRANSCODE_DAILY_LOG ($ENTRY_COUNT entries, $OLDEST → $NEWEST)"
else
# Update existing — keep highest peak, accumulate flips, sessions, files
local old_peak old_flips old_ram old_ssd old_files
old_peak=$(echo "$existing" | cut -d'|' -f2)
old_flips=$(echo "$existing" | cut -d'|' -f3)
old_ram=$(echo "$existing" | cut -d'|' -f4)
old_ssd=$(echo "$existing" | cut -d'|' -f5)
old_files=$(echo "$existing" | cut -d'|' -f6)
# Peak — keep highest
local new_peak
new_peak=$(awk "BEGIN {print ($peak_gb > $old_peak) ? $peak_gb : $old_peak}")
# Accumulate
local new_flips=$(( old_flips + flips ))
local new_ram=$(( old_ram + ram_sessions ))
local new_ssd=$(( old_ssd + ssd_sessions ))
local new_files=$(( old_files + files_cleaned ))
# Replace line
sed -i "s|^${today_key}|.*|${today_key}|${new_peak}|${new_flips}|${new_ram}|${new_ssd}|${new_files}|" \
"$TRANSCODE_DAILY_LOG" 2>/dev/null || {
# sed replacement failed — remove and re-add
sed -i "/^${today_key}|/d" "$TRANSCODE_DAILY_LOG"
echo "${today_key}|${new_peak}|${new_flips}|${new_ram}|${new_ssd}|${new_files}" \
>> "$TRANSCODE_DAILY_LOG"
}
echo " $ICON_SKIP $TRANSCODE_DAILY_LOG — no data yet"
fi
echo ""
echo "━━━ Current State ━━━"
if [[ -f "${TRANSCODE_STATE_FILE:-/tmp/transcode_state.db}" ]]; then
while IFS='=' read -r key val; do
[[ -n "$key" ]] && echo " $key = $val"
done < "${TRANSCODE_STATE_FILE:-/tmp/transcode_state.db}"
else
echo " State file not found (ramdisk_setup.sh creates it at array start)"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
fi
# Purge old entries beyond retention
local cutoff
cutoff=$(date -d "${TRANSCODE_LOG_RETENTION} days ago" '+%Y-%m-%d')
awk -F'|' -v cutoff="$cutoff" '$1 >= cutoff' \
"$TRANSCODE_DAILY_LOG" > "${TRANSCODE_DAILY_LOG}.tmp" && \
mv "${TRANSCODE_DAILY_LOG}.tmp" "$TRANSCODE_DAILY_LOG"
}
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SYNC Run Cleanup ━━━
# -----------------------------------------------------------------------------------------------
CLEANUP_SCRIPT="$SCRIPT_DIR/../Transcodes/transcode_cleanup.sh"
MANAGER_SCRIPT="$SCRIPT_DIR/../Transcodes/transcode_manager.sh"
# ==============================================================================================
# ━━━ Validate Child Scripts ━━━
# ==============================================================================================
if [[ ! -f "$CLEANUP_SCRIPT" ]]; then
error "transcode_cleanup.sh not found: $CLEANUP_SCRIPT"
exit 1
@@ -137,50 +132,26 @@ if [[ ! -f "$MANAGER_SCRIPT" ]]; then
exit 1
fi
# Capture cleanup output for file count
CLEANUP_OUTPUT=$(bash "$CLEANUP_SCRIPT" $([[ "$DRY_RUN" == true ]] && echo "--dry-run") 2>&1)
# ==============================================================================================
# ━━━ Run Cleanup ━━━
# ==============================================================================================
DRY_FLAG=""
[[ "$DRY_RUN" == true ]] && DRY_FLAG="--dry-run"
bash "$CLEANUP_SCRIPT" $DRY_FLAG
CLEANUP_EXIT=$?
echo "$CLEANUP_OUTPUT"
# Extract files cleaned from cleanup output
FILES_CLEANED=$(echo "$CLEANUP_OUTPUT" | grep -oE "Removed [0-9]+ file" | grep -oE "[0-9]+" | head -1)
FILES_CLEANED="${FILES_CLEANED:-0}"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_WATCHDOG Run Manager ━━━
# -----------------------------------------------------------------------------------------------
MANAGER_OUTPUT=$(bash "$MANAGER_SCRIPT" $([[ "$DRY_RUN" == true ]] && echo "--dry-run") 2>&1)
# ==============================================================================================
# ━━━ Run Manager ━━━
# ==============================================================================================
# transcode_manager.sh writes to TRANSCODE_DAILY_LOG after each run
# No --no-log flag here — manager owns the log write for this cycle ✅
bash "$MANAGER_SCRIPT" $DRY_FLAG
MANAGER_EXIT=$?
echo "$MANAGER_OUTPUT"
# -----------------------------------------------------------------------------------------------
# Collect stats for daily log
# -----------------------------------------------------------------------------------------------
# Ramdisk usage from state file
RAMDISK_USED_GB=$(state_get "ramdisk_used_gb" 2>/dev/null || echo "0")
[[ -z "$RAMDISK_USED_GB" || "$RAMDISK_USED_GB" == "0" ]] && \
RAMDISK_USED_GB=$(df "$RAMDISK_PATH" --output=used 2>/dev/null | tail -1 | \
awk '{printf "%.2f", $1/1048576}' || echo "0")
# Flip count from state file
FLIP_COUNT=$(state_get "flip_count_hour" 2>/dev/null || echo "0")
FLIP_COUNT="${FLIP_COUNT:-0}"
# Session counts from manager output
RAM_SESSIONS=$(echo "$MANAGER_OUTPUT" | grep -oE "ramdisk \([0-9]+\)" | grep -oE "[0-9]+" | head -1)
SSD_SESSIONS=$(echo "$MANAGER_OUTPUT" | grep -oE "SSD \([0-9]+\)" | grep -oE "[0-9]+" | head -1)
RAM_SESSIONS="${RAM_SESSIONS:-0}"
SSD_SESSIONS="${SSD_SESSIONS:-0}"
# Update daily log
if [[ "$DRY_RUN" == false ]]; then
update_daily_log "$RAMDISK_USED_GB" "$FLIP_COUNT" "$RAM_SESSIONS" "$SSD_SESSIONS" "$FILES_CLEANED"
fi
# -----------------------------------------------------------------------------------------------
# Exit with worst exit code
# -----------------------------------------------------------------------------------------------
if [[ "$CLEANUP_EXIT" -ne 0 || "$MANAGER_EXIT" -ne 0 ]]; then
exit 1
fi
# ==============================================================================================
# ━━━ Exit ━━━
# ==============================================================================================
# Return worst exit code — caller knows if either script failed
[[ "$CLEANUP_EXIT" -ne 0 || "$MANAGER_EXIT" -ne 0 ]] && exit 1
exit 0
Orchestrators/weekly_sync_maintenance.sh
+238 -180
View File
@@ -1,99 +1,193 @@
#!/bin/bash
# -----------------------------------------------------------------------------------------------
# ----------------------------- Weekly Sync Maintenance ----------------------------------------
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ============================= Weekly Sync Maintenance ========================================
# ==============================================================================================
# Weekly maintenance window orchestrator — clean sync, container updates, weekly restarts.
# Schedule: 30 2 * * 0 (Sunday 2:30am — fits before 3am network reboot)
# Schedule: 30 2 * * 0 (Sunday 2:30am — before Sunday 7am coffee report)
#
# Execution order:
# 1. Stop local containers — Emby + auth stack stopped locally
# 2. Stop remote containers — Emby + auth stack stopped remotely via SSH
# 3. Pull updates locally — if WEEKLY_SYNC_UPDATES=true
# 4. Pull updates remotely — if WEEKLY_SYNC_UPDATES_REMOTE=true
# 5. rsync Emby — full clean mirror, both instances stopped
# 6. rsync Critical-Data — auth stack clean sync, databases flushed
# 7. Start remote containers — correct order, delayed start respected
# 8. Start local containers — correct order, delayed start respected
# 9. docker_weekly_restart.sh — weekly container restarts
# ── EXECUTION ORDER ───────────────────────────────────────────────────────────────────────────
# 1. Stop local containers — Emby + auth stack stopped locally
# 2. Stop remote containers — Emby + auth stack stopped remotely via SSH
# 3. Pull updates locally — if WEEKLY_SYNC_UPDATES=true (zero extra downtime)
# 4. Pull updates remotely — if WEEKLY_SYNC_UPDATES_REMOTE=true
# 5. rsync WEEKLY_SYNC_SHARES — full clean mirror, containers stopped both sides
# 6. Start remote containers — correct order, delayed start respected
# 7. Start local containers — correct order, delayed start respected
# 8. WEEKLY_MAINTENANCE_SCRIPTS — weekly restarts etc. (docker_weekly_restart.sh)
#
# Synced shares (WEEKLY_SYNC_SHARES in Master.conf):
# /mnt/user/Media_Server/Emby — emby profile — full mirror, cache resets weekly
# /mnt/user/appdata-Failover/Critical-Data — critical-data — auth stack clean state
# ── WHY WEEKLY NOT NIGHTLY FOR EMBY ──────────────────────────────────────────────────────────
# Emby builds a warm image cache on HOST2 throughout the week.
# Syncing nightly resets cache — cold loads every morning for users.
# Weekly sync: cache stays warm 6 days, resets Sunday night while users sleep.
# emby-failover dirty sync covers watch states + library every 15min between weekly syncs.
#
# Why weekly instead of nightly for Emby:
# Emby builds a warm image cache on HOST2 throughout the week
# Syncing nightly resets cache — cold loads every morning for users
# Weekly sync: cache stays warm 6 days, resets Sunday night while users sleep
# emby-failover dirty sync covers watch states + library every 30-60min between syncs
# ── CONTAINER UPDATES ─────────────────────────────────────────────────────────────────────────
# Containers already stopped for sync — updates pull at zero extra downtime.
# Both servers start on identical image versions after the window completes.
# Toggle: WEEKLY_SYNC_UPDATES / WEEKLY_SYNC_UPDATES_REMOTE in master.conf
#
# Container updates during the window:
# Containers already stopped for sync — updates pull at zero extra downtime
# Both servers start on identical image versions after the window completes
# Toggle: WEEKLY_SYNC_UPDATES / WEEKLY_SYNC_UPDATES_REMOTE in Master.conf
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
# detect_hosts() sets MY_ID — used in banner, summary, and notifications.
# WEEKLY_SYNC_SHARES and WEEKLY_MAINTENANCE_SCRIPTS configured in master.conf.
# Same script runs correctly on both servers.
#
# What triggers weekly_health_digest.sh:
# NOT this script — weekly_health_digest.sh runs on its own Saturday schedule
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
# Root check — stop/start containers, rsync require root
# acquire_lock — prevents concurrent weekly windows
# check_connectivity — verifies remote before any remote operations
# check_remote_rootfs — aborts if remote rootfs nearly full
# DOCKER_TIMEOUT — all docker calls protected
# SSH_TIMEOUT — all SSH calls protected
# validate_unraid_cmd — notify validated before use
# Silent on success — runs weekly, only failures warrant notification
#
# Configuration in Master.conf:
# WEEKLY_SYNC_SHARES — shares synced during the maintenance window
# WEEKLY_MAINTENANCE_SCRIPTS — scripts run after sync (docker_weekly_restart)
# WEEKLY_SYNC_UPDATES — toggle container updates on/off
# WEEKLY_SYNC_UPDATES_REMOTE — toggle remote container updates on/off
# -----------------------------------------------------------------------------------------------
# All configuration in Master.conf.
# Supports --dry-run to walk through without stopping containers, syncing, or updating.
# -----------------------------------------------------------------------------------------------
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
# WEEKLY_SYNC_SHARES — shares synced during window
# WEEKLY_MAINTENANCE_SCRIPTS — scripts run after sync
# WEEKLY_SYNC_UPDATES — toggle local container updates
# WEEKLY_SYNC_UPDATES_REMOTE — toggle remote container updates
# WEEKLY_RSYNC_ENABLED — enable/disable rsync section
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
# weekly_sync_maintenance.sh — normal run
# weekly_sync_maintenance.sh --dry-run — preview without stopping containers or syncing
# weekly_sync_maintenance.sh --log — verbose per-share/per-job output
# weekly_sync_maintenance.sh --status — show configuration and exit
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../Master.conf"
source "$SCRIPT_DIR/../common.sh"
source "$SCRIPT_DIR/../load_config.sh"
RSYNC_SCRIPT="$SCRIPT_DIR/../Rsync/rsync.sh"
SCRIPTS_ROOT="$SCRIPT_DIR/.."
parse_args "$@"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Setup ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_GEAR Setup ━━━"
DOCKER_TIMEOUT=30 # container stop/start needs longer than normal
SSH_TIMEOUT=30 # remote pulls can be slow
# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
error "Must be run as root"
exit 1
fi
success "Running as root"
validate_unraid_cmd \
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
"" "" \
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
acquire_lock
detect_hosts
resolve_remote_ip
# Load container list from emby profile — used for update pulls
read -r -a MAINTENANCE_CONTAINERS <<< "${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-}"
WINDOW_START=$(date +%s)
PASS=()
FAIL=()
JOB_PASS=()
JOB_FAIL=()
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SHIELD Pre-flight Checks ━━━
# -----------------------------------------------------------------------------------------------
# Load container lists from profile config
read -r -a MAINTENANCE_CONTAINERS <<< \
"${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-}"
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be stopped, no sync, no updates"
# ── Helper — run a post-sync maintenance script ────────────────────────────────────────────────
run_job() {
local script_entry="$1"
local extra_dry=""
[[ "$DRY_RUN" == true ]] && extra_dry="--dry-run"
read -r -a script_args <<< "$script_entry"
local script_path="$SCRIPTS_ROOT/${script_args[0]}"
local script_name
script_name=$(basename "${script_args[0]}")
local extra_args=("${script_args[@]:1}")
if [[ ! -f "$script_path" ]]; then
error "$script_name — not found at $script_path"
JOB_FAIL+=("$script_name")
return 1
fi
log "Running: $script_name ${extra_args[*]}"
# shellcheck disable=SC2086
if bash "$script_path" "${extra_args[@]}" $extra_dry; then
log "$script_name — done ✅"
JOB_PASS+=("$script_name ${extra_args[*]}")
else
error "$script_name — failed (exit $?)"
JOB_FAIL+=("$script_name ${extra_args[*]}")
fi
}
# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
echo ""
echo "━━━━━ $ICON_SUMMARY WEEKLY SYNC STATUS ━━━━━"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_NET Remote: $REMOTE_ID ($REMOTE_SERVER_NAME)"
echo "$ICON_SYNC Rsync enabled: ${RSYNC_ENABLED:-true}"
echo "$ICON_SYNC Weekly enabled: ${WEEKLY_RSYNC_ENABLED:-false}"
echo "$ICON_GEAR Local updates: ${WEEKLY_SYNC_UPDATES:-false}"
echo "$ICON_GEAR Remote updates: ${WEEKLY_SYNC_UPDATES_REMOTE:-false}"
echo ""
echo "━━━ Weekly Sync Shares ━━━"
if [[ ${#WEEKLY_SYNC_SHARES[@]} -eq 0 ]]; then
warn " No WEEKLY_SYNC_SHARES configured"
else
for share in "${WEEKLY_SYNC_SHARES[@]:-}"; do
[[ -n "$share" ]] && echo " $ICON_SYNC $(basename "$share") ($share)"
done
fi
echo ""
echo "━━━ Weekly Maintenance Scripts ━━━"
if [[ ${#WEEKLY_MAINTENANCE_SCRIPTS[@]} -eq 0 ]]; then
echo " None configured"
else
for entry in "${WEEKLY_MAINTENANCE_SCRIPTS[@]:-}"; do
[[ -n "$entry" ]] && echo " $ICON_GEAR $(basename "${entry%% *}") ${entry#* }"
done
fi
echo ""
echo "━━━ Containers (from profile config) ━━━"
for c in "${MAINTENANCE_CONTAINERS[@]:-}"; do
[[ -n "$c" ]] && echo " $ICON_CONTAINERS $c"
done
echo "━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
fi
# ==============================================================================================
# ━━━ Pre-flight Checks ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_SHIELD Pre-flight Checks ━━━"
echo "━━━ $ICON_GEAR Weekly Sync Maintenance — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
echo ""
echo "━━━ $ICON_SHIELD Pre-flight ━━━"
check_connectivity
check_remote_rootfs
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_STOP $ICON_CONTAINERS Stop Containers ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Stop Containers ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_STOP $ICON_CONTAINERS Stop Containers ━━━"
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — containers will not be stopped"
else
# Load critical-data + emby container list for stops
read -r -a CRITICAL_CONTAINER_NAMES <<< "${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-}"
# Load container lists for stop functions
read -r -a CRITICAL_CONTAINER_NAMES <<< \
"${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-}"
read -r -a DELAYED_CONTAINERS <<< "${PROFILE_DELAYED_CONTAINERS[critical-data]:-}"
CONTAINER_DELAY="${PROFILE_CONTAINER_DELAY[critical-data]:-15}"
@@ -101,123 +195,118 @@ else
stop_containers
fi
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Container Updates ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Container Updates ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_GEAR Container Updates ━━━"
if [[ "$WEEKLY_SYNC_UPDATES" == true ]]; then
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would pull updates for local containers"
for c in "${MAINTENANCE_CONTAINERS[@]}"; do
[[ -z "$c" ]] && continue
warn "DRY RUN — would docker pull: $c"
for c in "${MAINTENANCE_CONTAINERS[@]:-}"; do
[[ -n "$c" ]] && warn "DRY RUN — would pull: $c"
done
else
info "Pulling local container updates..."
for c in "${MAINTENANCE_CONTAINERS[@]}"; do
log "Pulling local container updates..."
for c in "${MAINTENANCE_CONTAINERS[@]:-}"; do
[[ -z "$c" ]] && continue
# Get image name from running or stopped container
IMAGE=$(docker inspect "$c" --format '{{.Config.Image}}' 2>/dev/null)
IMAGE=$(timeout "$DOCKER_TIMEOUT" docker inspect \
"$c" --format '{{.Config.Image}}' 2>/dev/null)
if [[ -z "$IMAGE" ]]; then
log "$c — not found locally, skipping update"
continue
fi
info "Pulling $IMAGE for $c..."
log "Pulling $IMAGE for $c..."
if docker pull "$IMAGE" >/dev/null 2>&1; then
success "$c — image updated"
log "$c — image updated ✅"
else
warn "$c — pull failed, will start on existing image"
fi
done
fi
else
info "WEEKLY_SYNC_UPDATES=false — skipping local updates"
log "WEEKLY_SYNC_UPDATES=false — skipping local updates"
fi
if [[ "$WEEKLY_SYNC_UPDATES_REMOTE" == true ]]; then
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would pull updates on $REMOTE_SERVER_NAME"
else
info "Pulling remote container updates on $REMOTE_SERVER_NAME..."
for c in "${MAINTENANCE_CONTAINERS[@]}"; do
log "Pulling remote container updates on $REMOTE_SERVER_NAME..."
for c in "${MAINTENANCE_CONTAINERS[@]:-}"; do
[[ -z "$c" ]] && continue
IMAGE=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
IMAGE=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
-o ConnectTimeout="$SSH_TIMEOUT" \
root@"$REMOTE_SERVER" \
"docker inspect $c --format '{{.Config.Image}}' 2>/dev/null" 2>/dev/null)
if [[ -z "$IMAGE" ]]; then
log "$c — not found on remote, skipping update"
continue
fi
info "Pulling $IMAGE for $c on $REMOTE_SERVER_NAME..."
if ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" "docker pull $IMAGE" >/dev/null 2>&1; then
success "$c — remote image updated"
log "Pulling $IMAGE for $c on $REMOTE_SERVER_NAME..."
if timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
-o ConnectTimeout="$SSH_TIMEOUT" \
root@"$REMOTE_SERVER" \
"docker pull $IMAGE" >/dev/null 2>&1; then
log "$c — remote image updated ✅"
else
warn "$c — remote pull failed, will start on existing image"
fi
done
fi
else
info "WEEKLY_SYNC_UPDATES_REMOTE=false — skipping remote updates"
log "WEEKLY_SYNC_UPDATES_REMOTE=false — skipping remote updates"
fi
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SYNC Critical Shares Sync ━━━
# -----------------------------------------------------------------------------------------------
PASS=()
FAIL=()
TOTAL_START=$(date +%s)
SYNC_JOBS=("${WEEKLY_SYNC_SHARES[@]}")
SHARE_COUNT=${#SYNC_JOBS[@]}
# ==============================================================================================
# ━━━ Critical Shares Sync ━━━
# ==============================================================================================
SHARE_COUNT=${#WEEKLY_SYNC_SHARES[@]}
echo ""
echo "━━━ $ICON_SYNC Critical Shares Sync — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
echo "$ICON_SUMMARY Jobs: $SHARE_COUNT"
echo ""
echo "━━━ $ICON_SYNC Critical Shares Sync — $SHARE_COUNT share(s) ━━━"
SYNC_START=$(date +%s)
JOB_NUM=0
# Tier 1 + Tier 2 rsync gate check
if ! check_rsync_enabled "WEEKLY"; then
warn "Rsync disabled — skipping all $SHARE_COUNT weekly sync jobs"
warn "Proceeding to container updates and maintenance scripts..."
warn "Weekly rsync disabled — skipping all $SHARE_COUNT sync job(s)"
warn "Proceeding to container start and maintenance scripts..."
elif [[ "$SHARE_COUNT" -eq 0 ]]; then
warn "No WEEKLY_SYNC_SHARES configured — skipping sync"
warn "Check WEEKLY_SYNC_SHARES in master.conf"
else
for JOB in "${SYNC_JOBS[@]}"; do
((JOB_NUM++))
RSYNC_DRY=""
[[ "$DRY_RUN" == true ]] && RSYNC_DRY="--dry-run"
for JOB in "${WEEKLY_SYNC_SHARES[@]}"; do
(( JOB_NUM++ ))
JOB_NAME=$(basename "$JOB")
echo ""
echo "━━━ [$JOB_NUM/$SHARE_COUNT] $JOB_NAME ━━━"
JOB_START=$(date +%s)
JOB_START=$(date +%s)
bash "$RSYNC_SCRIPT" "$JOB" $RSYNC_DRY
EXIT_CODE=$?
JOB_DUR=$(format_duration $(( $(date +%s) - JOB_START )))
# Containers already stopped — rsync profile won't try to stop them again
# Pass --no-container-stop flag would be ideal but profiles handle this naturally
# since containers are already stopped, stop_containers finds nothing running
if [[ "$DRY_RUN" == true ]]; then
bash "$RSYNC_SCRIPT" "$JOB" --dry-run
else
bash "$RSYNC_SCRIPT" "$JOB"
fi
if [[ "$EXIT_CODE" -eq 0 ]]; then
PASS+=("$JOB_NAME")
log "$JOB_NAME — done in $JOB_DUR ✅"
else
FAIL+=("$JOB_NAME")
error "$JOB_NAME — failed after $JOB_DUR (exit $EXIT_CODE)"
fi
echo ""
done
fi
EXIT_CODE=$?
JOB_END=$(date +%s)
JOB_DURATION=$(format_duration $(( JOB_END - JOB_START )))
SYNC_END=$(date +%s)
if [[ "$EXIT_CODE" -eq 0 ]]; then
PASS+=("$JOB_NAME")
success "$JOB_NAME — $ICON_SUCCESS done in $JOB_DURATION"
else
FAIL+=("$JOB_NAME")
error "$JOB_NAME — $ICON_ERROR failed after $JOB_DURATION"
fi
echo ""
done
fi # end check_rsync_enabled "WEEKLY"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_START $ICON_CONTAINERS Start Containers ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Start Containers ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_START $ICON_CONTAINERS Start Containers ━━━"
@@ -228,88 +317,57 @@ else
start_local_containers
fi
TOTAL_END=$(date +%s)
TOTAL_DURATION=$(format_duration $(( TOTAL_END - TOTAL_START )))
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Post-sync Jobs ━━━
# docker_weekly_restart.sh and any other WEEKLY_MAINTENANCE_SCRIPTS run after sync
# -----------------------------------------------------------------------------------------------
JOB_PASS=()
JOB_FAIL=()
SCRIPTS_ROOT="$SCRIPT_DIR/.."
# ==============================================================================================
# ━━━ Post-sync Jobs ━━━
# ==============================================================================================
if [[ ${#WEEKLY_MAINTENANCE_SCRIPTS[@]} -gt 0 ]]; then
echo ""
echo "━━━ $ICON_GEAR Post-sync Jobs ━━━"
for script_entry in "${WEEKLY_MAINTENANCE_SCRIPTS[@]}"; do
for script_entry in "${WEEKLY_MAINTENANCE_SCRIPTS[@]:-}"; do
[[ -z "$script_entry" ]] && continue
script_args=($script_entry)
script_path="$SCRIPTS_ROOT/${script_args[0]}"
script_name=$(basename "${script_args[0]}")
extra_args=("${script_args[@]:1}")
echo ""
info "$ICON_START Running: $script_name"
if [[ ! -f "$script_path" ]]; then
error "$script_name — not found at $script_path"
JOB_FAIL+=("$script_name")
continue
fi
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would run: $script_name"
JOB_PASS+=("$script_name (dry run)")
elif bash "$script_path" "${extra_args[@]}"; then
success "$script_name — done"
JOB_PASS+=("$script_name")
else
error "$script_name — failed"
JOB_FAIL+=("$script_name")
fi
run_job "$script_entry"
done
fi
WINDOW_END=$(date +%s)
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Summary ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
echo ""
echo "━━━━━ $ICON_SUMMARY WEEKLY SYNC MAINTENANCE SUMMARY ━━━━━"
echo "$ICON_TIME Duration: $TOTAL_DURATION"
echo "$ICON_GEAR Updates: local=$WEEKLY_SYNC_UPDATES remote=$WEEKLY_SYNC_UPDATES_REMOTE"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_TIME Window: $(date -d @"$WINDOW_START" '+%Y-%m-%d %H:%M:%S') → $(date -d @"$WINDOW_END" '+%H:%M:%S')"
echo "$ICON_TIME Duration: $(format_duration $(( WINDOW_END - WINDOW_START )))"
echo "$ICON_GEAR Updates: local=${WEEKLY_SYNC_UPDATES:-false} remote=${WEEKLY_SYNC_UPDATES_REMOTE:-false}"
echo ""
echo "$ICON_SYNC Sync jobs:"
if [[ ${#PASS[@]} -gt 0 ]]; then
for job in "${PASS[@]}"; do echo " $ICON_SUCCESS $job"; done
fi
if [[ ${#FAIL[@]} -gt 0 ]]; then
for job in "${FAIL[@]}"; do echo " $ICON_ERROR $job"; done
fi
echo " Passed: ${#PASS[@]} Failed: ${#FAIL[@]}"
echo "$ICON_SYNC Sync jobs ($SHARE_COUNT):"
for job in "${PASS[@]:-}"; do echo " $ICON_DONE $job"; done
for job in "${FAIL[@]:-}"; do echo " $ICON_ERROR $job"; done
echo " Passed: ${#PASS[@]} Failed: ${#FAIL[@]}"
if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then
echo ""
echo "$ICON_GEAR Post-sync jobs:"
for job in "${JOB_PASS[@]}"; do echo " $ICON_SUCCESS $job"; done
for job in "${JOB_FAIL[@]}"; do echo " $ICON_ERROR $job"; done
for job in "${JOB_PASS[@]:-}"; do echo " $ICON_DONE $job"; done
for job in "${JOB_FAIL[@]:-}"; do echo " $ICON_ERROR $job"; done
fi
echo ""
TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} ))
echo ""
if [[ "$DRY_RUN" == true ]]; then
echo "$ICON_WARN Status: DRY RUN — no changes made"
warn "DRY RUN — no changes made"
elif [[ "$TOTAL_FAIL" -eq 0 ]]; then
echo "$ICON_DONE Status: $ICON_SUCCESS ALL COMPLETE"
notify "Weekly sync maintenance complete on $(hostname) — synced + updated (local=$WEEKLY_SYNC_UPDATES remote=$WEEKLY_SYNC_UPDATES_REMOTE)" "Weekly Maintenance" "normal"
log "$ICON_DONE Status: all complete ✅ — ${#PASS[@]} share(s) synced, ${#JOB_PASS[@]} job(s) run"
else
echo "$ICON_ERROR Status: $TOTAL_FAIL failure(s) — check logs"
notify "Weekly sync maintenance failed on $(hostname) — sync: ${#FAIL[@]} failed, jobs: ${#JOB_FAIL[@]} failed" "Weekly Maintenance" "warning"
warn "Status: $TOTAL_FAIL failure(s)"
notify "Weekly maintenance failed on $(hostname) ($MY_ID) — sync: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \
"Weekly Maintenance" "warning"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
[[ "$TOTAL_FAIL" -gt 0 ]] && exit 1
exit 0
Partnership/README-Partnership.md
+854 -347
View File
File diff suppressed because it is too large Load Diff
Partnership/partnership_manager.sh
+310 -268
View File
File diff suppressed because it is too large Load Diff
README-User_Script_Plug-in.md
+1605 -257
View File
File diff suppressed because it is too large Load Diff
README.md
+678 -647
View File
File diff suppressed because it is too large Load Diff
Rsync/README-Rsync_Setup.md
+924 -559
View File
File diff suppressed because it is too large Load Diff
Rsync/rsync.sh
+232 -104
View File
@@ -1,30 +1,71 @@
#!/bin/bash
# ----------------------------------------------------------------------------------------------
# --------------------------------- Rsync Core Script ------------------------------------------
# ----------------------------------------------------------------------------------------------
# ==============================================================================================
# ================================= Rsync Core Script ==========================================
# ==============================================================================================
# Core rsync script — called per share or per appdata profile.
# Called by orchestrators (daily/weekly/critical sync) and directly for manual syncs.
#
# ── PROFILE SYSTEM ────────────────────────────────────────────────────────────────────────────
# Profile is inferred from the directory basename (lowercased).
# If no profile match is found all settings fall through to global defaults in Master.conf.
# Override with --profile=name for explicit profile selection.
# If no profile match found → all settings fall back to global defaults in master.conf.
#
# After each successful sync, logs the transfer to bandwidth_monitor.sh for weekly reporting.
# Log entry: date | time | profile | duration | status
# Profiles define:
# PROFILE_RSYNC_OPTS — rsync flags (does NOT inherit DEFAULT_RSYNC_OPTS)
# PROFILE_BW_LIMIT — bandwidth limit in KB/s
# PROFILE_RETRY_COUNT — retry attempts before giving up
# PROFILE_SLEEP — seconds between retry attempts
# PROFILE_CRITICAL_CONTAINER_NAMES — containers stopped both sides before sync
# PROFILE_DELAYED_CONTAINERS — containers needing delay before starting after sync
# PROFILE_CONTAINER_DELAY — seconds before starting delayed containers
# PROFILE_EXCLUDE_DIRS — paths excluded from transfer
# PROFILE_REMOTE_RESTART_CONTAINERS — containers restarted on remote after dirty sync
# (critical-failover, emby-failover profiles)
# Was running → restart. Was stopped → leave stopped.
#
# Usage:
# rsync.sh /mnt/user/Movies — media share, uses global defaults
# rsync.sh /mnt/user/appdata-Failover/Arrs_Stack — matched to [arrs_stack] profile
# ── RSYNC ENABLE/DISABLE ──────────────────────────────────────────────────────────────────────
# Two-tier toggle system — checked at entry:
# Tier 1: RSYNC_ENABLED=false → all rsync stops
# Tier 2: Per-orchestrator flag (DAILY_RSYNC_ENABLED etc.) — checked by caller
# Direct calls to rsync.sh only check Tier 1
#
# ── BANDWIDTH LOGGING ─────────────────────────────────────────────────────────────────────────
# After each sync logs to bandwidth_monitor.sh --log-transfer:
# profile | duration_seconds | status | bytes_transferred
# Bytes captured from rsync --stats output — version-proof parsing.
# bandwidth_monitor.sh flags syncs exceeding BANDWIDTH_WARN_GB.
#
# ── DIRTY SYNC REMOTE RESTART ─────────────────────────────────────────────────────────────────
# Profiles using dirty sync (critical-failover, emby-failover) define
# PROFILE_REMOTE_RESTART_CONTAINERS — containers restarted on remote after sync completes.
# This ensures the remote picks up config changes synced during the dirty window.
# Was running → restart. Was stopped → leave stopped.
#
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
# check_rsync_enabled() — Tier 1 gate before any operation
# check_unraid_version_parity — refuses if servers on incompatible unRAID versions
# check_remote_docker_daemon — verifies remote daemon before container operations
# check_local_disk_temps() — temp check before transfer (exit 1=skip, 2=abort all)
# check_connectivity() — verifies remote reachable
# check_remote_rootfs() — aborts if remote rootfs nearly full
# check_remote_share() — aborts if target directory missing or empty
# check_remote_disks() — verifies all backing disks online on remote
# acquire_rsync_lock() — per-profile lock + global concurrent limit
# validate_unraid_cmd — notify validated before use
# Silent by default — only failures produce visible output
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
# rsync.sh /mnt/user/Movies — media share, global defaults
# rsync.sh /mnt/user/appdata-Failover/Arrs_Stack — matched to [arrs_stack] profile
# rsync.sh /mnt/user/appdata-Failover/Arrs_Stack --dry-run --log
# ---------------- User Variables, Please adjust in Master.conf as needed ----------------------
# ----------------------------------------------------------------------------------------------
# rsync.sh /mnt/user/appdata-Failover/Critical-Data --profile=critical-failover
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../Master.conf"
source "$SCRIPT_DIR/../common.sh"
source "$SCRIPT_DIR/../load_config.sh"
# -----------------------------------------------------------------------------------------------
# Separate the positional directory argument from flag/key=value args.
# Optional --profile=name overrides the basename profile inference.
# -----------------------------------------------------------------------------------------------
# ── Separate positional directory arg from flags ───────────────────────────────────────────────
DIRECTORY=""
PROFILE_OVERRIDE=""
RAW_ARGS=()
@@ -32,185 +73,272 @@ RAW_ARGS=()
for ARG in "$@"; do
case "$ARG" in
--profile=*) PROFILE_OVERRIDE="${ARG#--profile=}" ;;
--*|*=*) RAW_ARGS+=("$ARG") ;;
*) [[ -z "$DIRECTORY" ]] && DIRECTORY="$ARG" || RAW_ARGS+=("$ARG") ;;
--*|*=*) RAW_ARGS+=("$ARG") ;;
*) [[ -z "$DIRECTORY" ]] && DIRECTORY="$ARG" || RAW_ARGS+=("$ARG") ;;
esac
done
parse_args "${RAW_ARGS[@]}"
[[ -z "$DIRECTORY" ]] && error "No directory specified. Usage: rsync.sh <dir> [--dry-run] [--log] [--profile=name]" && exit 1
[[ -z "$DIRECTORY" ]] && {
error "No directory specified"
error "Usage: rsync.sh <dir> [--dry-run] [--log] [--profile=name]"
exit 1
}
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Setup ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_GEAR Setup ━━━"
# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
error "Must be run as root"
exit 1
fi
validate_unraid_cmd \
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
"" "" \
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
detect_hosts
# Tier 1 global gate — check before doing anything
# Tier 2 (per-orchestrator) is handled by the calling orchestrator
# Direct calls to rsync.sh only check Tier 1
# Tier 1 global gate — Tier 2 (per-orchestrator) checked by caller
if ! check_rsync_enabled; then
warn "RSYNC_ENABLED=false — exiting cleanly"
exit 0
fi
resolve_remote_ip
# -----------------------------------------------------------------------------------------------
# Profile inference — basename of directory lowercased
# Optional --profile=name overrides basename inference
# -----------------------------------------------------------------------------------------------
echo ""
# ── Profile inference ─────────────────────────────────────────────────────────────────────────
if [[ -n "$PROFILE_OVERRIDE" ]]; then
PROFILE_NAME="$PROFILE_OVERRIDE"
info "$ICON_GEAR Profile override: $PROFILE_NAME"
log "Profile override: $PROFILE_NAME"
else
PROFILE_NAME=$(basename "$DIRECTORY" | tr '[:upper:]' '[:lower:]')
info "$ICON_GEAR Loading profile: $PROFILE_NAME"
log "Profile inferred: $PROFILE_NAME"
fi
# Acquire per-profile lock and check global concurrent limit
acquire_rsync_lock "$PROFILE_NAME"
# Scalar overrides
# ── Load profile settings ─────────────────────────────────────────────────────────────────────
BW_LIMIT=${PROFILE_BW_LIMIT[$PROFILE_NAME]:-$BW_LIMIT}
RETRY_COUNT=${PROFILE_RETRY_COUNT[$PROFILE_NAME]:-$RETRY_COUNT}
SLEEP=${PROFILE_SLEEP[$PROFILE_NAME]:-$SLEEP}
CONTAINER_DELAY=${PROFILE_CONTAINER_DELAY[$PROFILE_NAME]:-$CONTAINER_DELAY}
# Array overrides
read -r -a CRITICAL_CONTAINER_NAMES <<< "${PROFILE_CRITICAL_CONTAINER_NAMES[$PROFILE_NAME]:-}"
read -r -a DELAYED_CONTAINERS <<< "${PROFILE_DELAYED_CONTAINERS[$PROFILE_NAME]:-}"
read -r -a EXCLUDE_DIRS <<< "${PROFILE_EXCLUDE_DIRS[$PROFILE_NAME]:-}"
read -r -a REMOTE_RESTART_CONTAINERS <<< "${PROFILE_REMOTE_RESTART_CONTAINERS[$PROFILE_NAME]:-}"
# Local and remote use the same container list — same naming scheme on both servers
# Local containers use same names as remote (mirrored naming scheme)
LOCAL_CRITICAL_CONTAINER_NAMES=("${CRITICAL_CONTAINER_NAMES[@]}")
[[ "$SHOW_STATUS" == true ]] && show_status && exit 0
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SHIELD Pre-flight Checks ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Pre-flight Checks ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_SHIELD Pre-flight Checks ━━━"
echo "━━━ $ICON_SHIELD Pre-flight ━━━"
# Disk temp check — before touching remote or moving any data
# Returns: 0=OK 1=warn(skip this profile) 2=crit(abort all remaining)
# Disk temp — before touching remote or moving data
# Exit 1 = skip this profile | Exit 2 = abort all remaining profiles
check_local_disk_temps
TEMP_RESULT=$?
if [[ "$TEMP_RESULT" -eq 2 ]]; then
error "Drive temps CRITICAL — aborting sync for all remaining profiles"
exit 2 # caller (daily_sync_maintenance.sh) sees exit 2 → stops all syncs
error "Drive temps CRITICAL — aborting all remaining syncs"
exit 2
elif [[ "$TEMP_RESULT" -eq 1 ]]; then
warn "Drive temps too high — skipping profile [$PROFILE_NAME]"
exit 1 # caller sees exit 1 → skips this profile, continues to next
warn "Drive temps high — skipping profile [$PROFILE_NAME]"
exit 1
else
success "Drive temps OK — $TEMP_CHECK_RESULT"
log "Drive temps OK — $TEMP_CHECK_RESULT"
fi
# Version parity — refuse if servers on incompatible unRAID versions
check_unraid_version_parity || exit 1
check_connectivity
check_remote_rootfs
check_remote_share "$DIRECTORY"
check_remote_disks "$DIRECTORY"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_STOP $ICON_CONTAINERS Containers ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_STOP $ICON_CONTAINERS Containers ━━━"
# Remote Docker daemon — check before attempting container operations
if [[ ${#CRITICAL_CONTAINER_NAMES[@]} -gt 0 ]] || [[ ${#REMOTE_RESTART_CONTAINERS[@]} -gt 0 ]]; then
check_remote_docker_daemon || {
warn "Remote Docker daemon unresponsive — skipping container operations"
warn "Proceeding with rsync only — containers will not be stopped or restarted"
CRITICAL_CONTAINER_NAMES=()
LOCAL_CRITICAL_CONTAINER_NAMES=()
REMOTE_RESTART_CONTAINERS=()
}
fi
# Stop local containers first — flush local databases before pushing
stop_local_containers
# ==============================================================================================
# ━━━ Stop Containers ━━━
# ==============================================================================================
if [[ ${#CRITICAL_CONTAINER_NAMES[@]} -gt 0 || ${#LOCAL_CRITICAL_CONTAINER_NAMES[@]} -gt 0 ]]; then
echo ""
echo "━━━ $ICON_STOP $ICON_CONTAINERS Stop Containers ━━━"
# Local first — flush local databases before pushing
stop_local_containers
# Remote next — prevent writes while receiving
stop_containers
fi
# Stop remote containers — prevent writes while receiving
stop_containers
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SYNC Transfer ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Transfer ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_SYNC Transfer ━━━"
echo "$ICON_RUN Source: $DIRECTORY"
echo "$ICON_NET Remote: $REMOTE_SERVER:$DIRECTORY"
echo "$ICON_GEAR Profile: $PROFILE_NAME"
echo "$ICON_RUN Source: $DIRECTORY"
echo "$ICON_NET Remote: $REMOTE_SERVER:$DIRECTORY"
echo "$ICON_GEAR Profile: $PROFILE_NAME"
echo "$ICON_HOST Identity: $MY_ID → $REMOTE_ID"
echo ""
get_rsync_opts
# Append profile excludes
for ex in "${EXCLUDE_DIRS[@]}"; do
for ex in "${EXCLUDE_DIRS[@]:-}"; do
[[ -n "$ex" ]] && RSYNC_OPTS+=(--exclude="$ex")
done
[[ "$DRY_RUN" == true ]] && RSYNC_OPTS+=("--dry-run") && warn "DRY RUN — no changes will be made"
# Add --stats to capture bytes transferred for bandwidth logging
RSYNC_OPTS+=(--stats)
[[ "$DRY_RUN" == true ]] && RSYNC_OPTS+=("--dry-run")
START=$(date +%s)
RSYNC_SUCCESS=false
BYTES_TRANSFERRED=0
ATTEMPT=0
for i in $(seq 1 "$RETRY_COUNT"); do
info "$ICON_RETRY Attempt $i of $RETRY_COUNT..."
for (( ATTEMPT=1; ATTEMPT<=RETRY_COUNT; ATTEMPT++ )); do
log "$ICON_RETRY Attempt $ATTEMPT of $RETRY_COUNT..."
if rsync "${RSYNC_OPTS[@]}" \
RSYNC_OUTPUT=$(rsync "${RSYNC_OPTS[@]}" \
-e "ssh -i $SSH_KEY -T -o Compression=no -o IPQoS=throughput" \
"$DIRECTORY" "root@${REMOTE_SERVER}:$(dirname "$DIRECTORY")/"; then
echo "$ICON_DONE Rsync complete"
"$DIRECTORY" "root@${REMOTE_SERVER}:$(dirname "$DIRECTORY")/" 2>&1)
RSYNC_EXIT=$?
if [[ "$RSYNC_EXIT" -eq 0 ]]; then
# Parse bytes transferred from --stats output
BYTES_TRANSFERRED=$(echo "$RSYNC_OUTPUT" | \
awk '/Total transferred file size:/{gsub(/,/,"",$NF); gsub(/[^0-9]/,"",$NF); print $NF+0}')
BYTES_TRANSFERRED="${BYTES_TRANSFERRED:-0}"
log "$ICON_DONE Rsync complete — $BYTES_TRANSFERRED bytes transferred"
RSYNC_SUCCESS=true
break
else
warn "$ICON_RETRY Rsync failed (attempt $i/$RETRY_COUNT)"
[[ "$i" -lt "$RETRY_COUNT" ]] && info "Retrying in ${SLEEP}s..." && sleep "$SLEEP"
warn "$ICON_RETRY Rsync failed (attempt $ATTEMPT/$RETRY_COUNT)"
log "Exit code: $RSYNC_EXIT"
if [[ "$ATTEMPT" -lt "$RETRY_COUNT" ]]; then
log "Retrying in ${SLEEP}s..."
sleep "$SLEEP"
fi
fi
done
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_START $ICON_CONTAINERS Containers ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_START $ICON_CONTAINERS Containers ━━━"
# ==============================================================================================
# ━━━ Start Containers ━━━
# ==============================================================================================
if [[ ${#CRITICAL_CONTAINER_NAMES[@]} -gt 0 || ${#LOCAL_CRITICAL_CONTAINER_NAMES[@]} -gt 0 ]]; then
echo ""
echo "━━━ $ICON_START $ICON_CONTAINERS Start Containers ━━━"
# Remote first — can be coming up while local restarts
start_containers
# Local next
start_local_containers
fi
# Start remote containers first — they can be coming up while local restarts
start_containers
# ==============================================================================================
# ━━━ Remote Restart (dirty sync profiles) ━━━
# ==============================================================================================
# For dirty sync profiles (critical-failover, emby-failover) — restart containers on remote
# that were running before sync so they pick up config changes from the dirty sync window.
# Was running → restart. Was stopped → leave stopped.
if [[ ${#REMOTE_RESTART_CONTAINERS[@]} -gt 0 ]]; then
echo ""
echo "━━━ $ICON_START $ICON_CONTAINERS Remote Restart (post dirty sync) ━━━"
log "Restarting configured containers on $REMOTE_SERVER_NAME..."
# Start local containers
start_local_containers
for container in "${REMOTE_RESTART_CONTAINERS[@]}"; do
[[ -z "$container" ]] && continue
# Check if container was running before sync (still tracked via RUNNING_CONTAINERS)
WAS_RUNNING=false
for prev in "${RUNNING_CONTAINERS[@]:-}"; do
[[ "$prev" == "$container" ]] && WAS_RUNNING=true && break
done
if [[ "$WAS_RUNNING" == false ]]; then
# Not in stop list — check current remote state
REMOTE_STATUS=$(timeout 15 ssh -i "$SSH_KEY" \
-o ConnectTimeout=10 root@"$REMOTE_SERVER" \
"docker inspect -f '{{.State.Running}}' $container 2>/dev/null" 2>/dev/null)
[[ "$REMOTE_STATUS" != "true" ]] && \
log "$container not running on $REMOTE_SERVER_NAME — skipping remote restart" && \
continue
fi
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would restart $container on $REMOTE_SERVER_NAME"
continue
fi
timeout 15 ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \
"docker restart $container" >/dev/null 2>&1 && \
log "$ICON_STARTED $container restarted on $REMOTE_SERVER_NAME ✅" || \
warn "Failed to restart $container on $REMOTE_SERVER_NAME"
done
fi
END=$(date +%s)
DURATION=$((END - START))
DURATION=$(( END - START ))
# -----------------------------------------------------------------------------------------------
# Log transfer to bandwidth monitor — only on successful non-dry-run syncs
# Reliable format: date|time|profile|duration|status
# Does not parse rsync output — version-proof and always works
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Bandwidth Logging ━━━
# ==============================================================================================
# Logs to bandwidth_monitor.sh — new format includes bytes transferred and warn flag.
# Only logs on actual runs (not dry-run) and only when bandwidth_monitor.sh exists.
BANDWIDTH_MONITOR="$SCRIPT_DIR/../Monitors/bandwidth_monitor.sh"
if [[ "$DRY_RUN" == false ]] && [[ -f "$BANDWIDTH_MONITOR" ]]; then
STATUS="success"
[[ "$RSYNC_SUCCESS" == false ]] && STATUS="failed"
bash "$BANDWIDTH_MONITOR" --log-transfer "$PROFILE_NAME" "$DURATION" "$STATUS"
log "$ICON_BANDWIDTH Transfer logged to bandwidth monitor"
bash "$BANDWIDTH_MONITOR" --log-transfer \
"$PROFILE_NAME" "$DURATION" "$STATUS" "$BYTES_TRANSFERRED"
log "$ICON_BANDWIDTH Transfer logged to bandwidth monitor ($BYTES_TRANSFERRED bytes)"
fi
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Summary ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
echo ""
echo "━━━━━ $ICON_SUMMARY SUMMARY ━━━━━"
echo "$ICON_RUN Directory: $DIRECTORY"
echo "$ICON_GEAR Profile: $PROFILE_NAME"
echo "$ICON_DISK Disk check: $([[ "$SKIP_DISK_CHECK" == "true" ]] && echo "skipped (ZFS pool)" || echo "passed")"
echo "$ICON_TIME Duration: $(format_duration $DURATION)"
echo "━━━━━ $ICON_SUMMARY RSYNC SUMMARY ━━━━━"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_RUN Directory: $DIRECTORY"
echo "$ICON_GEAR Profile: $PROFILE_NAME"
echo "$ICON_TIME Duration: $(format_duration $DURATION)"
[[ "$BYTES_TRANSFERRED" -gt 0 ]] && \
echo "$ICON_BANDWIDTH Transferred: $(awk "BEGIN {printf \"%.2fGB\", $BYTES_TRANSFERRED / 1073741824}")"
if [[ "$RSYNC_SUCCESS" == true ]]; then
echo "$ICON_DONE Status: $ICON_SUCCESS DONE"
notify "Rsync complete — $DIRECTORY ($PROFILE_NAME) in $(format_duration $DURATION)" "Rsync" "normal"
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — no changes made"
elif [[ "$RSYNC_SUCCESS" == true ]]; then
log "$ICON_DONE Status: $ICON_SUCCESS DONE"
else
echo "$ICON_ERROR Status: $ICON_ERROR FAILED after $RETRY_COUNT attempts"
notify "Rsync failed — $DIRECTORY ($PROFILE_NAME) after $RETRY_COUNT attempts" "Rsync" "warning"
echo "$ICON_ERROR Status: FAILED after $RETRY_COUNT attempts"
notify "Rsync FAILED — $DIRECTORY ($PROFILE_NAME) after $RETRY_COUNT attempts on $(hostname)" \
"Rsync" "warning"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━"
[[ "$RSYNC_SUCCESS" == false ]] && exit 1
[[ "$RSYNC_SUCCESS" == false ]] && [[ "$DRY_RUN" == false ]] && exit 1
exit 0
Tools/README-Tools.md
+766 -194
View File
File diff suppressed because it is too large Load Diff
Tools/bulk_permissions_repair.sh
+124 -63
View File
@@ -1,64 +1,93 @@
#!/bin/bash
# -----------------------------------------------------------------------------------------------
# --------------------------------- Bulk Permissions Repair ------------------------------------
# -----------------------------------------------------------------------------------------------
# Applies correct permissions to a single share or specific path.
# Faster than running media_shares_permissions.sh which processes all shares.
# Use when a specific share has wrong ownership or permissions after:
# - A failed transfer that left files owned by wrong user
# - A container writing files as root instead of nobody:users
# - Manual file operations that bypassed normal permission handling
# ==============================================================================================
# ============================= Bulk Permissions Repair ========================================
# ==============================================================================================
# Applies correct ownership and permissions to one or more specific paths.
# Faster than running media_shares_permissions.sh which processes all configured shares.
#
# ── WHEN TO USE ───────────────────────────────────────────────────────────────────────────────
# Use for targeted repair after:
# - A failed transfer that left files owned by wrong user (root:root from rsync)
# - A container writing as root instead of nobody:users — before PUID/PGID was fixed
# - Manual file copies that bypassed normal permission handling
# - A new share that needs permissions applied before the next nightly run
# - A large rsync that imported thousands of files before media_shares_permissions.sh ran
#
# Usage:
# ── PERMISSIONS MODEL ─────────────────────────────────────────────────────────────────────────
# Directories: PERMISSIONS_DIR_MODE (default 755)
# Owner (nobody) — rwx enter, list, create files
# Group (users) — r-x enter and list
# Others — r-x Samba guests can browse
#
# Files: PERMISSIONS_FILE_MODE (default 664)
# Owner (nobody) — rw read + write
# Group (users) — rw arrs can import and rename
# Others — r Samba guests can read
# No execute bit — media files are never executable
#
# ── DIAGNOSTIC — HIGH WRONG OWNER COUNT ───────────────────────────────────────────────────────
# This script counts files with wrong ownership before applying the fix.
# A high count on a share that was recently written → a container has wrong PUID/PGID.
# Fix: add PUID=99 PGID=100 to the container's Docker template.
# Common culprits: SABnzbd, qBittorrent, slskd.
#
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
# Root check — required for chown
# Path existence check — skips missing paths with error
# Separate passes — directories and files chmod'd separately for correctness
# validate_unraid_cmd — notify validated before use
# Silent on success — only failures produce visible output
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
# bulk_permissions_repair.sh /mnt/user/Movies
# bulk_permissions_repair.sh /mnt/user/Movies --dry-run
# bulk_permissions_repair.sh /mnt/user/Movies /mnt/user/Tv_Shows
#
# Uses PERMISSIONS_MODE and PERMISSIONS_OWNER from Master.conf.
# Supports --dry-run to show what would be changed without applying.
# -----------------------------------------------------------------------------------------------
# bulk_permissions_repair.sh /mnt/user/Movies --dry-run
# bulk_permissions_repair.sh /mnt/user/Movies --log
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../Master.conf"
source "$SCRIPT_DIR/../common.sh"
source "$SCRIPT_DIR/../load_config.sh"
parse_args "$@"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Setup ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_GEAR Setup ━━━"
# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
error "Must be run as root"
error "Must be run as root — chown requires root"
exit 1
fi
success "Running as root"
validate_unraid_cmd \
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
"" "" \
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
# detect_hosts() sets MY_ID — used in summary
detect_hosts
if [[ ${#PARSED_ARGS[@]} -eq 0 ]]; then
error "No paths specified"
error "Usage: bulk_permissions_repair.sh /path/to/share [/another/path]"
error " bulk_permissions_repair.sh /mnt/user/Movies --dry-run"
error "Usage: bulk_permissions_repair.sh /path/to/share [/another/path] [--dry-run]"
exit 1
fi
info "Mode: $PERMISSIONS_MODE"
info "Owner: $PERMISSIONS_OWNER"
log "Dir mode: ${PERMISSIONS_DIR_MODE:-755}"
log "File mode: ${PERMISSIONS_FILE_MODE:-664}"
log "Owner: $PERMISSIONS_OWNER"
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no permissions will be changed"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_PERMS Apply Permissions ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Apply Permissions ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_PERMS Permissions Repair ━━━"
echo "━━━ $ICON_PERMS Permissions Repair — $MY_ID ━━━"
START=$(date +%s)
PASS=()
FAIL=()
TOTAL_WRONG_OWNER=0
for share_path in "${PARSED_ARGS[@]}"; do
[[ -z "$share_path" ]] && continue
@@ -72,62 +101,94 @@ for share_path in "${PARSED_ARGS[@]}"; do
continue
fi
# Count files for progress context
# Count files for context — warn level so user knows what they're in for on large shares
FILE_COUNT=$(find "$share_path" -type f 2>/dev/null | wc -l)
DIR_COUNT=$(find "$share_path" -type d 2>/dev/null | wc -l)
DIR_COUNT=$(find "$share_path" -type d 2>/dev/null | wc -l)
SIZE=$(du -sh "$share_path" 2>/dev/null | cut -f1)
warn "$share_path — $FILE_COUNT files, $DIR_COUNT dirs ($SIZE)"
info "$share_path — $FILE_COUNT files, $DIR_COUNT dirs ($SIZE)"
# Count files with wrong ownership before fixing — diagnostic
WRONG_OWNER=$(find "$share_path" \( ! -user nobody -o ! -group users \) \
2>/dev/null | wc -l)
if [[ "$WRONG_OWNER" -gt 0 ]]; then
warn "$WRONG_OWNER file(s) with wrong ownership — fixing..."
if [[ "$WRONG_OWNER" -gt 500 ]]; then
warn "High wrong-owner count — check container PUID/PGID settings (should be PUID=99 PGID=100)"
warn "Common culprits: SABnzbd, qBittorrent, slskd"
fi
TOTAL_WRONG_OWNER=$(( TOTAL_WRONG_OWNER + WRONG_OWNER ))
else
log "Ownership already correct — applying mode only"
fi
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would apply: chmod -R $PERMISSIONS_MODE $share_path"
warn "DRY RUN — would apply: chown -R $PERMISSIONS_OWNER $share_path"
warn "DRY RUN — would apply:"
warn " chown -R $PERMISSIONS_OWNER $share_path"
warn " find -type d → chmod ${PERMISSIONS_DIR_MODE:-755}"
warn " find -type f → chmod ${PERMISSIONS_FILE_MODE:-664}"
PASS+=("$(basename "$share_path")")
continue
fi
# Apply ownership first — chmod after so files are owned correctly before mode change
info "Applying ownership: $PERMISSIONS_OWNER..."
chown -R "$PERMISSIONS_OWNER" "$share_path" 2>/dev/null
CHOWN_EXIT=$?
CHOWN_OK=true
CHMOD_DIR_OK=true
CHMOD_FILE_OK=true
info "Applying permissions: $PERMISSIONS_MODE..."
chmod -R "$PERMISSIONS_MODE" "$share_path" 2>/dev/null
CHMOD_EXIT=$?
# Apply ownership first
log "Applying ownership: $PERMISSIONS_OWNER..."
chown -R "$PERMISSIONS_OWNER" "$share_path" 2>/dev/null || CHOWN_OK=false
if [[ "$CHOWN_EXIT" -eq 0 && "$CHMOD_EXIT" -eq 0 ]]; then
success "$ICON_UNLOCKED $(basename "$share_path") — permissions applied"
# Apply directory permissions — separate pass (dirs need execute bit)
log "Applying directory permissions: ${PERMISSIONS_DIR_MODE:-755}..."
find "$share_path" -type d \
-exec chmod "${PERMISSIONS_DIR_MODE:-755}" {} + 2>/dev/null || CHMOD_DIR_OK=false
# Apply file permissions — no execute bit on media files
log "Applying file permissions: ${PERMISSIONS_FILE_MODE:-664}..."
find "$share_path" -type f \
-exec chmod "${PERMISSIONS_FILE_MODE:-664}" {} + 2>/dev/null || CHMOD_FILE_OK=false
if [[ "$CHOWN_OK" == true && "$CHMOD_DIR_OK" == true && "$CHMOD_FILE_OK" == true ]]; then
log "$ICON_UNLOCKED $(basename "$share_path") — permissions applied ✅"
PASS+=("$(basename "$share_path")")
else
error "$(basename "$share_path") — permission repair failed"
error "$(basename "$share_path") — repair failed"
error " chown: $CHOWN_OK chmod dirs: $CHMOD_DIR_OK chmod files: $CHMOD_FILE_OK"
FAIL+=("$(basename "$share_path")")
fi
done
END=$(date +%s)
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Summary ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
echo ""
echo "━━━━━ $ICON_SUMMARY PERMISSIONS REPAIR SUMMARY ━━━━━"
echo "$ICON_PERMS Mode: $PERMISSIONS_MODE"
echo "$ICON_PERMS Owner: $PERMISSIONS_OWNER"
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_PERMS Dir mode: ${PERMISSIONS_DIR_MODE:-755}"
echo "$ICON_PERMS File mode: ${PERMISSIONS_FILE_MODE:-664}"
echo "$ICON_PERMS Owner: $PERMISSIONS_OWNER"
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
echo ""
echo " $ICON_SUCCESS Pass: ${#PASS[@]} $ICON_ERROR Fail: ${#FAIL[@]}"
echo ""
[[ ${#PASS[@]} -gt 0 ]] && for p in "${PASS[@]}"; do echo " $ICON_UNLOCKED $p"; done
[[ ${#FAIL[@]} -gt 0 ]] && for f in "${FAIL[@]}"; do echo " $ICON_ERROR $f"; done
[[ ${#PASS[@]} -gt 0 ]] && log "Pass: ${PASS[*]}"
[[ ${#FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Fail: ${FAIL[*]}"
if [[ "$TOTAL_WRONG_OWNER" -gt 0 ]]; then
warn "Total wrong-owner files fixed: $TOTAL_WRONG_OWNER"
fi
echo ""
if [[ "$DRY_RUN" == true ]]; then
echo "$ICON_WARN Status: DRY RUN — no changes made"
warn "DRY RUN — no changes made"
elif [[ ${#FAIL[@]} -gt 0 ]]; then
echo "$ICON_ERROR Status: SOME REPAIRS FAILED"
notify "Permissions repair failed on $(hostname) — failed shares: ${FAIL[*]}" "Permissions Repair" "warning"
notify "Permissions repair failed on $(hostname) — ${FAIL[*]}" \
"Permissions Repair" "warning"
else
echo "$ICON_DONE Status: $ICON_SUCCESS DONE"
notify "Permissions repair complete on $(hostname) — ${#PASS[@]} share(s) repaired" "Permissions Repair" "normal"
log "$ICON_DONE Status: done — ${#PASS[@]} path(s) repaired"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
[[ ${#FAIL[@]} -gt 0 ]] && exit 1
exit 0
Tools/container_data_export.sh
+199 -84
View File
@@ -1,50 +1,79 @@
#!/bin/bash
# -----------------------------------------------------------------------------------------------
# --------------------------------- Container Data Export --------------------------------------
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ============================= Container Data Export ==========================================
# ==============================================================================================
# Exports a container's appdata directory to a compressed tar archive.
# Stops the container before archiving and restarts it after — ensures clean consistent backup.
# Verifies the archive after creation — confirms backup is valid before restarting container.
#
# Usage:
# ── WHEN TO USE ───────────────────────────────────────────────────────────────────────────────
# - Before major container updates (roll back if update goes wrong)
# - Before pool migrations or disk replacements
# - When archiving a container being removed from the stack
# - Before destructive operations on appdata (database migrations etc.)
# - One-off backup of a specific container without running full backup
#
# ── OUTPUT FILE NAMING ────────────────────────────────────────────────────────────────────────
# ContainerName_YYYY-MM-DD_HH-MM.tar.gz
# Timestamp in filename — run multiple times safely, no overwrite ✅
#
# ── SPACE CHECK ───────────────────────────────────────────────────────────────────────────────
# Estimates required space as appdata size × 1.1 (10% buffer).
# Compressed archive will typically be much smaller — this is a conservative floor.
# gzip compression ratio depends heavily on content — database files compress well,
# media files do not. If output is on a media share estimate may be pessimistic.
#
# ── ARCHIVE VERIFICATION ──────────────────────────────────────────────────────────────────────
# After creation the archive is tested with tar --test-file before restarting the container.
# If verification fails the container is still restarted (data unchanged) and an error logged.
# A corrupt archive is not a usable backup — do not assume the archive is good without this.
#
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
# DOCKER_TIMEOUT — docker calls protected against hung daemon
# Container restart rule — was running → restart | was stopped → leave stopped ✅
# Archive cleanup — partial archive removed on tar failure
# Archive verification — tar --test-file after creation
# Container restart on — any failure path still restarts container if it was running
# validate_unraid_cmd — notify validated before use
# Silent on success — only problems produce visible output
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
# container_data_export.sh ContainerName /path/to/appdata /path/to/output/dir
# container_data_export.sh Emby /mnt/media-servers/Media_Server/Emby /mnt/user/Backups/
#
# Output file naming:
# ContainerName_YYYY-MM-DD_HH-MM.tar.gz
#
# Use before major container updates, pool migrations, or when archiving
# a container you are removing from the stack.
#
# Supports --dry-run to show what would be archived without making changes.
# -----------------------------------------------------------------------------------------------
# container_data_export.sh Emby /mnt/media-servers/Media_Server/Emby /mnt/user/Backups/ --dry-run
# container_data_export.sh Emby /mnt/media-servers/Media_Server/Emby /mnt/user/Backups/ --log
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../Master.conf"
source "$SCRIPT_DIR/../common.sh"
source "$SCRIPT_DIR/../load_config.sh"
parse_args "$@"
# -----------------------------------------------------------------------------------------------
# Args
# -----------------------------------------------------------------------------------------------
DOCKER_TIMEOUT=30 # longer timeout — stop can take time on large containers
# ── Positional args ───────────────────────────────────────────────────────────────────────────
CONTAINER_NAME="${PARSED_ARGS[0]:-}"
APPDATA_PATH="${PARSED_ARGS[1]:-}"
OUTPUT_DIR="${PARSED_ARGS[2]:-}"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Setup ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_GEAR Setup ━━━"
# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
error "Must be run as root"
exit 1
fi
success "Running as root"
validate_unraid_cmd \
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
"" "" \
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
# detect_hosts() sets MY_ID — used in summary
detect_hosts
# Arg validation
if [[ -z "$CONTAINER_NAME" || -z "$APPDATA_PATH" || -z "$OUTPUT_DIR" ]]; then
error "Usage: container_data_export.sh <ContainerName> <appdata_path> <output_dir>"
error "Example: container_data_export.sh Emby /mnt/media-servers/Media_Server/Emby /mnt/user/Backups/"
@@ -58,55 +87,94 @@ fi
if [[ ! -d "$OUTPUT_DIR" ]]; then
error "Output directory not found: $OUTPUT_DIR"
error "Create it first: mkdir -p \"$OUTPUT_DIR\""
exit 1
fi
# Check free space — rough estimate: appdata size × 1.1
# Space check — conservative: appdata × 1.1
APPDATA_SIZE_KB=$(du -sk "$APPDATA_PATH" 2>/dev/null | cut -f1)
OUTPUT_FREE_KB=$(df "$OUTPUT_DIR" --output=avail | tail -1 | tr -d ' ')
OUTPUT_FREE_KB=$(df "$OUTPUT_DIR" --output=avail 2>/dev/null | tail -1 | tr -d ' ')
REQUIRED_KB=$(( APPDATA_SIZE_KB * 11 / 10 ))
APPDATA_SIZE_H=$(du -sh "$APPDATA_PATH" 2>/dev/null | cut -f1)
OUTPUT_FREE_H=$(df -h "$OUTPUT_DIR" --output=avail | tail -1 | tr -d ' ')
info "Container: $CONTAINER_NAME"
info "Appdata: $APPDATA_PATH ($APPDATA_SIZE_H)"
info "Output dir: $OUTPUT_DIR ($OUTPUT_FREE_H free)"
OUTPUT_FREE_H=$(df -h "$OUTPUT_DIR" --output=avail 2>/dev/null | tail -1 | tr -d ' ')
if [[ "$OUTPUT_FREE_KB" -lt "$REQUIRED_KB" ]]; then
error "Insufficient space in $OUTPUT_DIR — need ~${APPDATA_SIZE_H}, have ${OUTPUT_FREE_H}"
error "Insufficient space in $OUTPUT_DIR"
error "Estimated need: ~${APPDATA_SIZE_H} (×1.1 conservative) — available: ${OUTPUT_FREE_H}"
exit 1
fi
success "Space check passed"
log "Container: $CONTAINER_NAME"
log "Appdata: $APPDATA_PATH ($APPDATA_SIZE_H)"
log "Output: $OUTPUT_DIR ($OUTPUT_FREE_H free)"
log "Space check passed"
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_STOP Stop Container ━━━
# -----------------------------------------------------------------------------------------------
# ── Ensure container is restarted on any exit if it was running ────────────────────────────────
CONTAINER_WAS_RUNNING=false
ARCHIVE_PATH=""
cleanup_on_exit() {
local exit_code=$?
# Remove partial archive on failure
if [[ "$exit_code" -ne 0 && -n "$ARCHIVE_PATH" && -f "$ARCHIVE_PATH" ]]; then
warn "Removing partial archive: $ARCHIVE_PATH"
rm -f "$ARCHIVE_PATH" 2>/dev/null
fi
# Always restart container if it was running
if [[ "$CONTAINER_WAS_RUNNING" == true && "$DRY_RUN" == false ]]; then
local status
status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \
"$CONTAINER_NAME" 2>/dev/null)
if [[ "$status" != "true" ]]; then
warn "Restarting $CONTAINER_NAME (cleanup)..."
timeout "$DOCKER_TIMEOUT" docker start "$CONTAINER_NAME" >/dev/null 2>&1 || \
error "Failed to restart $CONTAINER_NAME — start it manually"
fi
fi
}
trap cleanup_on_exit EXIT
# ==============================================================================================
# ━━━ Stop Container ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_STOP Stop Container ━━━"
CONTAINER_WAS_RUNNING=false
STATUS=$(docker inspect -f '{{.State.Running}}' "$CONTAINER_NAME" 2>/dev/null)
STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \
"$CONTAINER_NAME" 2>/dev/null)
if [[ "$STATUS" == "true" ]]; then
CONTAINER_WAS_RUNNING=true
info "$ICON_STOP Stopping $CONTAINER_NAME for clean export..."
if [[ "$DRY_RUN" == false ]]; then
docker stop "$CONTAINER_NAME" >/dev/null 2>&1 && \
success "$ICON_STOPPED $CONTAINER_NAME stopped" || \
{ error "Failed to stop $CONTAINER_NAME"; exit 1; }
else
warn "DRY RUN — would stop $CONTAINER_NAME"
fi
else
info "$CONTAINER_NAME is not running — archiving as-is"
fi
case "$STATUS" in
true)
CONTAINER_WAS_RUNNING=true
log "Stopping $CONTAINER_NAME for clean export..."
if [[ "$DRY_RUN" == false ]]; then
if timeout "$DOCKER_TIMEOUT" docker stop "$CONTAINER_NAME" >/dev/null 2>&1; then
log "$CONTAINER_NAME stopped ✅"
else
error "Failed to stop $CONTAINER_NAME — aborting export"
exit 1
fi
else
warn "DRY RUN — would stop $CONTAINER_NAME"
fi
;;
false)
log "$CONTAINER_NAME is not running — archiving as-is (was stopped state respected)"
;;
"")
error "$CONTAINER_NAME not found — check container name"
exit 1
;;
*)
warn "$CONTAINER_NAME status: $STATUS — proceeding with caution"
;;
esac
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SYNC Archive ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Archive ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_SYNC Archive ━━━"
@@ -114,60 +182,107 @@ TIMESTAMP=$(date '+%Y-%m-%d_%H-%M')
ARCHIVE_NAME="${CONTAINER_NAME}_${TIMESTAMP}.tar.gz"
ARCHIVE_PATH="${OUTPUT_DIR}/${ARCHIVE_NAME}"
info "Creating: $ARCHIVE_PATH"
warn "Creating: $ARCHIVE_PATH"
warn "Source: $APPDATA_PATH ($APPDATA_SIZE_H)"
START=$(date +%s)
ARCHIVE_VERIFIED=false
if [[ "$DRY_RUN" == false ]]; then
tar -czf "$ARCHIVE_PATH" -C "$(dirname "$APPDATA_PATH")" "$(basename "$APPDATA_PATH")" 2>/dev/null
TAR_EXIT=$?
if tar -czf "$ARCHIVE_PATH" \
-C "$(dirname "$APPDATA_PATH")" \
"$(basename "$APPDATA_PATH")" 2>/dev/null; then
if [[ "$TAR_EXIT" -ne 0 ]]; then
error "Archive failed (exit code $TAR_EXIT)"
# Restart container before exiting
[[ "$CONTAINER_WAS_RUNNING" == true ]] && docker start "$CONTAINER_NAME" >/dev/null 2>&1
ARCHIVE_SIZE=$(du -sh "$ARCHIVE_PATH" 2>/dev/null | cut -f1)
warn "Archive created: $ARCHIVE_NAME ($ARCHIVE_SIZE)"
# Verify archive integrity before declaring success
log "Verifying archive..."
if tar --test-label -f "$ARCHIVE_PATH" 2>/dev/null || \
tar -tzf "$ARCHIVE_PATH" >/dev/null 2>&1; then
log "Archive verified ✅"
ARCHIVE_VERIFIED=true
else
error "Archive verification FAILED — archive may be corrupt"
error "Container will be restarted but DO NOT rely on this backup"
notify "Container export archive corrupt — $CONTAINER_NAME backup may be unusable" \
"Container Export" "warning"
fi
else
error "tar failed — archive creation unsuccessful"
exit 1
fi
ARCHIVE_SIZE=$(du -sh "$ARCHIVE_PATH" 2>/dev/null | cut -f1)
success "Archive created: $ARCHIVE_NAME ($ARCHIVE_SIZE)"
else
warn "DRY RUN — would create: $ARCHIVE_PATH"
ARCHIVE_VERIFIED=true
fi
END=$(date +%s)
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_START Restart Container ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Restart Container ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_START Restart Container ━━━"
RESTART_OK=false
if [[ "$CONTAINER_WAS_RUNNING" == true ]]; then
info "$ICON_START Restarting $CONTAINER_NAME..."
log "Restarting $CONTAINER_NAME..."
if [[ "$DRY_RUN" == false ]]; then
docker start "$CONTAINER_NAME" >/dev/null 2>&1 && \
success "$ICON_STARTED $CONTAINER_NAME restarted" || \
if timeout "$DOCKER_TIMEOUT" docker start "$CONTAINER_NAME" >/dev/null 2>&1; then
# Brief settle then verify
sleep 3
POST_STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
'{{.State.Running}}' "$CONTAINER_NAME" 2>/dev/null)
if [[ "$POST_STATUS" == "true" ]]; then
log "$CONTAINER_NAME restarted and running ✅"
RESTART_OK=true
else
error "$CONTAINER_NAME started but crashed immediately — check container logs"
notify "$CONTAINER_NAME failed to stay running after export on $(hostname)" \
"Container Export" "warning"
fi
else
error "Failed to restart $CONTAINER_NAME — start it manually"
notify "$CONTAINER_NAME failed to restart after export on $(hostname)" \
"Container Export" "warning"
fi
else
warn "DRY RUN — would restart $CONTAINER_NAME"
RESTART_OK=true
fi
else
info "$CONTAINER_NAME was not running — not restarting"
log "$CONTAINER_NAME was not running — leaving stopped (state respected) ✅"
RESTART_OK=true
fi
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Summary ━━━
# -----------------------------------------------------------------------------------------------
# Clear trap — clean exit, cleanup_on_exit no longer needed
trap - EXIT
# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
echo ""
echo "━━━━━ $ICON_SUMMARY CONTAINER EXPORT SUMMARY ━━━━━"
echo "$ICON_CONTAINERS Container: $CONTAINER_NAME"
echo "$ICON_DISK Appdata: $APPDATA_PATH ($APPDATA_SIZE_H)"
echo "$ICON_SYNC Archive: ${ARCHIVE_NAME:-DRY RUN} ${ARCHIVE_SIZE:+($ARCHIVE_SIZE)}"
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_CONTAINERS Container: $CONTAINER_NAME"
echo "$ICON_DISK Appdata: $APPDATA_PATH ($APPDATA_SIZE_H)"
echo "$ICON_SYNC Archive: ${ARCHIVE_NAME:-DRY RUN} ${ARCHIVE_SIZE:+($ARCHIVE_SIZE)}"
echo "$ICON_SHIELD Verified: $([[ "$ARCHIVE_VERIFIED" == true ]] && echo "✅" || echo "❌ FAILED")"
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
echo ""
if [[ "$DRY_RUN" == true ]]; then
echo "$ICON_WARN Status: DRY RUN — no changes made"
warn "DRY RUN — no changes made"
elif [[ "$ARCHIVE_VERIFIED" == true && "$RESTART_OK" == true ]]; then
log "$ICON_DONE Status: done — $ARCHIVE_NAME"
elif [[ "$ARCHIVE_VERIFIED" == false ]]; then
echo "$ICON_ERROR Status: archive verification FAILED — check backup before relying on it"
else
echo "$ICON_DONE Status: $ICON_SUCCESS DONE"
notify "Container export complete — $CONTAINER_NAME archived to $ARCHIVE_NAME" "Container Export" "normal"
warn "Status: complete with warnings — check restart status above"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
[[ "$ARCHIVE_VERIFIED" == false ]] && exit 1
exit 0
Tools/emby_database_repair.sh
+234 -117
View File
@@ -1,122 +1,199 @@
#!/bin/bash
# -----------------------------------------------------------------------------------------------
# --------------------------------- Emby Database Repair ---------------------------------------
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ============================= Emby Database Repair ===========================================
# ==============================================================================================
# Stops Emby, runs SQLite integrity checks on all Emby databases, and restarts.
# Use when Emby reports database corruption, unexpected crashes, or playback state issues.
#
# Checks performed:
# integrity_check — full SQLite integrity verification per database file
# quick_check — faster check for common corruption patterns
# ── CHECKS PERFORMED ──────────────────────────────────────────────────────────────────────────
# PRAGMA integrity_check — full SQLite integrity verification per database
# Skips missing databases gracefully — not all files exist on all setups
#
# If corruption is found:
# Reports which database files are corrupted
# Does NOT automatically repair — corruption repair requires manual steps
# Provides guidance on next steps per database type
# ── DATABASES CHECKED ─────────────────────────────────────────────────────────────────────────
# library.db — media library metadata (largest, most critical)
# library.db-wal — write-ahead log (if exists — uncommitted transactions)
# librarydb.db — legacy library database
# users.db — user accounts and settings
# authentication.db — API keys and sessions
# activity.db — activity log (least critical, safe to delete)
#
# Emby database files checked:
# library.db — media library metadata
# library.db-wal — write-ahead log (if exists)
# librarydb.db — legacy library database
# users.db — user accounts and settings
# authentication.db — API keys and sessions
# activity.db — activity log
# ── IF CORRUPTION FOUND ───────────────────────────────────────────────────────────────────────
# Reports which databases are corrupted. Does NOT automatically repair.
# Corruption repair requires manual steps — see guidance in summary output.
# Always take a backup before deleting any database file.
#
# HOST1 repairs its own Emby (HOST1_EMBY_CONTAINER).
# HOST2 repairs its own Emby (HOST2_EMBY_CONTAINER).
# detect_hosts() selects the correct container at runtime.
# Container name defined in Host Configuration in Master.conf.
# Supports --dry-run to show what would be checked without stopping Emby.
# -----------------------------------------------------------------------------------------------
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
# detect_hosts() sets MY_ID and aliases HOST*_EMBY_CONTAINER → EMBY_CONTAINER.
# Each server checks its own Emby instance automatically.
#
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
# EXIT trap — Emby always restarted even if script crashes mid-check
# DOCKER_TIMEOUT — all docker calls protected against hung daemon
# jq validation — verifies jq available before config path detection
# validate_unraid_cmd — sqlite3 and notify validated before use
# Container verify — checks Emby stayed running after restart
# Silent healthy — only corruption produces visible output
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
# emby_database_repair.sh — stop Emby, check all databases, restart
# emby_database_repair.sh --dry-run — show what would be checked, no Emby stop
# emby_database_repair.sh --log — verbose output per database
# emby_database_repair.sh --status — show config and exit
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../Master.conf"
source "$SCRIPT_DIR/../common.sh"
source "$SCRIPT_DIR/../load_config.sh"
parse_args "$@"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Setup ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_GEAR Setup ━━━"
DOCKER_TIMEOUT=30 # Emby can take time to stop cleanly
# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
error "Must be run as root"
exit 1
fi
success "Running as root"
# Validate required tools
validate_unraid_cmd \
"$(command -v sqlite3 2>/dev/null || echo /usr/bin/sqlite3)" \
"--version" "." \
"sqlite3" || { error "sqlite3 not found — install sqlite package"; exit 1; }
acquire_lock
validate_unraid_cmd \
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
"" "" \
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
# Select correct Emby container based on which server is running this script
detect_hosts
if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then
EMBY_CONTAINER="$HOST1_EMBY_CONTAINER"
else
EMBY_CONTAINER="$HOST2_EMBY_CONTAINER"
fi
info "Emby container: $LOCAL_SERVER_NAME → $EMBY_CONTAINER"
if ! command -v sqlite3 >/dev/null 2>&1; then
error "sqlite3 not found — install sqlite package"
if ! command -v jq >/dev/null 2>&1; then
error "jq not found — required to detect Emby config path from Docker mounts"
exit 1
fi
success "sqlite3 available"
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — Emby will not be stopped"
acquire_lock
# Detect Emby config path from Docker mount
EMBY_CONFIG_HOST=$(docker inspect "$EMBY_CONTAINER" 2>/dev/null | \
# detect_hosts() sets MY_ID and aliases HOST*_EMBY_CONTAINER → EMBY_CONTAINER
detect_hosts
if [[ -z "${EMBY_CONTAINER:-}" ]]; then
error "EMBY_CONTAINER not set for $MY_ID — check HOST*_EMBY_CONTAINER in master_host*.conf"
exit 1
fi
log "Identity: $MY_ID ($LOCAL_SERVER_NAME)"
log "Emby container: $EMBY_CONTAINER"
# Detect Emby config path from Docker container mounts
EMBY_CONFIG_HOST=$(timeout "$DOCKER_TIMEOUT" docker inspect "$EMBY_CONTAINER" 2>/dev/null | \
jq -r '.[] | .Mounts[] | select(.Destination == "/config") | .Source' 2>/dev/null)
if [[ -z "$EMBY_CONFIG_HOST" ]]; then
error "Could not detect Emby config path from Docker mounts"
error "Make sure $EMBY_CONTAINER is the correct container name in Master.conf"
error "Is $EMBY_CONTAINER the correct container name? Check HOST*_EMBY_CONTAINER in master_host*.conf"
exit 1
fi
success "Emby config path: $EMBY_CONFIG_HOST"
log "Emby config: $EMBY_CONFIG_HOST"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_STOP Stop Emby ━━━
# -----------------------------------------------------------------------------------------------
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — Emby will not be stopped, no checks run"
# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
echo ""
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_EMBY Container: $EMBY_CONTAINER"
echo "$ICON_EMBY Config: $EMBY_CONFIG_HOST"
echo "$ICON_GEAR Dry Run: $DRY_RUN"
echo ""
echo "━━━ Database Files ━━━"
for db_rel in "data/library.db" "data/library.db-wal" "data/librarydb.db" \
"data/users.db" "data/authentication.db" "data/activity.db"; do
db_path="${EMBY_CONFIG_HOST}/${db_rel}"
db_name=$(basename "$db_rel")
if [[ -f "$db_path" ]]; then
db_size=$(du -sh "$db_path" 2>/dev/null | cut -f1)
echo " $ICON_SUCCESS $db_name ($db_size)"
else
echo " $ICON_SKIP $db_name — not found"
fi
done
echo "━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
fi
# ── EXIT trap — Emby always restarted if it was running ───────────────────────────────────────
EMBY_WAS_RUNNING=false
cleanup_on_exit() {
local exit_code=$?
if [[ "$EMBY_WAS_RUNNING" == true && "$DRY_RUN" == false ]]; then
local status
status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \
"$EMBY_CONTAINER" 2>/dev/null)
if [[ "$status" != "true" ]]; then
warn "Restarting $EMBY_CONTAINER (cleanup)..."
timeout "$DOCKER_TIMEOUT" docker start "$EMBY_CONTAINER" >/dev/null 2>&1 || \
error "Failed to restart $EMBY_CONTAINER — start it manually"
fi
fi
}
trap cleanup_on_exit EXIT
# ==============================================================================================
# ━━━ Stop Emby ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_STOP Stop Emby ━━━"
EMBY_WAS_RUNNING=false
STATUS=$(docker inspect -f '{{.State.Running}}' "$EMBY_CONTAINER" 2>/dev/null)
STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \
"$EMBY_CONTAINER" 2>/dev/null)
if [[ "$STATUS" == "true" ]]; then
EMBY_WAS_RUNNING=true
warn "Stopping $EMBY_CONTAINER — active sessions will be interrupted"
if [[ "$DRY_RUN" == false ]]; then
docker stop "$EMBY_CONTAINER" >/dev/null 2>&1 && \
success "$ICON_STOPPED $EMBY_CONTAINER stopped" || \
{ error "Failed to stop $EMBY_CONTAINER"; exit 1; }
sleep 3 # brief wait for file handles to release
else
warn "DRY RUN — would stop $EMBY_CONTAINER"
fi
else
info "$EMBY_CONTAINER is not running — proceeding with checks"
fi
case "$STATUS" in
true)
EMBY_WAS_RUNNING=true
warn "Stopping $EMBY_CONTAINER — active sessions will be interrupted"
if [[ "$DRY_RUN" == false ]]; then
if timeout "$DOCKER_TIMEOUT" docker stop "$EMBY_CONTAINER" >/dev/null 2>&1; then
log "$EMBY_CONTAINER stopped ✅"
sleep 3 # let file handles release
else
error "Failed to stop $EMBY_CONTAINER — aborting"
exit 1
fi
else
warn "DRY RUN — would stop $EMBY_CONTAINER"
fi
;;
false)
log "$EMBY_CONTAINER is not running — proceeding with checks"
;;
"")
error "$EMBY_CONTAINER not found — check container name"
exit 1
;;
*)
warn "$EMBY_CONTAINER status: $STATUS — proceeding with caution"
;;
esac
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_HEALTH Database Integrity Check ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Database Integrity Check ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_HEALTH Database Integrity Check ━━━"
START=$(date +%s)
# Emby database files to check
DB_FILES=(
"data/library.db"
"data/library.db-wal"
"data/librarydb.db"
"data/users.db"
"data/authentication.db"
@@ -138,25 +215,38 @@ for db_rel in "${DB_FILES[@]}"; do
fi
DB_SIZE=$(du -sh "$db_path" 2>/dev/null | cut -f1)
info "$ICON_HEALTH Checking $db_name ($DB_SIZE)..."
log "Checking $db_name ($DB_SIZE)..."
# WAL file — different check (not a full SQLite database)
if [[ "$db_name" == "*.wal" || "$db_name" == "library.db-wal" ]]; then
if [[ -s "$db_path" ]]; then
warn "$db_name exists and is non-empty (${DB_SIZE})"
warn "Uncommitted WAL data — will be merged when Emby next starts cleanly"
PASS_DBS+=("$db_name (WAL — see warning)")
else
log "$db_name exists but is empty — no pending transactions ✅"
PASS_DBS+=("$db_name")
fi
continue
fi
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would check: $db_path"
continue
fi
# Run integrity check
# Full integrity check
RESULT=$(sqlite3 "$db_path" "PRAGMA integrity_check;" 2>/dev/null)
EXIT_CODE=$?
if [[ "$EXIT_CODE" -ne 0 ]]; then
error "$db_name — sqlite3 could not open database (may be locked or corrupt)"
error "$db_name — sqlite3 could not open database (locked or corrupt)"
FAIL_DBS+=("$db_name")
elif [[ "$RESULT" == "ok" ]]; then
success "$db_name — integrity check passed"
log "$db_name — integrity check passed ✅"
PASS_DBS+=("$db_name")
else
error "$db_name — integrity check FAILED"
error "$db_name — CORRUPTION DETECTED"
echo "$RESULT" | head -10 | while IFS= read -r line; do
error " $line"
done
@@ -166,61 +256,88 @@ done
END=$(date +%s)
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_START Restart Emby ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Restart Emby ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_START Restart Emby ━━━"
if [[ "$EMBY_WAS_RUNNING" == true && "$DRY_RUN" == false ]]; then
docker start "$EMBY_CONTAINER" >/dev/null 2>&1 && \
success "$ICON_STARTED $EMBY_CONTAINER restarted" || \
error "Failed to restart $EMBY_CONTAINER — start it manually"
elif [[ "$DRY_RUN" == true && "$EMBY_WAS_RUNNING" == true ]]; then
warn "DRY RUN — would restart $EMBY_CONTAINER"
RESTART_OK=false
if [[ "$EMBY_WAS_RUNNING" == true ]]; then
if [[ "$DRY_RUN" == false ]]; then
log "Restarting $EMBY_CONTAINER..."
if timeout "$DOCKER_TIMEOUT" docker start "$EMBY_CONTAINER" >/dev/null 2>&1; then
sleep 5 # Emby takes longer to initialise than most containers
POST_STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
'{{.State.Running}}' "$EMBY_CONTAINER" 2>/dev/null)
if [[ "$POST_STATUS" == "true" ]]; then
log "$EMBY_CONTAINER restarted and running ✅"
RESTART_OK=true
else
error "$EMBY_CONTAINER started but crashed — database may be corrupt"
error "Check Docker logs: docker logs $EMBY_CONTAINER"
notify "$EMBY_CONTAINER crashed on restart — possible database corruption on $(hostname)" \
"Emby DB Repair" "warning"
fi
else
error "Failed to restart $EMBY_CONTAINER — start it manually"
notify "$EMBY_CONTAINER failed to restart after integrity check on $(hostname)" \
"Emby DB Repair" "warning"
fi
else
warn "DRY RUN — would restart $EMBY_CONTAINER"
RESTART_OK=true
fi
else
info "$EMBY_CONTAINER was not running — not restarting"
log "$EMBY_CONTAINER was not running — leaving stopped (state respected) ✅"
RESTART_OK=true
fi
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Summary ━━━
# -----------------------------------------------------------------------------------------------
# Clear EXIT trap — clean exit
trap - EXIT
# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
echo ""
echo "━━━━━ $ICON_SUMMARY EMBY DATABASE REPAIR SUMMARY ━━━━━"
echo "$ICON_HEALTH Container: $EMBY_CONTAINER"
echo "$ICON_HEALTH Config path: $EMBY_CONFIG_HOST"
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_EMBY Container: $EMBY_CONTAINER"
echo "$ICON_EMBY Config: $EMBY_CONFIG_HOST"
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
echo ""
echo " $ICON_SUCCESS Passed: ${#PASS_DBS[@]} $ICON_ERROR Failed: ${#FAIL_DBS[@]} $ICON_INFO Missing: ${#MISSING_DBS[@]}"
echo " $ICON_SUCCESS Passed: ${#PASS_DBS[@]}"
[[ ${#FAIL_DBS[@]} -gt 0 ]] && echo " $ICON_ERROR Failed: ${#FAIL_DBS[@]}"
[[ ${#MISSING_DBS[@]} -gt 0 ]] && log "Skipped: ${#MISSING_DBS[@]} (not found)"
echo ""
if [[ ${#PASS_DBS[@]} -gt 0 ]]; then
for db in "${PASS_DBS[@]}"; do echo " $ICON_SUCCESS $db"; done
fi
if [[ ${#FAIL_DBS[@]} -gt 0 ]]; then
for db in "${FAIL_DBS[@]}"; do echo " $ICON_ERROR $db"; done
fi
[[ ${#PASS_DBS[@]} -gt 0 ]] && for db in "${PASS_DBS[@]}"; do log " $ICON_SUCCESS $db"; done
[[ ${#FAIL_DBS[@]} -gt 0 ]] && for db in "${FAIL_DBS[@]}"; do echo " $ICON_ERROR $db"; done
echo ""
if [[ "$DRY_RUN" == true ]]; then
echo "$ICON_WARN Status: DRY RUN — no checks performed"
warn "DRY RUN — no checks performed"
elif [[ ${#FAIL_DBS[@]} -gt 0 ]]; then
echo "$ICON_ERROR Status: CORRUPTION FOUND"
echo "$ICON_ERROR Status: CORRUPTION FOUND — manual intervention needed"
echo ""
echo "$ICON_INFO Next steps for corrupted databases:"
echo " library.db — Stop Emby, delete library.db, restart"
echo " Emby will rebuild from media files (slow first start)"
echo " users.db — Stop Emby, restore from backup or delete"
echo " Deleting resets all user accounts"
echo " authentication.db — Stop Emby, delete, restart"
echo " API keys and sessions will be regenerated"
echo " activity.db — Stop Emby, delete, restart — activity log only"
echo "$ICON_INFO Next steps per corrupted database:"
echo " library.db — Delete file, restart Emby — rebuilds from media (slow first start)"
echo " library.db-wal — Delete WAL file, restart Emby — safe, no permanent data loss"
echo " librarydb.db — Delete file, restart Emby — legacy, Emby recreates"
echo " users.db — Restore from backup or delete — deleting resets all user accounts"
echo " authentication.db — Delete file, restart Emby — API keys regenerated automatically"
echo " activity.db — Delete file, restart Emby — activity log only, no media data"
echo ""
echo "$ICON_WARN Always take a backup before deleting any database file"
notify "Emby database corruption found on $(hostname) — failed: ${FAIL_DBS[*]} — manual intervention needed" "Emby DB Repair" "warning"
warn "⚠️ Always take a backup before deleting any database file"
warn " Run: container_data_export.sh $EMBY_CONTAINER <config_path> <backup_dir>"
notify "Emby database CORRUPTION on $(hostname) — failed: ${FAIL_DBS[*]} — manual intervention needed" \
"Emby DB Repair" "warning"
else
echo "$ICON_DONE Status: $ICON_SUCCESS ALL DATABASES HEALTHY"
notify "Emby database integrity check passed on $(hostname) — ${#PASS_DBS[@]} databases healthy" "Emby DB Repair" "normal"
log "$ICON_DONE Status: all ${#PASS_DBS[@]} databases healthy ✅"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
[[ ${#FAIL_DBS[@]} -gt 0 ]] && exit 1
exit 0
Tools/failover_state_reset.sh
+168 -66
View File
@@ -1,127 +1,229 @@
#!/bin/bash
# -----------------------------------------------------------------------------------------------
# --------------------------------- Failover State Reset ---------------------------------------
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ============================= Failover State Reset ===========================================
# ==============================================================================================
# Resets the failover state file to NORMAL and clears all tier flags.
# Use when the failover state file is stuck in a non-NORMAL state after testing,
# a failed handback, or manual intervention that left state inconsistent.
# Use when the failover state file is stuck in a non-NORMAL state after:
# - Failover testing that left state as FAILOVER
# - A failed handback that did not complete cleanly
# - Manual intervention that left state inconsistent
# - failover.sh was killed mid-cycle and state is unknown
#
# ── WHAT THIS DOES ────────────────────────────────────────────────────────────────────────────
# Writes a fresh state file with:
# state=NORMAL
# failover_start=0
# handback_strikes=0
# tier2_started=false / tier3_started=false / tier4_started=false
#
# Does NOT start or stop any containers — state file only.
# After reset, failover.sh will resume from NORMAL on its next cycle.
#
# ⚠️ Only run this when you have manually verified both servers are in their
# correct states — right containers running on the right server, DDNS correct.
# Resetting state without verifying the actual state can cause failover.sh
# to make incorrect decisions on its next cycle.
# ── ⚠️ ONLY RUN WHEN SAFE ────────────────────────────────────────────────────────────────────
# Verify BEFORE resetting:
# ✓ Right containers running on the right server
# ✓ DDNS pointing at the correct server
# ✓ No active failover actually in progress
# ✓ Both servers can see each other
#
# Supports --dry-run to show what would be reset without changing anything.
# Supports --status to show the current state file contents.
# -----------------------------------------------------------------------------------------------
# Resetting state while a real failover is happening causes failover.sh to stop
# covering the remote server — services go offline until next detection cycle.
#
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
# failover.sh running check — warns if failover.sh is active when reset is attempted
# acquire_lock — prevents concurrent resets
# flock on state write — prevents race with failover.sh mid-cycle read
# Confirmation required — interactive: type YES | non-interactive: --force flag
# validate_unraid_cmd — notify validated before use
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
# failover_state_reset.sh — interactive reset (prompts for YES)
# failover_state_reset.sh --dry-run — show current state, show what would be written
# failover_state_reset.sh --status — show current state file contents and exit
# failover_state_reset.sh --force — non-interactive reset (no prompt, use in scripts)
# failover_state_reset.sh --force --dry-run — dry run without prompt
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../Master.conf"
source "$SCRIPT_DIR/../common.sh"
source "$SCRIPT_DIR/../load_config.sh"
parse_args "$@"
# ── Handle --force flag before parse_args ─────────────────────────────────────────────────────
FORCE=false
FILTERED_ARGS=()
for arg in "$@"; do
case "$arg" in
--force) FORCE=true ;;
*) FILTERED_ARGS+=("$arg") ;;
esac
done
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Setup ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_GEAR Setup ━━━"
parse_args "${FILTERED_ARGS[@]}"
# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
error "Must be run as root"
exit 1
fi
success "Running as root"
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
validate_unraid_cmd \
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
"" "" \
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Current State ━━━
# -----------------------------------------------------------------------------------------------
acquire_lock
# detect_hosts() sets MY_ID — used in summary and notification
detect_hosts
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
[[ "$FORCE" == true ]] && warn "FORCE mode — confirmation prompt skipped"
# ==============================================================================================
# ━━━ Current State ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_FAILOVER Current Failover State ━━━"
echo "$ICON_HOST My ID: $MY_ID ($LOCAL_SERVER_NAME)"
echo ""
echo "━━━ $ICON_SUMMARY Current State ━━━"
if [[ ! -f "$FAILOVER_STATE_FILE" ]]; then
warn "State file not found: $FAILOVER_STATE_FILE"
warn "Will be created fresh on reset"
CURRENT_STATE="NOT FOUND"
else
info "State file: $FAILOVER_STATE_FILE"
log "State file: $FAILOVER_STATE_FILE"
echo ""
while IFS='=' read -r key value; do
[[ -z "$key" ]] && continue
echo " $ICON_INFO $key = $value"
done < "$FAILOVER_STATE_FILE"
CURRENT_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
fi
if [[ "$SHOW_STATUS" == true ]]; then
echo ""
# Check if failover.sh is running — informational in status mode
if pgrep -f "failover.sh" >/dev/null 2>&1; then
warn "failover.sh is currently RUNNING — any reset would race with active cycle"
else
log "failover.sh is not running"
fi
exit 0
fi
# -----------------------------------------------------------------------------------------------
# ━━━ Confirmation ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Safety Checks ━━━
# ==============================================================================================
echo ""
warn "$ICON_WARN This will reset the failover state to NORMAL"
warn "Only proceed if you have verified both servers are in their correct states"
warn " — Right containers running on the right server"
warn " — DDNS pointing at the correct server"
warn " — No active failover in progress"
echo "━━━ $ICON_SHIELD Safety Checks ━━━"
# Check if failover.sh is actively running
FAILOVER_RUNNING=false
if pgrep -f "failover.sh" >/dev/null 2>&1; then
FAILOVER_RUNNING=true
warn "⚠️ failover.sh is currently RUNNING"
warn "Resetting state mid-cycle may cause incorrect decisions on the next iteration"
warn "Consider stopping failover.sh first (click Abort in User Scripts)"
warn "Then reset state, then restart failover.sh"
echo ""
warn "If you are sure you want to proceed anyway, confirm below"
else
log "failover.sh is not running — safe to reset ✅"
fi
# Check current state — if already NORMAL warn user
if [[ "$CURRENT_STATE" == "NORMAL" ]]; then
warn "State is already NORMAL — reset may not be necessary"
warn "Proceeding anyway (will refresh the state file)"
fi
# ==============================================================================================
# ━━━ Confirmation ━━━
# ==============================================================================================
echo ""
warn "This will reset failover state to NORMAL on $MY_ID ($LOCAL_SERVER_NAME)"
warn "Verify before proceeding:"
warn " ✓ Right containers running on the right server"
warn " ✓ DDNS pointing at correct server"
warn " ✓ No real failover actually in progress"
warn " ✓ Both servers can reach each other"
echo ""
if [[ "$DRY_RUN" == false ]]; then
read -r -p "Type YES to confirm reset: " CONFIRM
if [[ "$CONFIRM" != "YES" ]]; then
info "Reset cancelled"
exit 0
if [[ "$FORCE" == true ]]; then
log "FORCE flag set — skipping confirmation prompt"
elif [[ -t 0 ]]; then
# Interactive terminal — prompt for confirmation
read -r -p "Type YES to confirm reset: " CONFIRM
if [[ "$CONFIRM" != "YES" ]]; then
warn "Reset cancelled"
exit 0
fi
else
# Non-interactive — no terminal, cannot prompt
error "Non-interactive mode — use --force flag to skip confirmation"
error "Usage: failover_state_reset.sh --force"
exit 1
fi
fi
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_FAILOVER Reset State File ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Reset State File ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_FAILOVER Resetting State File ━━━"
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would write:"
echo " state=NORMAL"
echo " failover_start=0"
echo " handback_strikes=0"
echo " tier2_started=false"
echo " tier3_started=false"
echo " tier4_started=false"
echo " last_reset=$(date '+%Y-%m-%d %H:%M:%S')"
else
mkdir -p "$(dirname "$FAILOVER_STATE_FILE")"
cat > "$FAILOVER_STATE_FILE" << EOF
state=NORMAL
NEW_STATE_CONTENT="state=NORMAL
failover_start=0
handback_strikes=0
tier2_started=false
tier3_started=false
tier4_started=false
last_reset=$(date '+%Y-%m-%d %H:%M:%S')
EOF
success "State file reset to NORMAL"
reset_by=$MY_ID"
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would write to $FAILOVER_STATE_FILE:"
echo ""
echo "$NEW_STATE_CONTENT" | while IFS= read -r line; do
echo " $line"
done
else
mkdir -p "$(dirname "$FAILOVER_STATE_FILE")"
# flock prevents race with failover.sh mid-cycle read/write
(
flock -x 200
echo "$NEW_STATE_CONTENT" > "$FAILOVER_STATE_FILE"
) 200>"${FAILOVER_STATE_FILE}.lock"
warn "State file reset to NORMAL ✅"
log "Written to: $FAILOVER_STATE_FILE"
fi
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Summary ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
echo ""
echo "━━━━━ $ICON_SUMMARY FAILOVER STATE RESET SUMMARY ━━━━━"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_FAILOVER File: $FAILOVER_STATE_FILE"
echo "$ICON_TIME Reset at: $(date '+%Y-%m-%d %H:%M:%S')"
echo ""
if [[ "$DRY_RUN" == true ]]; then
echo "$ICON_WARN Status: DRY RUN — no changes made"
warn "DRY RUN — no changes made"
else
echo "$ICON_DONE Status: $ICON_SUCCESS State reset to NORMAL"
echo "$ICON_TIME Reset at: $(date '+%Y-%m-%d %H:%M:%S')"
warn "$ICON_DONE State reset to NORMAL"
log "failover.sh will resume from NORMAL on next cycle"
log "No containers were started or stopped"
echo ""
echo "$ICON_INFO failover.sh will resume from NORMAL on next cycle"
echo "$ICON_INFO No containers were started or stopped"
notify "Failover state manually reset to NORMAL on $(hostname)" "Failover State Reset" "normal"
[[ "$FAILOVER_RUNNING" == true ]] && \
warn "⚠️ failover.sh was running during reset — monitor next cycle carefully"
notify "Failover state manually reset to NORMAL on $(hostname) ($MY_ID)" \
"Failover State Reset" "warning"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
Tools/recreate_shares.sh
+232 -93
View File
@@ -1,134 +1,273 @@
#!/bin/bash
# -----------------------------------------------------------------------------------------------
# --------------------------------- Recreate Shares Script -------------------------------------
# -----------------------------------------------------------------------------------------------
# Reads all .cfg files from /boot/config/shares/ and creates the corresponding share
# directories on the correct disks based on shareInclude settings.
# Also drops a .recovery marker file in each share via /mnt/user/ so that an initial
# rsync push can run without --delete and self-clean on the second nightly run.
# ==============================================================================================
# ============================= Recreate Shares ================================================
# ==============================================================================================
# Creates share directories on the correct disks after a fresh unRAID install or disk rebuild.
# Reads all .cfg files from /boot/config/shares/ and creates the corresponding directories
# on each disk listed in the shareInclude setting.
#
# Run this script directly on the secondary server after array is started.
# Usage: bash recreate_shares.sh
# -----------------------------------------------------------------------------------------------
# ── WHEN TO USE ───────────────────────────────────────────────────────────────────────────────
# Run directly on HOST2 after array is started following:
# - A full disk replacement or rebuild where share folders were lost
# - A fresh unRAID install where /boot/config/shares/*.cfg files were restored
# - Any situation where the share folder structure exists in config but not on disk
#
# The array must be started before running this script — /mnt/user must be mounted.
#
# ── WHAT IT DOES ──────────────────────────────────────────────────────────────────────────────
# For each share .cfg file:
# 1. Reads shareInclude= to determine which disks own this share
# 2. Creates /mnt/diskN/ShareName/ on each listed disk if it doesn't exist
# 3. Places a .recovery marker file in /mnt/user/ShareName/ via the union filesystem
#
# ── .RECOVERY MARKER FILE ─────────────────────────────────────────────────────────────────────
# The .recovery marker signals to rsync.sh that this is a fresh share with no existing data.
# rsync.sh checks for .recovery before running with --delete:
# .recovery present → rsync WITHOUT --delete (safe — new files only, nothing removed)
# .recovery absent → rsync WITH --delete (normal — mirror mode)
#
# The marker self-cleans: after the first successful rsync the source side has no .recovery
# file so the second nightly run will delete it from the mirror, restoring normal --delete
# behaviour automatically. No manual cleanup needed. ✅
#
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
# This script runs on the server that needs shares recreated — typically HOST2 during rebuild.
# detect_hosts() sets MY_ID for output clarity.
#
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
# acquire_lock — prevents duplicate runs placing duplicate markers
# Root check — mkdir on /mnt/diskN requires root
# Array mount check — exits cleanly if array not started
# Empty cfg guard — warns if no share cfg files found
# Per-disk guards — skips missing disks with warning, continues others
# validate_unraid_cmd — notify validated before use
# Silent on success — only failures produce visible output
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
# recreate_shares.sh — create all shares from .cfg files
# recreate_shares.sh --dry-run — preview what would be created, no changes
# recreate_shares.sh --log — verbose output per disk
# recreate_shares.sh --status — show current share state and exit
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../Master.conf"
source "$SCRIPT_DIR/../common.sh"
source "$SCRIPT_DIR/../load_config.sh"
parse_args "$@"
SHARE_CFG_DIR="/boot/config/shares"
MARKER_FILE=".recovery"
CREATED=()
SKIPPED=()
FAILED=()
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Setup ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_GEAR Setup ━━━"
if ! mountpoint -q /mnt/user; then
error "Array is not started — /mnt/user is not mounted"
info "Start the array in the unRAID UI before running this script"
# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
error "Must be run as root — mkdir on /mnt/diskN requires root"
exit 1
fi
success "Array is started"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_DISK Recreating Shares ━━━
# -----------------------------------------------------------------------------------------------
validate_unraid_cmd \
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
"" "" \
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
acquire_lock
# detect_hosts() sets MY_ID — used in summary
detect_hosts
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no directories or markers will be created"
# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
echo ""
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_DISK Cfg dir: $SHARE_CFG_DIR"
echo "$ICON_DISK Marker: $MARKER_FILE"
echo ""
if ! mountpoint -q /mnt/user; then
warn "Array: NOT STARTED — /mnt/user not mounted"
else
echo " Array: started ✅"
fi
echo ""
echo "━━━ Share Config Files ━━━"
CFG_COUNT=0
for cfg in "$SHARE_CFG_DIR"/*.cfg; do
[[ ! -f "$cfg" ]] && continue
(( CFG_COUNT++ ))
SHARE_NAME=$(basename "$cfg" .cfg)
INCLUDE=$(grep '^shareInclude=' "$cfg" 2>/dev/null | cut -d'"' -f2)
MARKER_EXISTS="no"
[[ -f "/mnt/user/${SHARE_NAME}/${MARKER_FILE}" ]] && MARKER_EXISTS="yes"
echo " $ICON_DISK $SHARE_NAME — disks: ${INCLUDE:-none} — recovery marker: $MARKER_EXISTS"
done
[[ "$CFG_COUNT" -eq 0 ]] && warn "No .cfg files found in $SHARE_CFG_DIR"
echo "━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
fi
# ==============================================================================================
# ━━━ Pre-flight ━━━
# ==============================================================================================
# Array must be started — /mnt/user must be mounted
if ! mountpoint -q /mnt/user; then
error "Array is not started — /mnt/user is not mounted"
warn "Start the array in the unRAID UI before running this script"
notify "Recreate shares failed on $(hostname) — array is not started" \
"Recreate Shares" "warning"
exit 1
fi
log "Array is started — /mnt/user is mounted ✅"
# Check share cfg directory exists and has files
if [[ ! -d "$SHARE_CFG_DIR" ]]; then
error "Share config directory not found: $SHARE_CFG_DIR"
error "Is /boot mounted? Is this the correct server?"
exit 1
fi
CFG_FILES=("$SHARE_CFG_DIR"/*.cfg)
if [[ ! -f "${CFG_FILES[0]}" ]]; then
warn "No share .cfg files found in $SHARE_CFG_DIR"
warn "Nothing to recreate — are share configs present on /boot?"
exit 0
fi
log "Found ${#CFG_FILES[@]} share .cfg file(s) in $SHARE_CFG_DIR"
# ==============================================================================================
# ━━━ Recreate Shares ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_DISK Recreating Shares ━━━"
echo "━━━ $ICON_DISK Recreate Shares — $MY_ID ━━━"
echo ""
for cfg in "$SHARE_CFG_DIR"/*.cfg; do
for cfg in "${CFG_FILES[@]}"; do
[[ ! -f "$cfg" ]] && continue
SHARE_NAME=$(basename "$cfg" .cfg)
INCLUDE=$(grep '^shareInclude=' "$cfg" | cut -d'"' -f2)
INCLUDE=$(grep '^shareInclude=' "$cfg" 2>/dev/null | cut -d'"' -f2)
echo "━━━ $ICON_DISK $SHARE_NAME ━━━"
if [[ -z "$INCLUDE" ]]; then
warn "$SHARE_NAME — no shareInclude defined, skipping"
warn "$SHARE_NAME — no shareInclude in .cfg — skipping"
SKIPPED+=("$SHARE_NAME")
echo ""
continue
fi
info "$ICON_DISK Processing $SHARE_NAME (disks: $INCLUDE)..."
log "$SHARE_NAME — disks: $INCLUDE"
SHARE_OK=true
DIRS_CREATED=0
DIRS_EXISTED=0
# Create directory on each listed disk
IFS=',' read -ra DISKS <<< "$INCLUDE"
for disk in "${DISKS[@]}"; do
DISK_PATH="/mnt/${disk}/${SHARE_NAME}"
disk="${disk// /}" # trim whitespace
[[ -z "$disk" ]] && continue
DISK_MOUNT="/mnt/${disk}"
DISK_PATH="${DISK_MOUNT}/${SHARE_NAME}"
# Verify disk is mounted
if ! mountpoint -q "$DISK_MOUNT" 2>/dev/null; then
warn "$disk not mounted — skipping $DISK_PATH"
continue
fi
if [[ -d "$DISK_PATH" ]]; then
echo "$ICON_RUNNING $disk/$SHARE_NAME already exists, skipping"
log "$disk/$SHARE_NAME already exists — skipping"
(( DIRS_EXISTED++ ))
else
if mkdir -p "$DISK_PATH"; then
echo "$ICON_STARTED Created $DISK_PATH"
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would create: $DISK_PATH"
(( DIRS_CREATED++ ))
elif mkdir -p "$DISK_PATH"; then
log "Created: $DISK_PATH ✅"
(( DIRS_CREATED++ ))
else
error "Failed to create $DISK_PATH"
error "Failed to create: $DISK_PATH"
SHARE_OK=false
fi
fi
done
# Place .recovery marker via /mnt/user (union filesystem)
MARKER_PATH="/mnt/user/${SHARE_NAME}/${MARKER_FILE}"
if [[ "$SHARE_OK" == true ]]; then
if touch "$MARKER_PATH" 2>/dev/null; then
echo "$ICON_DONE Marker placed: $MARKER_PATH"
if [[ -f "$MARKER_PATH" ]]; then
log ".recovery marker already exists in $SHARE_NAME"
CREATED+=("$SHARE_NAME")
elif [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would place marker: $MARKER_PATH"
CREATED+=("$SHARE_NAME")
elif touch "$MARKER_PATH" 2>/dev/null; then
log "Marker placed: $MARKER_PATH ✅"
CREATED+=("$SHARE_NAME")
else
warn "Could not place marker in $SHARE_NAME — share may not be visible yet"
warn "$SHARE_NAME — could not place .recovery marker"
warn "Share directory may not be visible via /mnt/user yet"
warn "Try: touch /mnt/user/${SHARE_NAME}/.recovery manually after verifying share"
SKIPPED+=("$SHARE_NAME")
fi
else
FAILED+=("$SHARE_NAME")
fi
[[ "$DIRS_CREATED" -gt 0 ]] && warn "$SHARE_NAME — created $DIRS_CREATED dir(s) on disk"
[[ "$DIRS_EXISTED" -gt 0 ]] && log "$SHARE_NAME — $DIRS_EXISTED dir(s) already existed"
echo ""
done
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Summary ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
echo "━━━━━ $ICON_SUMMARY RECREATE SHARES SUMMARY ━━━━━"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo ""
if [[ ${#CREATED[@]} -gt 0 ]]; then
echo " $ICON_DONE Created & marked:"
for s in "${CREATED[@]}"; do
echo " $ICON_STARTED $s"
done
echo ""
fi
if [[ ${#SKIPPED[@]} -gt 0 ]]; then
echo " $ICON_WARN Skipped:"
for s in "${SKIPPED[@]}"; do
echo " $ICON_NOT_RUNNING $s"
done
echo ""
fi
if [[ ${#FAILED[@]} -gt 0 ]]; then
echo " $ICON_ERROR Failed:"
for s in "${FAILED[@]}"; do
echo " $ICON_ERROR $s"
done
echo ""
fi
echo " $ICON_SUCCESS Created: ${#CREATED[@]}"
echo " $ICON_NOT_RUNNING Skipped: ${#SKIPPED[@]}"
echo " $ICON_ERROR Failed: ${#FAILED[@]}"
[[ ${#CREATED[@]} -gt 0 ]] && warn "Created + marked: ${CREATED[*]}"
[[ ${#SKIPPED[@]} -gt 0 ]] && warn "Skipped: ${SKIPPED[*]}"
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}"
echo ""
echo "Next steps:"
echo " 1. $ICON_HEALTH Verify shares are visible in unRAID UI"
echo " 2. $ICON_GEAR Remove --delete from Master.conf rsync opts"
echo " 3. $ICON_RUN Run initial push — marker files self-clean on second nightly run"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
[[ ${#FAILED[@]} -gt 0 ]] && exit 1
exit 0
echo " Created: ${#CREATED[@]}"
echo " Skipped: ${#SKIPPED[@]}"
echo " Failed: ${#FAILED[@]}"
if [[ "$DRY_RUN" == false && ${#CREATED[@]} -gt 0 ]]; then
echo ""
echo "━━━ Next Steps ━━━"
echo " 1. $ICON_HEALTH Verify shares are visible in unRAID UI"
echo " 2. $ICON_SYNC Run initial rsync push from HOST1 → HOST2"
echo " rsync.sh will detect .recovery markers and skip --delete"
echo " Normal --delete mode restores automatically on second nightly run"
echo " 3. $ICON_GEAR No manual config changes needed — markers self-clean ✅"
fi
echo ""
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — no changes made"
elif [[ ${#FAILED[@]} -gt 0 ]]; then
echo "$ICON_ERROR Status: completed with failures"
notify "Recreate shares failed on $(hostname) ($MY_ID) — failed: ${FAILED[*]}" \
"Recreate Shares" "warning"
exit 1
else
log "$ICON_DONE Status: done — ${#CREATED[@]} created, ${#SKIPPED[@]} skipped"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
Tools/watchdog_skip_list_manager.sh
+160 -76
View File
@@ -1,175 +1,259 @@
#!/bin/bash
# -----------------------------------------------------------------------------------------------
# --------------------------------- Watchdog Skip List Manager ---------------------------------
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# =========================== Watchdog Skip List Manager =======================================
# ==============================================================================================
# View and manage the persistent container skip list used by docker_watchdog.sh.
# Containers are added to the skip list when they exceed the restart loop limit.
# They stay there until manually cleared or until found running again automatically.
#
# Usage:
# watchdog_skip_list_manager.sh --status — show current skip list and restart history
# watchdog_skip_list_manager.sh --clear-all — clear all skip lists and restart history
# watchdog_skip_list_manager.sh --clear ContainerName — clear specific container
# ── WHAT THE SKIP LIST IS ─────────────────────────────────────────────────────────────────────
# docker_watchdog.sh adds a container to the skip list when it exceeds the restart loop
# limit (WATCHDOG_CONTAINER_RESTART_LIMIT in WATCHDOG_CONTAINER_RESTART_WINDOW hours).
# Once on the skip list the watchdog stops restarting it — prevents infinite restart loops.
#
# After clearing a container from the skip list:
# Skip list persists on /boot/config — survives reboots.
# Auto-clears when docker_watchdog.sh sees the container running on a cycle.
# This script clears it manually when you have fixed the underlying problem.
#
# ── ACTIONS ───────────────────────────────────────────────────────────────────────────────────
# --status — show skip list, container states, restart history
# --clear ContainerName — clear a specific container from skip list + history
# --clear-all — clear all skip lists and restart history
#
# ── AFTER CLEARING ────────────────────────────────────────────────────────────────────────────
# 1. Fix whatever was causing the container to fail
# 2. Start the container manually: docker start ContainerName
# 3. The watchdog will monitor it normally on the next cycle
# 2. Start it manually: docker start ContainerName
# 3. docker_watchdog.sh monitors it normally on the next cycle
# 4. If it crashes again → watchdog adds it back and notifies
#
# Files managed:
# SYS_WATCHDOG_FAILED_FILE — persistent container skip list
# WATCHDOG_CONTAINER_RESTART_LOG — restart history for loop detection
# -----------------------------------------------------------------------------------------------
# ── SKIP LIST AUTO-CLEAR ──────────────────────────────────────────────────────────────────────
# docker_watchdog.sh auto-clears a container from the skip list when it sees it running.
# So if a container recovers on its own (Docker restart policy eventually works),
# the watchdog will see it running, remove it from the skip list, and resume monitoring.
# Manual clear only needed when container is stuck stopped and needs intervention.
#
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
# acquire_lock — prevents concurrent access with docker_watchdog.sh writing files
# docker_watchdog check — warns if watchdog is running during clear (could re-add instantly)
# DOCKER_TIMEOUT — docker inspect calls protected against daemon hangs
# Confirmation required — interactive: YES | non-interactive: --force flag
# validate_unraid_cmd — notify validated before use
#
# ── FILES MANAGED ─────────────────────────────────────────────────────────────────────────────
# SYS_WATCHDOG_FAILED_FILE — persistent container skip list
# WATCHDOG_CONTAINER_RESTART_LOG — restart history for loop detection
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
# watchdog_skip_list_manager.sh — show status
# watchdog_skip_list_manager.sh --status — show status explicitly
# watchdog_skip_list_manager.sh --clear ContainerName — clear specific container
# watchdog_skip_list_manager.sh --clear-all — clear everything
# Any action supports --dry-run and --force
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../Master.conf"
source "$SCRIPT_DIR/../common.sh"
source "$SCRIPT_DIR/../load_config.sh"
parse_args "$@"
DOCKER_TIMEOUT=15
# Parse action from args
ACTION=""
# ── Parse action flags before parse_args ──────────────────────────────────────────────────────
ACTION="status"
TARGET_CONTAINER=""
FORCE=false
FILTERED_ARGS=()
for arg in "${PARSED_ARGS[@]}"; do
for arg in "$@"; do
case "$arg" in
--clear-all) ACTION="clear-all" ;;
--clear) ACTION="clear" ;;
--status) ACTION="status" ;;
--force) FORCE=true ;;
*)
[[ "$ACTION" == "clear" && -z "$TARGET_CONTAINER" ]] && TARGET_CONTAINER="$arg"
if [[ "$ACTION" == "clear" && -z "$TARGET_CONTAINER" ]]; then
TARGET_CONTAINER="$arg"
else
FILTERED_ARGS+=("$arg")
fi
;;
esac
done
[[ -z "$ACTION" ]] && ACTION="status"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Setup ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_GEAR Setup ━━━"
parse_args "${FILTERED_ARGS[@]}"
# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
error "Must be run as root"
exit 1
fi
success "Running as root"
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
validate_unraid_cmd \
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
"" "" \
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
acquire_lock
# detect_hosts() sets MY_ID — used in output
detect_hosts
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
[[ "$FORCE" == true ]] && warn "FORCE mode — confirmation prompt skipped"
# Ensure state files exist
touch "$SYS_WATCHDOG_FAILED_FILE" "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null
# -----------------------------------------------------------------------------------------------
# ━━━ STATUS ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Status — always shown regardless of action ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_WATCHDOG Skip List Status ━━━"
echo "━━━ $ICON_WATCHDOG Skip List Status — $MY_ID ━━━"
SKIP_COUNT=$(grep -c "." "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null || echo 0)
SKIP_COUNT="${SKIP_COUNT//[^0-9]/}"; SKIP_COUNT="${SKIP_COUNT:-0}"
RESTART_COUNT=$(wc -l < "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null || echo 0)
RESTART_COUNT="${RESTART_COUNT//[^0-9]/}"; RESTART_COUNT="${RESTART_COUNT:-0}"
# docker_watchdog.sh running check
WATCHDOG_RUNNING=false
if pgrep -f "docker_watchdog.sh" >/dev/null 2>&1; then
WATCHDOG_RUNNING=true
warn "docker_watchdog.sh is currently RUNNING"
[[ "$ACTION" != "status" ]] && \
warn "Clearing during an active cycle — watchdog may re-add container on next iteration"
fi
echo ""
if [[ "$SKIP_COUNT" -eq 0 ]]; then
success "Skip list is empty — all containers healthy"
log "Skip list: empty — all containers monitored normally ✅"
else
warn "$SKIP_COUNT container(s) on skip list:"
warn "$SKIP_COUNT container(s) on skip list — manual intervention needed:"
echo ""
while IFS= read -r container; do
[[ -z "$container" ]] && continue
# Check if container is currently running
STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null || echo "unknown")
if [[ "$STATUS" == "true" ]]; then
echo " $ICON_RUNNING $container — currently RUNNING (will auto-clear on next watchdog cycle)"
elif [[ "$STATUS" == "false" ]]; then
echo " $ICON_STOPPED $container — currently STOPPED — fix and start manually"
else
echo " $ICON_INFO $container — container not found"
fi
STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
'{{.State.Running}}' "$container" 2>/dev/null || echo "unknown")
case "$STATUS" in
true)
echo " $ICON_RUNNING $container — RUNNING (watchdog will auto-clear next cycle)"
;;
false)
echo " $ICON_NOT_RUNNING $container — STOPPED — fix and start manually"
;;
*)
echo " $ICON_WARN $container — not found on this server"
;;
esac
done < "$SYS_WATCHDOG_FAILED_FILE"
fi
echo ""
echo "━━━ $ICON_WATCHDOG Restart History ━━━"
if [[ "$RESTART_COUNT" -eq 0 ]]; then
success "No restart history"
log "No restart history"
else
info "$RESTART_COUNT restart entries (window: ${WATCHDOG_CONTAINER_RESTART_WINDOW}h)"
log "$RESTART_COUNT restart entries (window: ${WATCHDOG_CONTAINER_RESTART_WINDOW}h)"
echo ""
# Show per-container restart counts
awk -F'|' '{counts[$1]++} END {for (c in counts) printf " %-30s %d restart(s)\n", c, counts[c]}' \
"$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | sort
awk -F'|' '{counts[$1]++} END {
for (c in counts)
printf " %-30s %d restart(s)\n", c, counts[c]
}' "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | sort
fi
[[ "$ACTION" == "status" ]] && exit 0
# -----------------------------------------------------------------------------------------------
# ━━━ CLEAR ALL ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Clear All ━━━
# ==============================================================================================
if [[ "$ACTION" == "clear-all" ]]; then
echo ""
echo "━━━ $ICON_TRASH Clear All Skip Lists ━━━"
warn "This will clear the skip list and restart history for ALL containers"
echo ""
if [[ "$DRY_RUN" == false ]]; then
read -r -p "Type YES to confirm: " CONFIRM
if [[ "$CONFIRM" != "YES" ]]; then
info "Cancelled"
exit 0
if [[ "$FORCE" == true ]]; then
log "FORCE flag set — skipping confirmation"
elif [[ -t 0 ]]; then
read -r -p "Type YES to confirm: " CONFIRM
if [[ "$CONFIRM" != "YES" ]]; then
warn "Cancelled"
exit 0
fi
else
error "Non-interactive mode — use --force flag to skip confirmation"
exit 1
fi
> "$SYS_WATCHDOG_FAILED_FILE"
> "$WATCHDOG_CONTAINER_RESTART_LOG"
success "Skip list cleared"
success "Restart history cleared"
notify "Watchdog skip list manually cleared on $(hostname) — all containers will be monitored normally" "Watchdog Manager" "normal"
warn "Skip list cleared ✅"
warn "Restart history cleared ✅"
[[ "$WATCHDOG_RUNNING" == true ]] && \
warn "Note: watchdog is running — containers will be monitored on next cycle"
notify "Watchdog skip list cleared on $(hostname) ($MY_ID) — all containers will be monitored normally" \
"Watchdog Manager" "warning"
else
warn "DRY RUN — would clear: $SYS_WATCHDOG_FAILED_FILE"
warn "DRY RUN — would clear: $WATCHDOG_CONTAINER_RESTART_LOG"
fi
fi
# -----------------------------------------------------------------------------------------------
# ━━━ CLEAR SPECIFIC CONTAINER ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Clear Specific Container ━━━
# ==============================================================================================
if [[ "$ACTION" == "clear" ]]; then
echo ""
echo "━━━ $ICON_TRASH Clear Container: $TARGET_CONTAINER ━━━"
if [[ -z "$TARGET_CONTAINER" ]]; then
error "No container specified. Usage: --clear ContainerName"
error "No container specified"
error "Usage: watchdog_skip_list_manager.sh --clear ContainerName"
exit 1
fi
# Remove from skip list
if ! grep -q "^${TARGET_CONTAINER}$" "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null; then
warn "$TARGET_CONTAINER is not on the skip list"
else
if [[ "$DRY_RUN" == false ]]; then
sed -i "/^${TARGET_CONTAINER}$/d" "$SYS_WATCHDOG_FAILED_FILE"
success "$TARGET_CONTAINER removed from skip list"
warn "$TARGET_CONTAINER removed from skip list ✅"
else
warn "DRY RUN — would remove $TARGET_CONTAINER from skip list"
fi
fi
# Clear restart history for this container
HIST_COUNT=$(grep -c "^${TARGET_CONTAINER}|" "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null || echo 0)
HIST_COUNT=$(grep -c "^${TARGET_CONTAINER}|" \
"$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null || echo 0)
HIST_COUNT="${HIST_COUNT//[^0-9]/}"; HIST_COUNT="${HIST_COUNT:-0}"
if [[ "$HIST_COUNT" -gt 0 ]]; then
if [[ "$DRY_RUN" == false ]]; then
sed -i "/^${TARGET_CONTAINER}|/d" "$WATCHDOG_CONTAINER_RESTART_LOG"
success "Cleared $HIST_COUNT restart history entries for $TARGET_CONTAINER"
warn "Cleared $HIST_COUNT restart history entries for $TARGET_CONTAINER ✅"
else
warn "DRY RUN — would clear $HIST_COUNT restart history entries"
fi
else
info "No restart history for $TARGET_CONTAINER"
log "No restart history for $TARGET_CONTAINER"
fi
echo ""
echo "$ICON_INFO Next steps:"
echo " 1. Fix whatever was causing $TARGET_CONTAINER to fail"
echo " 2. Start it manually: docker start $TARGET_CONTAINER"
echo " 3. Watchdog will monitor it normally on the next cycle"
[[ "$WATCHDOG_RUNNING" == true ]] && \
warn "Note: watchdog is running — $TARGET_CONTAINER may be re-added if still failing"
if [[ "$DRY_RUN" == false ]]; then
echo ""
echo "━━━ $ICON_INFO Next Steps ━━━"
echo " 1. Fix whatever was causing $TARGET_CONTAINER to fail"
echo " 2. Start it manually: docker start $TARGET_CONTAINER"
echo " 3. docker_watchdog.sh monitors it on the next cycle"
echo " 4. If it crashes again → watchdog adds it back and notifies"
notify "$TARGET_CONTAINER cleared from watchdog skip list on $(hostname) ($MY_ID)" \
"Watchdog Manager" "warning"
fi
fi
echo ""
echo "━━━━━ $ICON_SUMMARY DONE ━━━━━"
echo "━━━━━ $ICON_SUMMARY DONE — $MY_ID ━━━━━"
Tools/zfs_pool_scrub.sh
+175 -91
View File
@@ -1,84 +1,128 @@
#!/bin/bash
# -----------------------------------------------------------------------------------------------
# --------------------------------- ZFS Pool Scrub ---------------------------------------------
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ================================= ZFS Pool Scrub ============================================
# ==============================================================================================
# Triggers a ZFS scrub on all pools (or a specific pool) and waits for completion.
# Sends a notification when scrub completes with a summary of any errors found.
#
# ZFS scrub reads every block on every pool and verifies checksums — it catches
# silent data corruption that would otherwise only surface when you try to read
# the corrupted data. Running monthly is recommended for all ZFS pools.
# ── WHAT ZFS SCRUB DOES ───────────────────────────────────────────────────────────────────────
# Reads every block on every pool and verifies checksums against the stored hash.
# Catches silent data corruption that would otherwise only surface when you read the
# corrupted data — by then it may be too late for redundancy to help.
#
# Usage:
# zfs_pool_scrub.sh — scrub all pools
# zfs_pool_scrub.sh poolname — scrub specific pool only
# zfs_pool_scrub.sh --status — show scrub status for all pools
# zfs_pool_scrub.sh --dry-run — show what would be scrubbed
# Scrub is safe to run while the pool is in use — it does not interrupt normal I/O.
# It does consume I/O bandwidth — run during off-peak hours or maintenance windows.
# Monthly is recommended for all pools. Quarterly minimum for large pools.
#
# Pools in ZFS_REPORT_IGNORE_POOLS are skipped unless specified explicitly.
# Scrub runs in background — script polls until complete then reports.
# Safe to run while the pool is in use — scrub does not interrupt normal I/O.
# -----------------------------------------------------------------------------------------------
# ── BEHAVIOUR ─────────────────────────────────────────────────────────────────────────────────
# Starts scrub on each pool then polls every 60 seconds until all complete.
# Progress shown via warn() every poll (visible) when scrub is running.
# Safe to leave running or interrupt — scrub continues even if script is stopped.
# On completion reports errors per pool and notifies if any found.
#
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
# detect_hosts() sets MY_ID and aliases HOST*_ZFS_REPORT_IGNORE_POOLS → ZFS_REPORT_IGNORE_POOLS.
# Pools in ZFS_REPORT_IGNORE_POOLS are skipped (single-disk VMs, temp pools etc.)
# unless specified explicitly as a positional argument.
#
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
# acquire_lock — prevents concurrent scrub starts on same server
# detect_hosts() — correct pool ignore list per host
# validate_unraid_cmd — zpool and notify validated before use
# Scrub-in-progress check — skips pools already scrubbing rather than erroring
# SIGTERM trap — poll loop exits cleanly on signal
# Silent when clean — only errors produce visible output and notification
#
# ── CONFIGURATION (master_host*.conf) ─────────────────────────────────────────────────────────
# HOST*_ZFS_REPORT_IGNORE_POOLS — pools excluded from automatic scrub
# Aliased by detect_hosts() — script uses ZFS_REPORT_IGNORE_POOLS
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
# zfs_pool_scrub.sh — scrub all non-ignored pools
# zfs_pool_scrub.sh poolname — scrub specific pool (bypasses ignore list)
# zfs_pool_scrub.sh --status — show scrub status for all pools
# zfs_pool_scrub.sh --dry-run — show what would be scrubbed
# zfs_pool_scrub.sh --log — verbose progress output
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../Master.conf"
source "$SCRIPT_DIR/../common.sh"
source "$SCRIPT_DIR/../load_config.sh"
parse_args "$@"
TARGET_POOL="${PARSED_ARGS[0]:-}"
SCRUB_RUNNING=true
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Setup ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_GEAR Setup ━━━"
# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
error "Must be run as root"
exit 1
fi
success "Running as root"
validate_unraid_cmd \
"$(command -v zpool 2>/dev/null || echo /sbin/zpool)" \
"--version" "" \
"zpool" || {
error "ZFS not available on this system — zpool not found"
exit 1
}
validate_unraid_cmd \
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
"" "" \
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
acquire_lock
if ! command -v zpool >/dev/null 2>&1; then
error "ZFS not available on this system"
exit 1
fi
# detect_hosts() sets MY_ID and aliases HOST*_ZFS_REPORT_IGNORE_POOLS
detect_hosts
success "ZFS available"
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no scrubs will be started"
# Build ignore map
# Build ignore pool map
declare -A IGNORE_MAP
for pool in "${ZFS_REPORT_IGNORE_POOLS[@]}"; do
for pool in "${ZFS_REPORT_IGNORE_POOLS[@]:-}"; do
[[ -n "$pool" ]] && IGNORE_MAP["$pool"]=1
done
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Status ━━━
# -----------------------------------------------------------------------------------------------
log "Identity: $MY_ID ($LOCAL_SERVER_NAME)"
log "Ignore pools: ${ZFS_REPORT_IGNORE_POOLS[*]:-none}"
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no scrubs will be started"
# SIGTERM trap — exit poll loop cleanly
trap 'warn "ZFS scrub script interrupted — scrub continues in background"; SCRUB_RUNNING=false; exit 0' \
SIGTERM SIGINT
# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
echo ""
echo "━━━━━ $ICON_SUMMARY SCRUB STATUS ━━━━━"
zpool list -H -o name 2>/dev/null | while read -r pool; do
echo "━━━━━ $ICON_SUMMARY ZFS SCRUB STATUS ━━━━━"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo ""
while IFS= read -r pool; do
[[ -z "$pool" ]] && continue
SCAN=$(zpool status "$pool" 2>/dev/null | grep "scan:")
echo " $ICON_ZFS $pool — $SCAN"
done
IGNORED=""
[[ -n "${IGNORE_MAP[$pool]:-}" ]] && IGNORED=" (ignored)"
echo " $ICON_ZFS $pool${IGNORED} — ${SCAN:-no scan data}"
done < <(zpool list -H -o name 2>/dev/null)
echo ""
echo " Ignored pools: ${ZFS_REPORT_IGNORE_POOLS[*]:-none}"
echo "━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
fi
# -----------------------------------------------------------------------------------------------
# Build pool list to scrub
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ── Build pool list ────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
POOLS_TO_SCRUB=()
if [[ -n "$TARGET_POOL" ]]; then
# Specific pool requested — validate it exists
# Specific pool — bypass ignore list, validate exists
if ! zpool list "$TARGET_POOL" >/dev/null 2>&1; then
error "Pool not found: $TARGET_POOL"
exit 1
@@ -89,7 +133,7 @@ else
while IFS= read -r pool; do
[[ -z "$pool" ]] && continue
if [[ -n "${IGNORE_MAP[$pool]:-}" ]]; then
info "Skipping $pool (in ZFS_REPORT_IGNORE_POOLS)"
log "Skipping $pool (in ZFS_REPORT_IGNORE_POOLS)"
continue
fi
POOLS_TO_SCRUB+=("$pool")
@@ -97,99 +141,139 @@ else
fi
if [[ ${#POOLS_TO_SCRUB[@]} -eq 0 ]]; then
warn "No pools to scrub"
warn "No pools to scrub — all pools may be on the ignore list"
warn "Ignored: ${ZFS_REPORT_IGNORE_POOLS[*]:-none}"
exit 0
fi
info "Pools to scrub: ${POOLS_TO_SCRUB[*]}"
log "Pools to scrub: ${POOLS_TO_SCRUB[*]}"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_ZFS Start Scrubs ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Start Scrubs ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_ZFS Starting ZFS Scrubs ━━━"
echo "━━━ $ICON_ZFS Starting ZFS Scrubs — $MY_ID ━━━"
START=$(date +%s)
STARTED=()
SKIPPED_POOLS=()
for pool in "${POOLS_TO_SCRUB[@]}"; do
info "$ICON_ZFS Starting scrub on $pool..."
if [[ "$DRY_RUN" == false ]]; then
zpool scrub "$pool" 2>/dev/null && \
success "$pool scrub started" || \
error "Failed to start scrub on $pool"
else
# Check if scrub already in progress
ALREADY=$(zpool status "$pool" 2>/dev/null | grep "scan:" | grep -c "in progress" || true)
if [[ "$ALREADY" -gt 0 ]]; then
warn "$pool — scrub already in progress — joining existing scrub"
STARTED+=("$pool")
continue
fi
log "Starting scrub on $pool..."
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would scrub: $pool"
STARTED+=("$pool")
elif zpool scrub "$pool" 2>/dev/null; then
log "$pool scrub started ✅"
STARTED+=("$pool")
else
error "Failed to start scrub on $pool"
SKIPPED_POOLS+=("$pool")
fi
done
[[ "$DRY_RUN" == true ]] && {
if [[ "$DRY_RUN" == true ]]; then
echo ""
echo "━━━━━ $ICON_SUMMARY ZFS SCRUB SUMMARY ━━━━━"
echo "$ICON_WARN Status: DRY RUN — no scrubs started"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
warn "DRY RUN — no scrubs started"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
}
fi
# -----------------------------------------------------------------------------------------------
# ━━━ Poll until complete ━━━
# -----------------------------------------------------------------------------------------------
if [[ ${#STARTED[@]} -eq 0 ]]; then
error "No scrubs were started — check pool status"
exit 1
fi
# ==============================================================================================
# ━━━ Poll Until Complete ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_TIME Waiting for scrubs to complete ━━━"
info "Polling every 60 seconds — this may take a while on large pools"
info "Safe to leave running — scrub continues even if this script is stopped"
echo "━━━ $ICON_TIME Waiting for Scrubs to Complete ━━━"
log "Polling every 60 seconds — scrubs may take hours on large pools"
log "Safe to interrupt — scrubs continue in background if script is stopped"
STILL_RUNNING=true
while [[ "$STILL_RUNNING" == true ]]; do
while [[ "$SCRUB_RUNNING" == true ]]; do
sleep 60
STILL_RUNNING=false
for pool in "${POOLS_TO_SCRUB[@]}"; do
STATUS=$(zpool status "$pool" 2>/dev/null | grep "scan:" | grep -c "in progress" || true)
if [[ "$STATUS" -gt 0 ]]; then
for pool in "${STARTED[@]}"; do
IN_PROGRESS=$(zpool status "$pool" 2>/dev/null | \
grep "scan:" | grep -c "in progress" || true)
if [[ "$IN_PROGRESS" -gt 0 ]]; then
STILL_RUNNING=true
REPAIRED=$(zpool status "$pool" 2>/dev/null | grep "scan:" | grep -oE "[0-9]+ repaired")
log "$pool — scrub in progress ${REPAIRED:+($REPAIRED)}"
# Show progress — always visible so user knows it's running
PROGRESS=$(zpool status "$pool" 2>/dev/null | \
grep "scan:" | grep -oE "[0-9]+\.[0-9]+% done")
REPAIRED=$(zpool status "$pool" 2>/dev/null | \
grep "scan:" | grep -oE "[0-9]+ repaired")
warn "$pool — scrub in progress ${PROGRESS:+$PROGRESS}${REPAIRED:+ ($REPAIRED)}"
fi
done
[[ "$STILL_RUNNING" == false ]] && SCRUB_RUNNING=false
done
END=$(date +%s)
success "All scrubs complete"
warn "All scrubs complete — $(format_duration $(( END - START )))"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Results ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Results ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_ZFS Scrub Results ━━━"
POOLS_OK=()
POOLS_ERRORS=()
for pool in "${POOLS_TO_SCRUB[@]}"; do
for pool in "${STARTED[@]}"; do
SCAN_LINE=$(zpool status "$pool" 2>/dev/null | grep "scan:")
ERRORS=$(zpool status "$pool" 2>/dev/null | grep "errors:" | grep -v "No known data errors")
ERRORS=$(zpool status "$pool" 2>/dev/null | \
grep "errors:" | grep -v "No known data errors")
if [[ -n "$ERRORS" ]]; then
error "$pool — $SCAN_LINE"
error "$pool — $ERRORS"
error "$pool — ERRORS FOUND"
error " $SCAN_LINE"
error " $ERRORS"
POOLS_ERRORS+=("$pool")
else
success "$pool — $SCAN_LINE"
log "$pool — $SCAN_LINE"
POOLS_OK+=("$pool")
fi
done
# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
echo ""
echo "━━━━━ $ICON_SUMMARY ZFS SCRUB SUMMARY ━━━━━"
echo "$ICON_ZFS Pools scrubbed: ${#POOLS_TO_SCRUB[@]}"
echo "$ICON_SUCCESS Clean: ${#POOLS_OK[@]}"
echo "$ICON_ERROR Errors: ${#POOLS_ERRORS[@]}"
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_ZFS Pools: ${#POOLS_TO_SCRUB[@]} to scrub"
echo "$ICON_SUCCESS Clean: ${#POOLS_OK[@]}"
[[ ${#POOLS_ERRORS[@]} -gt 0 ]] && echo "$ICON_ERROR Errors: ${#POOLS_ERRORS[@]}"
[[ ${#SKIPPED_POOLS[@]} -gt 0 ]] && warn "Failed start: ${SKIPPED_POOLS[*]}"
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
echo ""
if [[ ${#POOLS_ERRORS[@]} -gt 0 ]]; then
echo "$ICON_ERROR Status: ERRORS FOUND — ${POOLS_ERRORS[*]}"
notify "ZFS scrub complete on $(hostname) — ERRORS found in pools: ${POOLS_ERRORS[*]}" "ZFS Scrub" "warning"
else
echo "$ICON_DONE Status: $ICON_SUCCESS ALL POOLS CLEAN"
notify "ZFS scrub complete on $(hostname) — ${#POOLS_OK[@]} pools clean in $(format_duration $((END - START)))" "ZFS Scrub" "normal"
notify "ZFS scrub errors on $(hostname) ($MY_ID) — pools with errors: ${POOLS_ERRORS[*]}" \
"ZFS Scrub" "warning"
elif [[ ${#POOLS_OK[@]} -gt 0 ]]; then
log "$ICON_DONE Status: all ${#POOLS_OK[@]} pools clean ✅"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
[[ ${#POOLS_ERRORS[@]} -gt 0 ]] && exit 1
exit 0
Transcodes/README-Transcoding.md
+653 -285
View File
File diff suppressed because it is too large Load Diff
Transcodes/ramdisk_setup.sh
+168 -114
View File
@@ -1,79 +1,136 @@
#!/bin/bash
# -----------------------------------------------------------------------------------------------
# --------------------------------- Ramdisk Setup Script ---------------------------------------
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ================================= Ramdisk Setup ==============================================
# ==============================================================================================
# Creates a tmpfs ramdisk for Emby transcodes and points the transcode symlink at it.
# Run once at array start via User Scripts plugin — scheduled as "At Startup of Array".
# If ramdisk already mounted reports status and exits cleanly without remounting.
# If ramdisk is already mounted reports status and exits cleanly without remounting.
#
# Creates:
# RAMDISK_PATH — tmpfs mount point (fast, in-memory transcode location)
# TRANSCODE_SSD — SSD fallback directory (created if missing)
# TRANSCODE_LINK — symlink pointing at ramdisk by default
# ── WHAT IT CREATES ───────────────────────────────────────────────────────────────────────────
# RAMDISK_PATH — tmpfs mount point (in-memory transcode location)
# Size: HOST*_RAMDISK_SIZE (e.g. 8G) — must fit in available RAM
# TRANSCODE_SSD — SSD fallback directory (created if missing)
# transcode_manager.sh flips symlink here if ramdisk fills up
# TRANSCODE_LINK — symlink pointing at RAMDISK_PATH by default
# transcoding-temp/ — pre-created inside ramdisk so Emby always finds it there
# Without this Emby creates it at its own first-writable location
# which may be SSD even when symlink points at ramdisk
#
# All configuration in Master.conf under Transcode Manager section.
# Supports --dry-run to preview what would be created without making changes.
# -----------------------------------------------------------------------------------------------
# ── TRANSCODE_LINK SYMLINK ────────────────────────────────────────────────────────────────────
# Emby's transcode path is set to TRANSCODE_LINK in Emby config.
# transcode_manager.sh flips the symlink between RAMDISK_PATH and TRANSCODE_SSD at runtime
# based on ramdisk usage — Emby sessions automatically follow without restart.
# This script always sets the link to RAMDISK_PATH at array start (clean state).
#
# ── STATE FILE ────────────────────────────────────────────────────────────────────────────────
# Initialises /tmp/transcode_state.db with current target and flip tracking counters.
# /tmp resets on reboot — correct, transcode state is ephemeral.
#
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
# detect_hosts() sets MY_ID and aliases HOST*_RAMDISK_SIZE → RAMDISK_SIZE.
# HOST*_RAMDISK_SIZE, HOST*_RAMDISK_WARN_GB, HOST*_RAMDISK_LOW_GB must all be
# configured per host — different servers have different amounts of RAM available.
#
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
# Root check — mount and symlink require root
# acquire_lock — prevents duplicate runs at array start
# detect_hosts() — correct RAMDISK_SIZE per host
# Already mounted — exits cleanly without remounting (idempotent)
# validate_unraid_cmd — notify validated before use
# Silent on success — startup script runs every boot — no noise when healthy
#
# ── CONFIGURATION (master_host*.conf) ─────────────────────────────────────────────────────────
# HOST*_RAMDISK_SIZE — tmpfs size (e.g. 8G) — must change together with WARN_GB/LOW_GB
# HOST*_RAMDISK_WARN_GB — warn threshold in GB
# HOST*_RAMDISK_LOW_GB — flip to SSD threshold in GB
# HOST*_TRANSCODE_SSD — SSD fallback path
# HOST*_TRANSCODE_SERVERS — which servers run transcoding
# Aliased by detect_hosts()
#
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
# TRANSCODE_LINK — symlink path Emby uses as transcode directory
# TRANSCODE_CHMOD — permissions applied to ramdisk and fallback
# TRANSCODE_OWNER — owner applied (default nobody:users)
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
# ramdisk_setup.sh — normal setup (runs at array start)
# ramdisk_setup.sh --dry-run — preview without making changes
# ramdisk_setup.sh --status — show current ramdisk and symlink state
# ramdisk_setup.sh --log — verbose output
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../Master.conf"
source "$SCRIPT_DIR/../common.sh"
source "$SCRIPT_DIR/../load_config.sh"
parse_args "$@"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Setup ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_GEAR Setup ━━━"
# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
error "Must be run as root"
error "Must be run as root — mount and symlink require root"
exit 1
fi
success "Running as root"
validate_unraid_cmd \
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
"" "" \
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Status ━━━
# -----------------------------------------------------------------------------------------------
acquire_lock
# detect_hosts() sets MY_ID and aliases RAMDISK_SIZE, TRANSCODE_SSD etc.
detect_hosts
log "Identity: $MY_ID ($LOCAL_SERVER_NAME)"
log "Ramdisk: $RAMDISK_PATH ($RAMDISK_SIZE)"
log "Fallback: $TRANSCODE_SSD"
log "Symlink: $TRANSCODE_LINK"
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
echo ""
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
echo "$ICON_RAM Ramdisk path: $RAMDISK_PATH"
echo "$ICON_RAM Ramdisk size: $RAMDISK_SIZE"
echo "$ICON_LINK Transcode link: $TRANSCODE_LINK"
echo "$ICON_DISK SSD fallback: $TRANSCODE_SSD"
echo "$ICON_GEAR Owner: $TRANSCODE_OWNER"
echo "$ICON_GEAR Mode: $TRANSCODE_MODE"
echo "$ICON_GEAR Dry Run: $DRY_RUN"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_RAM Ramdisk path: $RAMDISK_PATH"
echo "$ICON_RAM Ramdisk size: $RAMDISK_SIZE"
echo "$ICON_RAM Warn at: ${RAMDISK_WARN_GB}GB"
echo "$ICON_RAM Flip at: ${RAMDISK_LOW_GB}GB"
echo "$ICON_DISK SSD fallback: $TRANSCODE_SSD"
echo "$ICON_LINK Symlink: $TRANSCODE_LINK"
echo "$ICON_GEAR Owner: $TRANSCODE_OWNER"
echo "$ICON_GEAR Mode: $TRANSCODE_CHMOD"
echo ""
# Show current mount state
if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
CURRENT_USAGE=$(df -BG "$RAMDISK_PATH" | awk 'NR==2 {print $3}')
CURRENT_AVAIL=$(df -BG "$RAMDISK_PATH" | awk 'NR==2 {print $4}')
echo "$ICON_RAM Current usage: $CURRENT_USAGE used / $CURRENT_AVAIL available"
USAGE=$(df -BG "$RAMDISK_PATH" | awk 'NR==2 {print $3}')
AVAIL=$(df -BG "$RAMDISK_PATH" | awk 'NR==2 {print $4}')
echo " $ICON_RAM Ramdisk: mounted — $USAGE used / $AVAIL available ✅"
else
echo "$ICON_RAM Ramdisk: not mounted"
echo " $ICON_RAM Ramdisk: NOT mounted"
fi
if [[ -L "$TRANSCODE_LINK" ]]; then
echo "$ICON_LINK Current target: $(readlink "$TRANSCODE_LINK")"
TARGET=$(readlink "$TRANSCODE_LINK")
echo " $ICON_LINK Symlink: $TRANSCODE_LINK → $TARGET"
else
echo "$ICON_LINK Symlink: not set"
echo " $ICON_LINK Symlink: not set"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
fi
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_RAM Ramdisk Setup ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Ramdisk ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_RAM Ramdisk Setup ━━━"
echo "━━━ $ICON_RAM Ramdisk — $MY_ID ━━━"
echo "$ICON_RAM Path: $RAMDISK_PATH"
echo "$ICON_RAM Size: $RAMDISK_SIZE"
echo ""
@@ -81,39 +138,39 @@ echo ""
START=$(date +%s)
SETUP_SUCCESS=true
# Check if ramdisk already mounted
if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
CURRENT_USAGE=$(df -BG "$RAMDISK_PATH" | awk 'NR==2 {print $3}')
CURRENT_AVAIL=$(df -BG "$RAMDISK_PATH" | awk 'NR==2 {print $4}')
info "$ICON_RAM Ramdisk already mounted — $CURRENT_USAGE used / $CURRENT_AVAIL available"
info "Skipping mount — verifying symlink and permissions"
USAGE=$(df -BG "$RAMDISK_PATH" | awk 'NR==2 {print $3}')
AVAIL=$(df -BG "$RAMDISK_PATH" | awk 'NR==2 {print $4}')
log "Ramdisk already mounted — $USAGE used / $AVAIL available"
log "Skipping mount — verifying symlink and permissions"
else
# Create mount point
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would create $RAMDISK_PATH"
warn "DRY RUN — would mount tmpfs ${RAMDISK_SIZE} at $RAMDISK_PATH"
else
info "Creating ramdisk mount point: $RAMDISK_PATH"
log "Creating ramdisk mount point: $RAMDISK_PATH"
mkdir -p "$RAMDISK_PATH" || {
error "Failed to create $RAMDISK_PATH"
notify "Ramdisk setup failed on $(hostname) — could not create mount point" "Ramdisk Setup" "warning"
notify "Ramdisk setup failed on $(hostname) ($MY_ID) — could not create mount point" \
"Ramdisk Setup" "warning"
exit 1
}
info "$ICON_RAM Mounting tmpfs ${RAMDISK_SIZE} at $RAMDISK_PATH..."
log "Mounting tmpfs ${RAMDISK_SIZE} at $RAMDISK_PATH..."
if mount -t tmpfs -o size="$RAMDISK_SIZE" tmpfs "$RAMDISK_PATH"; then
success "Ramdisk mounted — ${RAMDISK_SIZE} at $RAMDISK_PATH"
warn "Ramdisk mounted — ${RAMDISK_SIZE} at $RAMDISK_PATH ✅"
else
error "Failed to mount ramdisk at $RAMDISK_PATH"
notify "Ramdisk setup failed on $(hostname) — mount failed" "Ramdisk Setup" "warning"
notify "Ramdisk setup failed on $(hostname) ($MY_ID) — mount failed" \
"Ramdisk Setup" "warning"
exit 1
fi
fi
fi
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_DISK SSD Fallback ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ SSD Fallback ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_DISK SSD Fallback ━━━"
@@ -121,11 +178,11 @@ if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would create SSD fallback: $TRANSCODE_SSD"
else
if [[ -d "$TRANSCODE_SSD" ]]; then
info "$ICON_DISK SSD fallback already exists: $TRANSCODE_SSD"
log "SSD fallback already exists: $TRANSCODE_SSD"
else
info "Creating SSD fallback directory: $TRANSCODE_SSD"
log "Creating SSD fallback directory: $TRANSCODE_SSD"
if mkdir -p "$TRANSCODE_SSD"; then
success "SSD fallback created: $TRANSCODE_SSD"
log "SSD fallback created: $TRANSCODE_SSD ✅"
else
error "Failed to create SSD fallback: $TRANSCODE_SSD"
SETUP_SUCCESS=false
@@ -133,22 +190,21 @@ else
fi
fi
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_LINK Symlink ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Symlink ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_LINK Symlink ━━━"
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would set $TRANSCODE_LINK → $RAMDISK_PATH"
else
# Remove existing symlink or directory at link path
if [[ -L "$TRANSCODE_LINK" ]]; then
CURRENT_TARGET=$(readlink "$TRANSCODE_LINK")
if [[ "$CURRENT_TARGET" == "$RAMDISK_PATH" ]]; then
info "$ICON_LINK Symlink already points to ramdisk — no change needed"
log "Symlink already points to ramdisk — no change needed ✅"
else
info "$ICON_LINK Updating symlink: $CURRENT_TARGET → $RAMDISK_PATH"
log "Updating symlink: $CURRENT_TARGET → $RAMDISK_PATH"
ln -sfn "$RAMDISK_PATH" "$TRANSCODE_LINK" || {
error "Failed to update symlink"
SETUP_SUCCESS=false
@@ -162,8 +218,7 @@ else
SETUP_SUCCESS=false
}
else
info "Creating symlink: $TRANSCODE_LINK → $RAMDISK_PATH"
# Ensure parent directory exists
log "Creating symlink: $TRANSCODE_LINK → $RAMDISK_PATH"
mkdir -p "$(dirname "$TRANSCODE_LINK")"
ln -sfn "$RAMDISK_PATH" "$TRANSCODE_LINK" || {
error "Failed to create symlink"
@@ -171,19 +226,15 @@ else
}
fi
if [[ "$SETUP_SUCCESS" == true ]]; then
success "$ICON_LINK $TRANSCODE_LINK → $RAMDISK_PATH"
fi
[[ "$SETUP_SUCCESS" == true ]] && log "Symlink: $TRANSCODE_LINK → $RAMDISK_PATH ✅"
fi
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Transcoding-temp Directory ━━━
# Creates the transcoding-temp folder on the ramdisk proactively.
# If this folder doesn't exist on the ramdisk Emby creates it wherever it finds
# a writable path first — which may be the SSD fallback — locking all sessions
# onto SSD even when the symlink points at the ramdisk.
# Creating it here guarantees Emby always finds it on the ramdisk at session start.
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Transcoding-temp Directory ━━━
# ==============================================================================================
# Pre-created inside ramdisk so Emby always finds it there at session start.
# Without this Emby creates it at its own first-writable path — which may be
# SSD even when the symlink points at the ramdisk — locking all sessions onto SSD.
echo ""
echo "━━━ $ICON_GEAR Transcoding Temp Directory ━━━"
@@ -193,42 +244,44 @@ if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would create $TRANSCODE_TEMP_DIR"
else
if [[ -d "$TRANSCODE_TEMP_DIR" ]]; then
info "transcoding-temp already exists on ramdisk"
log "transcoding-temp already exists on ramdisk"
else
mkdir -p "$TRANSCODE_TEMP_DIR" && \
success "Created transcoding-temp on ramdisk: $TRANSCODE_TEMP_DIR" || \
{ error "Failed to create transcoding-temp on ramdisk"; SETUP_SUCCESS=false; }
if mkdir -p "$TRANSCODE_TEMP_DIR"; then
log "Created transcoding-temp on ramdisk ✅"
else
error "Failed to create transcoding-temp on ramdisk"
SETUP_SUCCESS=false
fi
fi
# Apply correct permissions so Emby (abc/nobody:users) can write to it
if [[ -d "$TRANSCODE_TEMP_DIR" ]]; then
chmod "$TRANSCODE_CHMOD" "$TRANSCODE_TEMP_DIR"
chown "$TRANSCODE_OWNER" "$TRANSCODE_TEMP_DIR"
success "Permissions set on transcoding-temp"
log "Permissions set on transcoding-temp ($TRANSCODE_CHMOD $TRANSCODE_OWNER)"
fi
fi
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Permissions ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Permissions ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_GEAR Permissions ━━━"
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would apply $TRANSCODE_MODE $TRANSCODE_OWNER to $RAMDISK_PATH and $TRANSCODE_SSD"
warn "DRY RUN — would apply $TRANSCODE_CHMOD $TRANSCODE_OWNER to $RAMDISK_PATH and $TRANSCODE_SSD"
else
for path in "$RAMDISK_PATH" "$TRANSCODE_SSD"; do
if [[ -d "$path" ]]; then
chmod "$TRANSCODE_MODE" "$path"
chmod "$TRANSCODE_CHMOD" "$path"
chown "$TRANSCODE_OWNER" "$path"
success "Permissions set: $path"
log "Permissions set: $path ($TRANSCODE_CHMOD $TRANSCODE_OWNER)"
fi
done
fi
# -----------------------------------------------------------------------------------------------
# Initialise state file
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Initialise State File ━━━
# ==============================================================================================
if [[ "$DRY_RUN" == false ]]; then
STATE_FILE="/tmp/transcode_state.db"
NOW=$(date +%s)
@@ -243,25 +296,26 @@ fi
END=$(date +%s)
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Summary ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
echo ""
echo "━━━━━ $ICON_SUMMARY RAMDISK SETUP SUMMARY ━━━━━"
echo "$ICON_RAM Ramdisk: $RAMDISK_PATH ($RAMDISK_SIZE)"
echo "$ICON_DISK Fallback: $TRANSCODE_SSD"
echo "$ICON_LINK Symlink: $TRANSCODE_LINK"
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
if [[ "$DRY_RUN" == true ]]; then
echo "$ICON_WARN Status: DRY RUN — no changes made"
elif [[ "$SETUP_SUCCESS" == true ]]; then
echo "$ICON_DONE Status: $ICON_SUCCESS DONE"
notify "Ramdisk setup complete on $(hostname) — ${RAMDISK_SIZE} mounted at $RAMDISK_PATH" "Ramdisk Setup" "normal"
else
echo "$ICON_ERROR Status: $ICON_ERROR SETUP HAD ERRORS"
notify "Ramdisk setup completed with errors on $(hostname)" "Ramdisk Setup" "warning"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_RAM Ramdisk: $RAMDISK_PATH ($RAMDISK_SIZE)"
echo "$ICON_DISK Fallback: $TRANSCODE_SSD"
echo "$ICON_LINK Symlink: $TRANSCODE_LINK"
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
echo ""
[[ "$SETUP_SUCCESS" == false ]] && exit 1
exit 0
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — no changes made"
elif [[ "$SETUP_SUCCESS" == true ]]; then
log "$ICON_DONE Status: done ✅"
else
echo "$ICON_ERROR Status: SETUP HAD ERRORS"
notify "Ramdisk setup errors on $(hostname) ($MY_ID) — check output" \
"Ramdisk Setup" "warning"
exit 1
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
Transcodes/transcode_cleanup.sh
+163 -119
View File
@@ -1,115 +1,178 @@
#!/bin/bash
# -----------------------------------------------------------------------------------------------
# --------------------------------- Transcode Cleanup ------------------------------------------
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ================================= Transcode Cleanup ==========================================
# ==============================================================================================
# Removes old inactive transcode files from both ramdisk and SSD fallback locations.
# Called every 5 minutes by transcode_manager.sh — must be fast and non-blocking.
# Never deletes files that are currently open by any process.
# After cleanup checks if ramdisk usage dropped enough to flip symlink back to ramdisk.
#
# Safety rules — a file is eligible for deletion only if ALL conditions are true:
# 1. Older than TRANSCODE_MAX_AGE minutes (mtime)
# 2. Not currently open by any process (single lsof call per location — not per file)
# ── SAFETY RULES ──────────────────────────────────────────────────────────────────────────────
# A file is eligible for deletion only if ALL conditions are true:
# 1. Older than TRANSCODE_MAX_AGE minutes (mtime — last modified time)
# 2. Not currently open by any process (checked via lsof pre-built map)
#
# Performance note:
# lsof is called ONCE per location to build an open file list — not once per file.
# This is critical for locations with hundreds or thousands of segment files.
# A per-file lsof approach stalls on busy systems with live TV buffering.
# ── WHY NOT SESSION-AWARE CLEANUP ─────────────────────────────────────────────────────────────
# ffmpeg generates folder names independently of the media server API session IDs.
# There is no reliable correlation between API session IDs and transcoding-temp subfolder
# names — matching them would falsely treat active sessions as ended.
# lsof is the correct and reliable active file check — if ffmpeg has a file open,
# lsof sees it regardless of folder naming or session state.
#
# Run every 5 minutes via cron/User Scripts plugin.
# All configuration in Master.conf under Transcode Manager section.
# Supports --dry-run to preview what would be deleted without making changes.
# -----------------------------------------------------------------------------------------------
# ── TRANSCODING-TEMP PROTECTION ───────────────────────────────────────────────────────────────
# The transcoding-temp directory is excluded from deletion even when empty.
# If cleanup removes the empty transcoding-temp folder from the ramdisk, Emby finds
# the SSD version instead and all new sessions land on SSD until Emby restarts.
# ! -name "transcoding-temp" exclusion in find prevents this permanently.
#
# ── PERFORMANCE ───────────────────────────────────────────────────────────────────────────────
# lsof is called ONCE per location — never once per file.
# Per-file lsof stalls on busy systems with live TV buffering hundreds of segments.
#
# Open file check uses in-memory associative array (OPEN_FILES_MAP):
# Was: echo "$OPEN_FILES" | grep -qF "$file" — O(n) per file → O(n²) total
# Now: [[ -n "${OPEN_FILES_MAP[$file]:-}" ]] — O(1) per file → O(n) total
# Same lesson as TRACKED_MAP in arr cleanup scripts.
#
# ── POST-CLEANUP SYMLINK FLIP ─────────────────────────────────────────────────────────────────
# After cleanup, if ramdisk has recovered below RAMDISK_LOW_GB and symlink currently
# points at SSD → triggers transcode_manager.sh to flip back to ramdisk.
#
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
# detect_hosts() sets MY_ID and aliases RAMDISK_PATH, TRANSCODE_SSD, RAMDISK_LOW_GB.
# Each server cleans its own transcode locations at the correct thresholds.
#
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
# acquire_lock "wait" — wait if previous cleanup still running
# detect_hosts() — correct paths and thresholds per host
# lsof timeout — lsof call capped at 15 seconds per location
# OPEN_FILES_MAP — in-memory O(1) active file lookup
# transcoding-temp guard — never deletes this directory
# Silent by default — runs every 5 minutes, must not produce noise when healthy
#
# ── CONFIGURATION (master_host*.conf) ─────────────────────────────────────────────────────────
# HOST*_RAMDISK_PATH / HOST*_TRANSCODE_SSD / HOST*_RAMDISK_LOW_GB
# Aliased by detect_hosts()
#
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
# TRANSCODE_MAX_AGE — minutes before an inactive transcode file is eligible
# TRANSCODE_ORPHAN_AGE — minutes for orphan detection (informational — future use)
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
# transcode_cleanup.sh — normal cleanup run
# transcode_cleanup.sh --dry-run — show what would be deleted
# transcode_cleanup.sh --status — show current state
# transcode_cleanup.sh --log — verbose per-file output
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../Master.conf"
source "$SCRIPT_DIR/../common.sh"
source "$SCRIPT_DIR/../load_config.sh"
parse_args "$@"
STATE_FILE="/tmp/transcode_state.db"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Setup ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_GEAR Setup ━━━"
# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
error "Must be run as root"
exit 1
fi
success "Running as root"
validate_unraid_cmd \
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
"" "" \
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
acquire_lock "wait"
if ! command -v lsof >/dev/null 2>&1; then
warn "lsof not available — active file check will be skipped, all aged files will be eligible"
LSOF_AVAILABLE=false
else
# detect_hosts() sets MY_ID and aliases RAMDISK_PATH, TRANSCODE_SSD, RAMDISK_LOW_GB
detect_hosts
# lsof availability check
LSOF_AVAILABLE=false
if command -v lsof >/dev/null 2>&1; then
LSOF_AVAILABLE=true
log "lsof available — active file check enabled"
else
warn "lsof not available — active file check skipped, all aged files eligible for deletion"
fi
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Status ━━━
# -----------------------------------------------------------------------------------------------
log "Identity: $MY_ID ($LOCAL_SERVER_NAME)"
log "Max age: ${TRANSCODE_MAX_AGE} minutes"
# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
echo ""
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
echo "$ICON_RAM Ramdisk: $RAMDISK_PATH"
echo "$ICON_DISK SSD fallback: $TRANSCODE_SSD"
echo "$ICON_TRASH Max age: ${TRANSCODE_MAX_AGE} minutes"
echo "$ICON_TRASH Orphan age: ${TRANSCODE_ORPHAN_AGE} minutes"
echo "$ICON_GEAR lsof check: $LSOF_AVAILABLE"
echo "$ICON_GEAR Dry Run: $DRY_RUN"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_RAM Ramdisk: $RAMDISK_PATH"
echo "$ICON_DISK SSD fallback: $TRANSCODE_SSD"
echo "$ICON_TRASH Max age: ${TRANSCODE_MAX_AGE} minutes"
echo "$ICON_TRASH Orphan age: ${TRANSCODE_ORPHAN_AGE} minutes"
echo "$ICON_RAM Flip at: ${RAMDISK_LOW_GB}GB"
echo "$ICON_GEAR lsof check: $LSOF_AVAILABLE"
echo "$ICON_GEAR Dry Run: $DRY_RUN"
echo ""
if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
USAGE=$(df -BG "$RAMDISK_PATH" | awk 'NR==2 {print $3}')
AVAIL=$(df -BG "$RAMDISK_PATH" | awk 'NR==2 {print $4}')
echo " $ICON_RAM Ramdisk: mounted — $USAGE used / $AVAIL available ✅"
else
echo " $ICON_RAM Ramdisk: not mounted"
fi
CURRENT_TARGET=$(grep "^current_target=" "$STATE_FILE" 2>/dev/null | cut -d= -f2)
echo " $ICON_LINK Current target: ${CURRENT_TARGET:-unknown}"
echo "━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
fi
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be deleted"
# -----------------------------------------------------------------------------------------------
# CLEANUP FUNCTION
# ==============================================================================================
# ── CLEANUP FUNCTION ──────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Scans a location and removes eligible files.
# Calls lsof ONCE per location to build open file list — never per file.
# -----------------------------------------------------------------------------------------------
# Calls lsof ONCE per location — builds in-memory OPEN_FILES_MAP for O(1) lookup.
# Returns via LOCATION_REMOVED, LOCATION_FREED, LOCATION_SKIPPED, LOCATION_ACTIVE
cleanup_location() {
local location="$1" label="$2" max_age="$3"
local files_removed=0
local bytes_freed=0
local files_skipped=0
local files_active=0
local files_removed=0 bytes_freed=0 files_skipped=0 files_active=0
if [[ ! -d "$location" ]]; then
warn "$label does not exist — skipping"
LOCATION_REMOVED=0
LOCATION_FREED="0B"
LOCATION_SKIPPED=0
log "$label does not exist — skipping"
LOCATION_REMOVED=0 LOCATION_FREED="0B" LOCATION_SKIPPED=0 LOCATION_ACTIVE=0
return
fi
local file_count
file_count=$(find "$location" -type f 2>/dev/null | wc -l)
info "$ICON_TRASH $label: $file_count files to scan (age threshold: ${max_age}min)"
log "$label: $file_count files to scan (age threshold: ${max_age}min)"
# Build open file list with a single lsof call — timeout prevents stalling
local OPEN_FILES=""
# Build in-memory open file map — O(1) lookup per file
# One lsof call per location — never per file
declare -A OPEN_FILES_MAP
if [[ "$LSOF_AVAILABLE" == true ]]; then
info "Building open file list for $label..."
OPEN_FILES=$(timeout 15 lsof +D "$location" 2>/dev/null | awk 'NR>1 {print $9}' | sort -u)
local open_count
open_count=$(echo "$OPEN_FILES" | grep -c "." 2>/dev/null || echo 0)
info "$open_count files currently open in $label"
log "Building open file map for $label..."
while IFS= read -r open_file; do
[[ -n "$open_file" ]] && OPEN_FILES_MAP["$open_file"]=1
done < <(timeout 15 lsof +D "$location" 2>/dev/null | awk 'NR>1 {print $9}' | sort -u)
log "${#OPEN_FILES_MAP[@]} files currently open in $label"
fi
# Find files older than max_age and process them
# Process aged files
while IFS= read -r file; do
[[ -z "$file" ]] && continue
# Check if file is currently open — fast string match against pre-built list
if [[ "$LSOF_AVAILABLE" == true ]] && echo "$OPEN_FILES" | grep -qF "$file"; then
((files_active++))
# O(1) open file check — in-memory map
if [[ -n "${OPEN_FILES_MAP[$file]:-}" ]]; then
(( files_active++ ))
log "Skipping open file: $file"
continue
fi
@@ -118,30 +181,29 @@ cleanup_location() {
file_size=$(stat -c%s "$file" 2>/dev/null || echo 0)
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would delete: $(basename "$file")"
((files_skipped++))
log "DRY RUN — would delete: $(basename "$file")"
(( files_skipped++ ))
else
if rm -f "$file" 2>/dev/null; then
((files_removed++))
bytes_freed=$((bytes_freed + file_size))
(( files_removed++ ))
bytes_freed=$(( bytes_freed + file_size ))
log "Deleted: $file"
else
warn "Could not delete: $file"
((files_skipped++))
(( files_skipped++ ))
fi
fi
done < <(find "$location" -type f -mmin +"$max_age" 2>/dev/null)
# Remove empty directories left behind — but NEVER remove transcoding-temp itself
# transcoding-temp must always exist on the ramdisk so Emby finds it there first
# If deleted Emby falls back to the SSD version and all new sessions land on SSD
# Remove empty directories — but NEVER remove transcoding-temp
# transcoding-temp must always exist on ramdisk so Emby finds it there first
if [[ "$DRY_RUN" == false ]]; then
find "$location" -mindepth 1 -type d -empty \
! -name "transcoding-temp" -delete 2>/dev/null
fi
# Format bytes freed for display
# Format bytes freed
local freed_human
if (( bytes_freed > 1073741824 )); then
freed_human=$(awk "BEGIN {printf \"%.1fGB\", $bytes_freed / 1073741824}")
@@ -153,11 +215,7 @@ cleanup_location() {
freed_human="0B"
fi
if [[ "$DRY_RUN" == true ]]; then
info "$label — dry run complete ($file_count files scanned, $files_active active)"
else
success "$label — removed $files_removed files ($freed_human freed), $files_active active, $files_skipped skipped"
fi
log "$label — removed $files_removed files ($freed_human freed) | active: $files_active | skipped: $files_skipped"
LOCATION_REMOVED=$files_removed
LOCATION_FREED=$freed_human
@@ -165,79 +223,65 @@ cleanup_location() {
LOCATION_ACTIVE=$files_active
}
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_TRASH Transcode Cleanup ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_TRASH Transcode Cleanup — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
echo ""
# ==============================================================================================
# ━━━ Transcode Cleanup ━━━
# ==============================================================================================
START=$(date +%s)
TOTAL_REMOVED=0
TOTAL_SKIPPED=0
TOTAL_ACTIVE=0
RAMDISK_FREED="0B"
SSD_FREED="0B"
TOTAL_REMOVED=0 TOTAL_SKIPPED=0 TOTAL_ACTIVE=0
RAMDISK_FREED="0B" SSD_FREED="0B"
# Cleanup ramdisk
if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
echo "━━━ $ICON_RAM Ramdisk ━━━"
cleanup_location "$RAMDISK_PATH" "Ramdisk" "$TRANSCODE_MAX_AGE"
TOTAL_REMOVED=$((TOTAL_REMOVED + LOCATION_REMOVED))
TOTAL_SKIPPED=$((TOTAL_SKIPPED + LOCATION_SKIPPED))
TOTAL_ACTIVE=$((TOTAL_ACTIVE + LOCATION_ACTIVE))
TOTAL_REMOVED=$(( TOTAL_REMOVED + LOCATION_REMOVED ))
TOTAL_SKIPPED=$(( TOTAL_SKIPPED + LOCATION_SKIPPED ))
TOTAL_ACTIVE=$(( TOTAL_ACTIVE + LOCATION_ACTIVE ))
RAMDISK_FREED=$LOCATION_FREED
echo ""
else
warn "$ICON_RAM Ramdisk not mounted — skipping ramdisk cleanup"
log "Ramdisk not mounted — skipping ramdisk cleanup"
fi
# Cleanup SSD fallback
if [[ -d "$TRANSCODE_SSD" ]]; then
echo "━━━ $ICON_DISK SSD Fallback ━━━"
cleanup_location "$TRANSCODE_SSD" "SSD fallback" "$TRANSCODE_MAX_AGE"
TOTAL_REMOVED=$((TOTAL_REMOVED + LOCATION_REMOVED))
TOTAL_SKIPPED=$((TOTAL_SKIPPED + LOCATION_SKIPPED))
TOTAL_ACTIVE=$((TOTAL_ACTIVE + LOCATION_ACTIVE))
TOTAL_REMOVED=$(( TOTAL_REMOVED + LOCATION_REMOVED ))
TOTAL_SKIPPED=$(( TOTAL_SKIPPED + LOCATION_SKIPPED ))
TOTAL_ACTIVE=$(( TOTAL_ACTIVE + LOCATION_ACTIVE ))
SSD_FREED=$LOCATION_FREED
echo ""
else
info "$ICON_DISK SSD fallback not found — skipping"
log "SSD fallback not found — skipping SSD cleanup"
fi
# -----------------------------------------------------------------------------------------------
# Post-cleanup — check if ramdisk recovered enough to flip symlink back
# -----------------------------------------------------------------------------------------------
# Post-cleanup — check if ramdisk recovered enough to flip symlink back to ramdisk
if [[ "$DRY_RUN" == false ]] && mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
RAMDISK_USED_KB=$(df "$RAMDISK_PATH" --output=used | tail -1 | tr -d ' ')
RAMDISK_USED_GB=$(awk "BEGIN {printf \"%.2f\", $RAMDISK_USED_KB / 1048576}")
RAMDISK_USED_KB=$(df "$RAMDISK_PATH" --output=used 2>/dev/null | tail -1 | tr -d ' ')
RAMDISK_USED_GB=$(awk "BEGIN {printf \"%.2f\", ${RAMDISK_USED_KB:-0} / 1048576}")
LOW_RECOVERED=$(awk "BEGIN {print ($RAMDISK_USED_GB < $RAMDISK_LOW_GB) ? 1 : 0}")
CURRENT_TARGET=$(grep "^current_target=" "$STATE_FILE" 2>/dev/null | cut -d'=' -f2)
CURRENT_TARGET=$(grep "^current_target=" "$STATE_FILE" 2>/dev/null | cut -d= -f2)
if [[ "$LOW_RECOVERED" == "1" && "$CURRENT_TARGET" == "$TRANSCODE_SSD" ]]; then
info "$ICON_RAM Ramdisk has space after cleanup — triggering manager to flip back"
log "Ramdisk has space after cleanup (${RAMDISK_USED_GB}GB < ${RAMDISK_LOW_GB}GB) — triggering manager to flip back"
bash "$SCRIPT_DIR/transcode_manager.sh" --no-log
fi
fi
END=$(date +%s)
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Summary ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
echo "━━━━━ $ICON_SUMMARY TRANSCODE CLEANUP SUMMARY ━━━━━"
echo "$ICON_RAM Ramdisk freed: $RAMDISK_FREED"
echo "$ICON_DISK SSD freed: $SSD_FREED"
echo "$ICON_TRASH Removed: $TOTAL_REMOVED files"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_RAM Ramdisk freed: $RAMDISK_FREED"
echo "$ICON_DISK SSD freed: $SSD_FREED"
echo "$ICON_TRASH Removed: $TOTAL_REMOVED files"
echo "$ICON_RUNNING Active: $TOTAL_ACTIVE files (open — protected)"
echo "$ICON_TRASH Skipped: $TOTAL_SKIPPED files"
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
echo "$ICON_TRASH Skipped: $TOTAL_SKIPPED files"
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
if [[ "$DRY_RUN" == true ]]; then
echo "$ICON_WARN Status: DRY RUN — no files deleted"
warn "DRY RUN — no files deleted"
else
echo "$ICON_DONE Status: $ICON_SUCCESS DONE"
if [[ "$TOTAL_REMOVED" -gt 0 ]]; then
notify "Transcode cleanup on $(hostname) — removed $TOTAL_REMOVED files (RAM: $RAMDISK_FREED SSD: $SSD_FREED)" "Transcode Cleanup" "normal"
fi
log "$ICON_DONE Status: done ✅"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
Transcodes/transcode_manager.sh
+253 -235
View File
@@ -1,63 +1,105 @@
#!/bin/bash
# -----------------------------------------------------------------------------------------------
# --------------------------------- Transcode Manager ------------------------------------------
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ================================= Transcode Manager ==========================================
# ==============================================================================================
# Manages Emby transcode storage using filesystem symlink indirection.
# ffmpeg resolves the symlink ONCE at session start — existing sessions are never affected.
# Only new sessions care about where the symlink currently points.
# Called every 5 minutes by User Scripts — must be fast, non-blocking, and silent when healthy.
#
# Three operating modes — set TRANSCODE_MANAGER_MODE in Master.conf:
# ── HOW IT WORKS ──────────────────────────────────────────────────────────────────────────────
# Emby's transcode path is set to TRANSCODE_LINK (a symlink).
# ffmpeg resolves the symlink ONCE at session start — existing sessions are never affected.
# Only NEW sessions care about where the symlink currently points.
# Flipping the symlink mid-stream is safe — in-progress transcodes continue uninterrupted.
#
# ── THREE MODES ───────────────────────────────────────────────────────────────────────────────
# smart — auto-flips between ramdisk and SSD based on usage thresholds (default)
# ramdisk — always uses ramdisk, never flips to SSD regardless of usage
# ramdisk above RAMDISK_WARN_GB → flip to SSD
# ramdisk below RAMDISK_LOW_GB → flip back to ramdisk
# ramdisk — always uses ramdisk, warns if above threshold, never flips
# ssd — always uses SSD, never uses ramdisk
#
# Safety checks on every run regardless of mode:
# Ramdisk disappeared → auto-flip to SSD, notify warning
# SSD path missing → disable SSD fallback, notify warning
# Symlink broken → auto-recreate, notify
# Symlink missing → auto-recreate, notify
# Permissions drift → fix silently
# Emby not running → skip threshold checks, verify symlink only
# ── SAFETY CHECKS — EVERY RUN ─────────────────────────────────────────────────────────────────
# Symlink missing/broken → auto-recreate pointing at ramdisk, notify
# Ramdisk disappeared → auto-flip to SSD, notify warning
# SSD path missing → disable SSD fallback / error if mode=ssd
# transcoding-temp missing → recreate on ramdisk immediately
# Permissions drift → fix silently
# Emby not running → skip threshold checks, verify symlink only
#
# Session display shows active Emby streams with storage state.
# Split state detected when sessions exist on both ramdisk and SSD simultaneously —
# ── SESSION DISPLAY ───────────────────────────────────────────────────────────────────────────
# Shows active Emby/Jellyfin/Plex streams with user, title, type, and play method.
# Split state shown when sessions exist on both ramdisk and SSD simultaneously —
# this happens naturally when symlink flips mid-session.
#
# All configuration in Master.conf under Transcode Manager section.
# Supports --dry-run and --status.
# -----------------------------------------------------------------------------------------------
# ── DAILY LOG ─────────────────────────────────────────────────────────────────────────────────
# Appends to TRANSCODE_DAILY_LOG after each run — read by weekly_health_digest.sh.
# Format: DATE|RAMDISK_USED_GB|FLIP_COUNT|RAM_SESSION_COUNT|SSD_SESSION_COUNT|FILES_CLEANED
#
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
# detect_hosts() sets MY_ID and aliases HOST*_TRANSCODE_SERVERS, HOST*_RAMDISK_PATH,
# HOST*_TRANSCODE_SSD, HOST*_RAMDISK_WARN_GB, HOST*_RAMDISK_LOW_GB, HOST*_RAMDISK_SIZE.
#
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
# acquire_lock "wait" — wait if previous run still active
# detect_hosts() — correct paths and thresholds per host
# DOCKER_TIMEOUT — all docker calls protected against daemon hangs
# validate_unraid_cmd — notify validated before use
# Silent by default — runs every 5 minutes, only speaks when something changes
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
# transcode_manager.sh — normal run
# transcode_manager.sh --dry-run — preview without making changes
# transcode_manager.sh --status — show current state and exit
# transcode_manager.sh --log — verbose output
# transcode_manager.sh --no-log — suppress daily log write (called by cleanup)
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../Master.conf"
source "$SCRIPT_DIR/../common.sh"
source "$SCRIPT_DIR/../load_config.sh"
parse_args "$@"
# Handle --no-log flag before parse_args
NO_LOG=false
FILTERED_ARGS=()
for arg in "$@"; do
case "$arg" in
--no-log) NO_LOG=true ;;
*) FILTERED_ARGS+=("$arg") ;;
esac
done
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Setup ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_GEAR Setup ━━━"
parse_args "${FILTERED_ARGS[@]}"
DOCKER_TIMEOUT=15
STATE_FILE="/tmp/transcode_state.db"
# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
error "Must be run as root"
exit 1
fi
success "Running as root"
validate_unraid_cmd \
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
"" "" \
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
acquire_lock "wait"
# Primary server for container running check — first entry in TRANSCODE_SERVERS
# Full session display loops over all entries in the array
# detect_hosts() sets MY_ID and aliases all HOST*_TRANSCODE_* vars
detect_hosts
# Primary container — first entry in TRANSCODE_SERVERS
TRANSCODE_EMBY_CONTAINER="Emby"
if [[ "${#TRANSCODE_SERVERS[@]}" -gt 0 ]]; then
IFS='|' read -r PRIMARY_CONTAINER _ _ _ <<< "${TRANSCODE_SERVERS[0]}"
TRANSCODE_EMBY_CONTAINER="${PRIMARY_CONTAINER:-Emby}"
fi
case "$TRANSCODE_MANAGER_MODE" in
smart|ramdisk|ssd) info "$ICON_GEAR Mode: $TRANSCODE_MANAGER_MODE" ;;
smart|ramdisk|ssd) log "Mode: $TRANSCODE_MANAGER_MODE" ;;
*)
warn "Unknown TRANSCODE_MANAGER_MODE: $TRANSCODE_MANAGER_MODE — defaulting to smart"
TRANSCODE_MANAGER_MODE="smart"
@@ -66,12 +108,13 @@ esac
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Status ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
echo ""
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
echo "━━━━━ $ICON_SUMMARY TRANSCODE MANAGER STATUS ━━━━━"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_GEAR Mode: $TRANSCODE_MANAGER_MODE"
echo "$ICON_RAM Ramdisk: $RAMDISK_PATH ($RAMDISK_SIZE)"
echo "$ICON_LINK Symlink: $TRANSCODE_LINK"
@@ -85,19 +128,19 @@ if [[ "$SHOW_STATUS" == true ]]; then
CURRENT_TARGET=$(readlink "$TRANSCODE_LINK" 2>/dev/null || echo "missing")
echo "$ICON_LINK Symlink now: $TRANSCODE_LINK → $CURRENT_TARGET"
if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
USED_KB=$(df "$RAMDISK_PATH" --output=used | tail -1 | tr -d ' ')
USED_GB=$(awk "BEGIN {printf \"%.2f\", $USED_KB / 1048576}")
USED_GB=$(df "$RAMDISK_PATH" --output=used 2>/dev/null | \
tail -1 | tr -d ' ' | awk '{printf "%.2f", $1/1048576}')
echo "$ICON_RAM Ramdisk now: ${USED_GB}GB used"
else
echo "$ICON_RAM Ramdisk: NOT MOUNTED"
echo "$ICON_RAM Ramdisk: NOT MOUNTED"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
fi
# -----------------------------------------------------------------------------------------------
# HELPERS
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ── HELPER FUNCTIONS ──────────────────────────────────────────────────────────────────────────
# ==============================================================================================
flip_symlink() {
local target="$1" reason="$2"
@@ -106,7 +149,7 @@ flip_symlink() {
return 0
fi
ln -sfn "$target" "$TRANSCODE_LINK"
success "$ICON_LINK Symlink flipped to: $target ($reason)"
warn "$ICON_LINK Symlink flipped to: $target ($reason)"
}
fix_permissions() {
@@ -118,30 +161,27 @@ fix_permissions() {
}
get_ramdisk_used_gb() {
local used_kb
used_kb=$(df "$RAMDISK_PATH" --output=used 2>/dev/null | tail -1 | tr -d ' ')
awk "BEGIN {printf \"%.2f\", ${used_kb:-0} / 1048576}"
df "$RAMDISK_PATH" --output=used 2>/dev/null | \
tail -1 | tr -d ' ' | awk '{printf "%.2f", $1/1048576}'
}
get_ramdisk_avail_gb() {
local avail_kb
avail_kb=$(df "$RAMDISK_PATH" --output=avail 2>/dev/null | tail -1 | tr -d ' ')
awk "BEGIN {printf \"%.2f\", ${avail_kb:-0} / 1048576}"
df "$RAMDISK_PATH" --output=avail 2>/dev/null | \
tail -1 | tr -d ' ' | awk '{printf "%.2f", $1/1048576}'
}
get_ssd_free_gb() {
local free_kb
free_kb=$(df "$TRANSCODE_SSD" --output=avail 2>/dev/null | tail -1 | tr -d ' ')
awk "BEGIN {printf \"%.2f\", ${free_kb:-0} / 1048576}"
df "$TRANSCODE_SSD" --output=avail 2>/dev/null | \
tail -1 | tr -d ' ' | awk '{printf "%.2f", $1/1048576}'
}
get_flip_count() {
local state_file="/tmp/transcode_flip_state.db"
local current_hour
current_hour=$(date '+%Y-%m-%d-%H')
if [[ ! -f "$state_file" ]]; then echo "0"; return; fi
[[ ! -f "$state_file" ]] && echo "0" && return
local stored_hour stored_count
stored_hour=$(awk -F'|' 'NR==1{print $1}' "$state_file" 2>/dev/null)
stored_hour=$(awk -F'|' 'NR==1{print $1}' "$state_file" 2>/dev/null)
stored_count=$(awk -F'|' 'NR==1{print $2}' "$state_file" 2>/dev/null)
[[ "$stored_hour" == "$current_hour" ]] && echo "${stored_count:-0}" || echo "0"
}
@@ -152,76 +192,55 @@ increment_flip_count() {
current_hour=$(date '+%Y-%m-%d-%H')
local current_count
current_count=$(get_flip_count)
current_count=$((current_count + 1))
current_count=$(( current_count + 1 ))
echo "${current_hour}|${current_count}" > "$state_file"
echo "$current_count"
}
emby_api() {
local endpoint="$1"
[[ -z "$EMBY_API_KEY" ]] && return 1
curl -sf \
--max-time 10 \
-H "X-Emby-Token: $EMBY_API_KEY" \
"${EMBY_URL}/${endpoint}" 2>/dev/null
}
get_media_type_label() {
local type="$1"
case "$type" in
LiveTv) echo "Live TV" ;;
TvChannel) echo "Live TV" ;;
Episode) echo "TV Show" ;;
Movie) echo "Movie" ;;
Audio) echo "Music" ;;
MusicVideo) echo "Music Video" ;;
*) echo "$type" ;;
case "$1" in
LiveTv|TvChannel) echo "Live TV" ;;
Episode) echo "TV Show" ;;
Movie) echo "Movie" ;;
Audio) echo "Music" ;;
MusicVideo) echo "Music Video" ;;
*) echo "$1" ;;
esac
}
get_play_method_label() {
local method="$1"
case "$method" in
Transcode) echo "Transcode" ;;
DirectStream) echo "Direct Stream" ;;
DirectPlay) echo "Direct Play" ;;
*) echo "$method" ;;
case "$1" in
Transcode) echo "Transcode" ;;
DirectStream) echo "Direct Stream" ;;
DirectPlay) echo "Direct Play" ;;
*) echo "$1" ;;
esac
}
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_RAM Transcode Manager ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_RAM Transcode Manager — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
echo "$ICON_GEAR Mode: $TRANSCODE_MANAGER_MODE"
echo ""
# ==============================================================================================
# ━━━ Transcode Manager ━━━
# ==============================================================================================
START=$(date +%s)
RAMDISK_HEALTHY=true
SSD_HEALTHY=true
EMBY_RUNNING=true
SOMETHING_HAPPENED=false # controls whether summary is printed
# -----------------------------------------------------------------------------------------------
# CHECK 1 — Emby running
# -----------------------------------------------------------------------------------------------
# ── Check 1 — Emby running ───────────────────────────────────────────────────────────────────
if [[ "$TRANSCODE_CHECK_EMBY" == true ]]; then
info "$ICON_CONTAINERS Checking $TRANSCODE_EMBY_CONTAINER..."
if ! docker inspect "$TRANSCODE_EMBY_CONTAINER" \
log "Checking $TRANSCODE_EMBY_CONTAINER..."
if ! timeout "$DOCKER_TIMEOUT" docker inspect "$TRANSCODE_EMBY_CONTAINER" \
--format '{{.State.Running}}' 2>/dev/null | grep -q "true"; then
warn "$TRANSCODE_EMBY_CONTAINER is not running — skipping threshold checks"
EMBY_RUNNING=false
SOMETHING_HAPPENED=true
else
success "$TRANSCODE_EMBY_CONTAINER is running"
log "$TRANSCODE_EMBY_CONTAINER is running ✅"
fi
fi
# -----------------------------------------------------------------------------------------------
# CHECK 2 — Symlink integrity
# -----------------------------------------------------------------------------------------------
echo ""
info "$ICON_LINK Checking symlink integrity..."
# ── Check 2 — Symlink integrity ───────────────────────────────────────────────────────────────
log "Checking symlink integrity..."
CURRENT_TARGET=$(readlink "$TRANSCODE_LINK" 2>/dev/null)
if [[ -z "$CURRENT_TARGET" ]]; then
@@ -229,175 +248,162 @@ if [[ -z "$CURRENT_TARGET" ]]; then
if [[ "$DRY_RUN" == false ]]; then
ln -sfn "$RAMDISK_PATH" "$TRANSCODE_LINK"
CURRENT_TARGET="$RAMDISK_PATH"
notify "Transcode symlink was missing on $(hostname) — recreated" "Transcode Manager" "warning"
notify "Transcode symlink was missing on $(hostname) ($MY_ID) — recreated" \
"Transcode Manager" "warning"
fi
SOMETHING_HAPPENED=true
elif [[ ! -e "$CURRENT_TARGET" ]]; then
warn "$ICON_LINK Symlink target missing: $CURRENT_TARGET — resetting to ramdisk"
if [[ "$DRY_RUN" == false ]]; then
ln -sfn "$RAMDISK_PATH" "$TRANSCODE_LINK"
CURRENT_TARGET="$RAMDISK_PATH"
notify "Transcode symlink target was missing on $(hostname) — reset to ramdisk" "Transcode Manager" "warning"
notify "Transcode symlink target was missing on $(hostname) ($MY_ID) — reset to ramdisk" \
"Transcode Manager" "warning"
fi
SOMETHING_HAPPENED=true
else
success "$ICON_LINK Symlink valid: $TRANSCODE_LINK → $CURRENT_TARGET"
log "Symlink valid: $TRANSCODE_LINK → $CURRENT_TARGET ✅"
fi
# -----------------------------------------------------------------------------------------------
# CHECK 3 — Ramdisk health
# -----------------------------------------------------------------------------------------------
echo ""
info "$ICON_RAM Checking ramdisk..."
# ── Check 3 — Ramdisk health ──────────────────────────────────────────────────────────────────
log "Checking ramdisk..."
if ! mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
error "$ICON_RAM Ramdisk not mounted at $RAMDISK_PATH"
error "Ramdisk not mounted at $RAMDISK_PATH"
RAMDISK_HEALTHY=false
SOMETHING_HAPPENED=true
if [[ "$DRY_RUN" == false ]]; then
warn "Flipping symlink to SSD — ramdisk unavailable"
flip_symlink "$TRANSCODE_SSD" "ramdisk disappeared"
CURRENT_TARGET="$TRANSCODE_SSD"
notify "Ramdisk disappeared on $(hostname) — transcodes falling back to SSD. Run ramdisk_setup.sh to restore." "Transcode Manager" "warning"
notify "Ramdisk disappeared on $(hostname) ($MY_ID) — transcodes falling back to SSD. Run ramdisk_setup.sh to restore." \
"Transcode Manager" "warning"
fi
else
RAMDISK_SIZE_ACTUAL=$(df "$RAMDISK_PATH" --output=size -h | tail -1 | tr -d ' ')
success "$ICON_RAM Ramdisk mounted — size: $RAMDISK_SIZE_ACTUAL"
RAMDISK_SIZE_ACTUAL=$(df "$RAMDISK_PATH" --output=size -h 2>/dev/null | tail -1 | tr -d ' ')
log "Ramdisk mounted — size: $RAMDISK_SIZE_ACTUAL ✅"
fix_permissions "$RAMDISK_PATH"
# Guarantee transcoding-temp exists on ramdisk
# If missing Emby creates it wherever it finds a writable path first —
# which may be the SSD fallback — locking all new sessions onto SSD
# even when the symlink correctly points at the ramdisk.
TRANSCODE_TEMP_RAM="${RAMDISK_PATH}/transcoding-temp"
TRANSCODE_TEMP_SSD="${TRANSCODE_SSD}/transcoding-temp"
if [[ ! -d "$TRANSCODE_TEMP_RAM" ]]; then
warn "transcoding-temp missing from ramdisk — creating now"
mkdir -p "$TRANSCODE_TEMP_RAM"
chmod "$TRANSCODE_CHMOD" "$TRANSCODE_TEMP_RAM"
chown "$TRANSCODE_OWNER" "$TRANSCODE_TEMP_RAM"
success "transcoding-temp created on ramdisk — new sessions will use ramdisk"
warn "transcoding-temp created on ramdisk — new sessions will use ramdisk ✅"
SOMETHING_HAPPENED=true
else
log "transcoding-temp exists on ramdisk — ok"
log "transcoding-temp exists on ramdisk ✅"
fi
fi
# -----------------------------------------------------------------------------------------------
# CHECK 4 — SSD health
# -----------------------------------------------------------------------------------------------
echo ""
info "$ICON_DISK Checking SSD fallback..."
# ── Check 4 — SSD health ─────────────────────────────────────────────────────────────────────
log "Checking SSD fallback..."
if [[ ! -d "$TRANSCODE_SSD" ]]; then
warn "$ICON_DISK SSD fallback path missing: $TRANSCODE_SSD"
warn "SSD fallback path missing: $TRANSCODE_SSD"
SSD_HEALTHY=false
SOMETHING_HAPPENED=true
if [[ "$TRANSCODE_MANAGER_MODE" == "ssd" ]]; then
error "Mode is 'ssd' but SSD path is missing — cannot continue"
notify "Transcode SSD path missing on $(hostname) — mode is 'ssd', manual intervention needed" "Transcode Manager" "warning"
notify "Transcode SSD path missing on $(hostname) ($MY_ID) — mode is 'ssd', manual intervention needed" \
"Transcode Manager" "warning"
exit 1
else
warn "SSD fallback disabled — will stay on ramdisk"
fi
else
SSD_FREE_GB=$(get_ssd_free_gb)
success "$ICON_DISK SSD available — ${SSD_FREE_GB}GB free"
log "SSD available — ${SSD_FREE_GB}GB free ✅"
fix_permissions "$TRANSCODE_SSD"
fi
# -----------------------------------------------------------------------------------------------
# USAGE STATS
# -----------------------------------------------------------------------------------------------
echo ""
# ── Usage stats ───────────────────────────────────────────────────────────────────────────────
RAMDISK_USED_GB="0.00"
RAMDISK_AVAIL_GB="0.00"
RAMDISK_FILES=0
if [[ "$RAMDISK_HEALTHY" == true ]]; then
RAMDISK_USED_GB=$(get_ramdisk_used_gb)
RAMDISK_AVAIL_GB=$(get_ramdisk_avail_gb)
RAMDISK_FILES=$(find "$RAMDISK_PATH" -type f 2>/dev/null | wc -l)
RAMDISK_SIZE_H=$(du -sh "$RAMDISK_PATH" 2>/dev/null | cut -f1)
fi
SSD_FILES=$(find "$TRANSCODE_SSD" -type f 2>/dev/null | wc -l)
SSD_SIZE_H=$(du -sh "$TRANSCODE_SSD" 2>/dev/null | cut -f1)
info "$ICON_RAM Ramdisk usage: ${RAMDISK_USED_GB}GB used / ${RAMDISK_AVAIL_GB}GB available"
info "$ICON_RAM Ramdisk files: ${RAMDISK_FILES:-0} ($RAMDISK_SIZE_H)"
info "$ICON_DISK SSD files: $SSD_FILES ($SSD_SIZE_H)"
FLIP_COUNT=$(get_flip_count)
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_EMBY Active Transcode Sessions ━━━
# Queries all configured servers in TRANSCODE_SERVERS array
# Aggregates sessions, counts, and storage state across all servers
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_EMBY Active Transcode Sessions ━━━"
log "Ramdisk: ${RAMDISK_USED_GB}GB used / ${RAMDISK_AVAIL_GB}GB available ($RAMDISK_FILES files)"
log "SSD: $SSD_FILES files"
log "Flips: $FLIP_COUNT this hour"
# ==============================================================================================
# ━━━ Active Transcode Sessions ━━━
# ==============================================================================================
TOTAL_SESSIONS=0
LIVE_TV=0
TRANSCODING=0
DIRECT=0
ANY_SERVER_RUNNING=false
RAM_SESSION_COUNT=0
SSD_SESSION_COUNT=0
for server_entry in "${TRANSCODE_SERVERS[@]}"; do
# Parse entry — ContainerName|URL|APIKey|Type
IFS='|' read -r SRV_CONTAINER SRV_URL SRV_KEY SRV_TYPE <<< "$server_entry"
# Skip placeholder entries
if [[ "$SRV_KEY" == *"api-key"* ]] || [[ "$SRV_KEY" == *"token"* && ${#SRV_KEY} -lt 20 ]]; then
if [[ "$SRV_KEY" == *"api-key"* ]] || \
[[ "$SRV_KEY" == *"token"* && ${#SRV_KEY} -lt 20 ]]; then
log "Skipping $SRV_CONTAINER — placeholder API key"
continue
fi
# Check if container is running
if ! docker inspect "$SRV_CONTAINER" --format '{{.State.Running}}' 2>/dev/null | grep -q "true"; then
info "$ICON_CONTAINERS $SRV_CONTAINER — not running, skipping"
# Container running check with timeout
if ! timeout "$DOCKER_TIMEOUT" docker inspect "$SRV_CONTAINER" \
--format '{{.State.Running}}' 2>/dev/null | grep -q "true"; then
log "$SRV_CONTAINER — not running, skipping"
continue
fi
ANY_SERVER_RUNNING=true
# Check API reachability
if ! check_api "$SRV_URL" "$SRV_CONTAINER" 5; then
warn "$SRV_CONTAINER — API unreachable, skipping (container up but API not responding)"
warn "$SRV_CONTAINER — API unreachable, skipping"
continue
fi
# Query sessions based on server type
case "$SRV_TYPE" in
emby|jellyfin)
SESSION_DATA=$(curl -sf --max-time 10 \
-H "X-Emby-Token: $SRV_KEY" \
"${SRV_URL}/Sessions" 2>/dev/null)
;;
"${SRV_URL}/Sessions" 2>/dev/null) ;;
plex)
SESSION_DATA=$(curl -sf --max-time 10 \
-H "X-Plex-Token: $SRV_KEY" \
"${SRV_URL}/status/sessions" 2>/dev/null)
;;
"${SRV_URL}/status/sessions" 2>/dev/null) ;;
*)
warn "Unknown server type '$SRV_TYPE' for $SRV_CONTAINER — skipping"
continue
;;
continue ;;
esac
if [[ -z "$SESSION_DATA" ]] || ! command -v jq >/dev/null 2>&1; then
warn "Could not retrieve session data from $SRV_CONTAINER"
continue
fi
[[ -z "$SESSION_DATA" ]] && continue
! command -v jq >/dev/null 2>&1 && continue
# Count sessions (Emby/Jellyfin format)
if [[ "$SRV_TYPE" == "emby" || "$SRV_TYPE" == "jellyfin" ]]; then
SRV_TOTAL=$(echo "$SESSION_DATA" | \
jq '[.[] | select(.NowPlayingItem != null)] | length' 2>/dev/null || echo 0)
SRV_LIVE=$(echo "$SESSION_DATA" | \
jq '[.[] | select(.NowPlayingItem != null) | select(.NowPlayingItem.Type == "LiveTv" or .NowPlayingItem.Type == "TvChannel")] | length' \
jq '[.[] | select(.NowPlayingItem != null) |
select(.NowPlayingItem.Type == "LiveTv" or
.NowPlayingItem.Type == "TvChannel")] | length' \
2>/dev/null || echo 0)
SRV_TRANSCODE=$(echo "$SESSION_DATA" | \
jq '[.[] | select(.NowPlayingItem != null) | select(.PlayState.PlayMethod == "Transcode")] | length' \
jq '[.[] | select(.NowPlayingItem != null) |
select(.PlayState.PlayMethod == "Transcode")] | length' \
2>/dev/null || echo 0)
SRV_DIRECT=$(echo "$SESSION_DATA" | \
jq '[.[] | select(.NowPlayingItem != null) | select(.PlayState.PlayMethod != "Transcode")] | length' \
jq '[.[] | select(.NowPlayingItem != null) |
select(.PlayState.PlayMethod != "Transcode")] | length' \
2>/dev/null || echo 0)
TOTAL_SESSIONS=$(( TOTAL_SESSIONS + SRV_TOTAL ))
@@ -405,138 +411,134 @@ for server_entry in "${TRANSCODE_SERVERS[@]}"; do
TRANSCODING=$(( TRANSCODING + SRV_TRANSCODE ))
DIRECT=$(( DIRECT + SRV_DIRECT ))
# Display server header if more than one server configured and running
if [[ "${#TRANSCODE_SERVERS[@]}" -gt 1 ]]; then
echo " $ICON_EMBY $SRV_CONTAINER — $SRV_TOTAL streams"
fi
# List each active session
if [[ "$SRV_TOTAL" -gt 0 ]]; then
SOMETHING_HAPPENED=true
[[ "${#TRANSCODE_SERVERS[@]}" -gt 1 ]] && \
echo " $ICON_EMBY $SRV_CONTAINER — $SRV_TOTAL streams"
while IFS= read -r session; do
USER=$(echo "$session" | jq -r '.UserName // "Unknown"' 2>/dev/null)
USER=$(echo "$session" | jq -r '.UserName // "Unknown"' 2>/dev/null)
TITLE=$(echo "$session" | jq -r '.NowPlayingItem.Name // "Unknown"' 2>/dev/null)
MEDIA_TYPE=$(echo "$session" | jq -r '.NowPlayingItem.Type // "Unknown"' 2>/dev/null)
PLAY_METHOD=$(echo "$session" | jq -r '.PlayState.PlayMethod // "Unknown"' 2>/dev/null)
MEDIA_LABEL=$(get_media_type_label "$MEDIA_TYPE")
METHOD_LABEL=$(get_play_method_label "$PLAY_METHOD")
echo " $ICON_EMBY $(printf '%-12s' "$USER") — $(printf '%-30s' "$TITLE") — $(printf '%-10s' "$MEDIA_LABEL") — $METHOD_LABEL"
done < <(echo "$SESSION_DATA" | \
jq -c '.[] | select(.NowPlayingItem != null)' 2>/dev/null)
MTYPE=$(echo "$session" | jq -r '.NowPlayingItem.Type // "Unknown"' 2>/dev/null)
METH=$(echo "$session" | jq -r '.PlayState.PlayMethod // "Unknown"' 2>/dev/null)
echo " $ICON_EMBY $(printf '%-12s' "$USER") — $(printf '%-30s' "$TITLE") — $(get_media_type_label "$MTYPE") — $(get_play_method_label "$METH")"
done < <(echo "$SESSION_DATA" | jq -c '.[] | select(.NowPlayingItem != null)' 2>/dev/null)
fi
fi
done
# Totals header — shown after all servers
echo ""
echo " $ICON_EMBY Total: $TOTAL_SESSIONS | $ICON_RAM Live TV: $LIVE_TV | $ICON_SYNC Transcoding: $TRANSCODING | $ICON_DONE Direct: $DIRECT"
# Storage state
RAM_SESSION_COUNT=$(find "$RAMDISK_PATH/transcoding-temp" \
-mindepth 1 -maxdepth 1 -type d 2>/dev/null | wc -l)
SSD_SESSION_COUNT=$(find "$TRANSCODE_SSD/transcoding-temp" \
-mindepth 1 -maxdepth 1 -type d 2>/dev/null | wc -l)
# Storage state — based on total ramdisk/SSD session folders
if [[ "$TOTAL_SESSIONS" -gt 0 ]]; then
TRANSCODE_TEMP="transcoding-temp"
RAM_SESSION_COUNT=$(find "$RAMDISK_PATH/$TRANSCODE_TEMP" \
-mindepth 1 -maxdepth 1 -type d 2>/dev/null | wc -l)
SSD_SESSION_COUNT=$(find "$TRANSCODE_SSD/$TRANSCODE_TEMP" \
-mindepth 1 -maxdepth 1 -type d 2>/dev/null | wc -l)
echo ""
echo " $ICON_EMBY Total: $TOTAL_SESSIONS | Live TV: $LIVE_TV | Transcoding: $TRANSCODING | Direct: $DIRECT"
if [[ "$RAM_SESSION_COUNT" -gt 0 && "$SSD_SESSION_COUNT" -gt 0 ]]; then
warn "$ICON_WARN Split state — $RAM_SESSION_COUNT folder(s) on ramdisk / $SSD_SESSION_COUNT on SSD"
warn "Split state — $RAM_SESSION_COUNT folder(s) ramdisk / $SSD_SESSION_COUNT SSD"
warn "Older sessions remain on original location until they end naturally"
echo " $ICON_LINK Storage: $ICON_RAM ramdisk ($RAM_SESSION_COUNT) + $ICON_DISK SSD ($SSD_SESSION_COUNT)"
elif [[ "$RAM_SESSION_COUNT" -gt 0 ]]; then
echo " $ICON_LINK Storage: $ICON_RAM ramdisk"
log "Storage: ramdisk ($RAM_SESSION_COUNT sessions)"
elif [[ "$SSD_SESSION_COUNT" -gt 0 ]]; then
echo " $ICON_LINK Storage: $ICON_DISK SSD"
log "Storage: SSD ($SSD_SESSION_COUNT sessions)"
fi
fi
if [[ "$ANY_SERVER_RUNNING" == false && "$TRANSCODE_CHECK_EMBY" == true ]]; then
info "No configured media servers are running — skipping threshold checks"
EMBY_RUNNING=false
fi
# -----------------------------------------------------------------------------------------------
# MODE LOGIC
# -----------------------------------------------------------------------------------------------
echo ""
[[ "$ANY_SERVER_RUNNING" == false && "$TRANSCODE_CHECK_EMBY" == true ]] && \
{ log "No configured media servers running — skipping threshold checks"; EMBY_RUNNING=false; }
# ==============================================================================================
# ━━━ Mode Logic ━━━
# ==============================================================================================
case "$TRANSCODE_MANAGER_MODE" in
ramdisk)
info "$ICON_RAM Mode: RAMDISK — forcing symlink to ramdisk"
log "Mode: RAMDISK — forcing symlink to ramdisk"
if [[ "$RAMDISK_HEALTHY" == false ]]; then
error "Ramdisk mode selected but ramdisk is not available"
notify "Transcode ramdisk mode failed on $(hostname) — ramdisk not mounted" "Transcode Manager" "warning"
notify "Transcode ramdisk mode failed on $(hostname) ($MY_ID) — ramdisk not mounted" \
"Transcode Manager" "warning"
SOMETHING_HAPPENED=true
else
if [[ "$CURRENT_TARGET" != "$RAMDISK_PATH" ]]; then
flip_symlink "$RAMDISK_PATH" "ramdisk mode"
increment_flip_count > /dev/null
else
log "Symlink already points to ramdisk — no change"
FLIP_COUNT=$(get_flip_count)
SOMETHING_HAPPENED=true
fi
if (( $(awk "BEGIN {print ($RAMDISK_USED_GB >= $RAMDISK_WARN_GB) ? 1 : 0}") )); then
warn "$ICON_RAM Ramdisk usage ${RAMDISK_USED_GB}GB above threshold ${RAMDISK_WARN_GB}GB — consider switching to smart mode"
notify "Ramdisk usage high on $(hostname) — ${RAMDISK_USED_GB}GB used in ramdisk-only mode" "Transcode Manager" "warning"
warn "Ramdisk usage ${RAMDISK_USED_GB}GB above threshold ${RAMDISK_WARN_GB}GB — consider switching to smart mode"
notify "Ramdisk high on $(hostname) ($MY_ID) — ${RAMDISK_USED_GB}GB in ramdisk-only mode" \
"Transcode Manager" "warning"
SOMETHING_HAPPENED=true
fi
fi
;;
ssd)
info "$ICON_DISK Mode: SSD — forcing symlink to SSD"
log "Mode: SSD — forcing symlink to SSD"
if [[ "$SSD_HEALTHY" == false ]]; then
error "SSD mode selected but SSD path is not available"
SOMETHING_HAPPENED=true
else
if [[ "$CURRENT_TARGET" != "$TRANSCODE_SSD" ]]; then
flip_symlink "$TRANSCODE_SSD" "ssd mode"
increment_flip_count > /dev/null
else
log "Symlink already points to SSD — no change"
FLIP_COUNT=$(get_flip_count)
SOMETHING_HAPPENED=true
fi
fi
;;
smart)
info "$ICON_GEAR Mode: SMART — auto threshold management"
log "Mode: SMART — auto threshold management"
if [[ "$EMBY_RUNNING" == false ]]; then
info "No media servers running — skipping threshold checks"
log "No media servers running — skipping threshold checks"
elif [[ "$RAMDISK_HEALTHY" == false ]]; then
info "Ramdisk unavailable — staying on SSD until ramdisk recovers"
log "Ramdisk unavailable — staying on SSD until ramdisk recovers"
elif [[ "$CURRENT_TARGET" == "$RAMDISK_PATH" ]]; then
if (( $(awk "BEGIN {print ($RAMDISK_USED_GB >= $RAMDISK_WARN_GB) ? 1 : 0}") )); then
if [[ "$SSD_HEALTHY" == false ]]; then
error "Ramdisk above threshold but SSD unavailable — cannot flip"
notify "Transcode ramdisk full on $(hostname) and SSD unavailable — intervention needed" "Transcode Manager" "warning"
notify "Transcode ramdisk full on $(hostname) ($MY_ID) and SSD unavailable" \
"Transcode Manager" "warning"
SOMETHING_HAPPENED=true
else
SSD_FREE_GB=$(get_ssd_free_gb)
if (( $(awk "BEGIN {print ($SSD_FREE_GB < $RAMDISK_SSD_MIN_GB) ? 1 : 0}") )); then
warn "$ICON_DISK SSD only ${SSD_FREE_GB}GB free — below ${RAMDISK_SSD_MIN_GB}GB minimum, not flipping"
warn "SSD only ${SSD_FREE_GB}GB free — below ${RAMDISK_SSD_MIN_GB}GB minimum, not flipping"
SOMETHING_HAPPENED=true
else
warn "$ICON_RAM Ramdisk ${RAMDISK_USED_GB}GB — above ${RAMDISK_WARN_GB}GB, flipping to SSD"
warn "Ramdisk ${RAMDISK_USED_GB}GB — above ${RAMDISK_WARN_GB}GB, flipping to SSD"
flip_symlink "$TRANSCODE_SSD" "threshold exceeded"
CURRENT_TARGET="$TRANSCODE_SSD"
NEW_COUNT=$(increment_flip_count)
FLIP_COUNT=$NEW_COUNT
SOMETHING_HAPPENED=true
if [[ "$NEW_COUNT" -ge "$TRANSCODE_FLIP_WARN" ]]; then
notify "Transcode flipped to SSD on $(hostname) — ${NEW_COUNT} flips this hour. Consider increasing RAMDISK_SIZE." "Transcode Manager" "warning"
notify "Transcode flipped to SSD on $(hostname) ($MY_ID) — ${NEW_COUNT} flips this hour — consider increasing HOST*_RAMDISK_SIZE" \
"Transcode Manager" "warning"
fi
fi
fi
else
log "Symlink already points to ramdisk — no change"
log "Ramdisk ${RAMDISK_USED_GB}GB — below threshold — no action needed"
fi
elif [[ "$CURRENT_TARGET" == "$TRANSCODE_SSD" ]]; then
if (( $(awk "BEGIN {print ($RAMDISK_USED_GB <= $RAMDISK_LOW_GB) ? 1 : 0}") )); then
info "$ICON_RAM Ramdisk ${RAMDISK_USED_GB}GB — below ${RAMDISK_LOW_GB}GB, flipping back to ramdisk"
warn "Ramdisk ${RAMDISK_USED_GB}GB — below ${RAMDISK_LOW_GB}GB, flipping back to ramdisk"
flip_symlink "$RAMDISK_PATH" "usage recovered"
CURRENT_TARGET="$RAMDISK_PATH"
NEW_COUNT=$(increment_flip_count)
FLIP_COUNT=$NEW_COUNT
SOMETHING_HAPPENED=true
else
log "On SSD — ramdisk ${RAMDISK_USED_GB}GB still above low threshold ${RAMDISK_LOW_GB}GB"
fi
@@ -547,16 +549,32 @@ esac
END=$(date +%s)
CURRENT_TARGET=$(readlink "$TRANSCODE_LINK" 2>/dev/null || echo "unknown")
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Summary ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━━━ $ICON_SUMMARY TRANSCODE MANAGER SUMMARY ━━━━━"
echo "$ICON_GEAR Mode: $TRANSCODE_MANAGER_MODE"
echo "$ICON_RAM Ramdisk: ${RAMDISK_USED_GB}GB used / ${RAMDISK_AVAIL_GB}GB available"
echo "$ICON_LINK Symlink: $TRANSCODE_LINK → $CURRENT_TARGET"
echo "$ICON_LINK Flips: $FLIP_COUNT this hour (warn at $TRANSCODE_FLIP_WARN)"
echo "$ICON_EMBY Streams: ${TOTAL_SESSIONS} total | Live TV: ${LIVE_TV} | Transcoding: ${TRANSCODING} | Direct: ${DIRECT}"
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
echo "$ICON_DONE Status: $ICON_SUCCESS DONE"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
# ==============================================================================================
# ━━━ Daily Log Write ━━━
# ==============================================================================================
# Read by weekly_health_digest.sh — format: DATE|USED_GB|FLIPS|RAM_SESSIONS|SSD_SESSIONS
if [[ "$DRY_RUN" == false && "$NO_LOG" == false && -n "${TRANSCODE_DAILY_LOG:-}" ]]; then
TODAY=$(date '+%Y-%m-%d')
mkdir -p "$(dirname "$TRANSCODE_DAILY_LOG")"
echo "${TODAY}|${RAMDISK_USED_GB}|${FLIP_COUNT}|${RAM_SESSION_COUNT}|${SSD_SESSION_COUNT}" \
>> "$TRANSCODE_DAILY_LOG" 2>/dev/null || true
log "Daily log written: $TRANSCODE_DAILY_LOG"
fi
# ==============================================================================================
# ━━━ Summary — only shown when something happened ━━━
# ==============================================================================================
if [[ "$SOMETHING_HAPPENED" == true ]]; then
echo ""
echo "━━━━━ $ICON_SUMMARY TRANSCODE MANAGER SUMMARY ━━━━━"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_GEAR Mode: $TRANSCODE_MANAGER_MODE"
echo "$ICON_RAM Ramdisk: ${RAMDISK_USED_GB}GB used / ${RAMDISK_AVAIL_GB}GB available"
echo "$ICON_LINK Symlink: $TRANSCODE_LINK → $CURRENT_TARGET"
echo "$ICON_LINK Flips: $FLIP_COUNT this hour (warn at $TRANSCODE_FLIP_WARN)"
echo "$ICON_EMBY Streams: $TOTAL_SESSIONS total | Live TV: $LIVE_TV | Transcoding: $TRANSCODING | Direct: $DIRECT"
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
else
log "Transcode manager — all healthy — $MY_ID ($(format_duration $(( END - START ))))"
fi
common.sh
+851 -353
View File
File diff suppressed because it is too large Load Diff
git_pull_execute.sh
+158 -46
View File
@@ -1,30 +1,60 @@
#!/bin/bash
# -----------------------------------------------------------------------------------------------
# --------------------------------- Git Pull & Execute Script -----------------------------------
# -----------------------------------------------------------------------------------------------
# Pulls latest scripts from Gitea repo via SSH and sets executable permissions.
# Lives at the repo root — sources Master.conf and common.sh from the same directory.
# Uses GITEA_SSH_KEY, SSH_PORT, GITEA_CONTAINER and GITEA_REPO_PATH from Master.conf.
# ==============================================================================================
# ================================= Git Pull & Execute =========================================
# ==============================================================================================
# Pulls the latest scripts from the Gitea repository via SSH.
# Lives at the repo root — sources load_config.sh from the same directory.
#
# Detects where the Gitea container is running at runtime:
# Gitea running locally → connects via local IP
# Gitea running remotely → connects via remote server's Tailscale IP
# Works correctly through failover — no hardcoded assumptions about which server hosts Gitea.
# ── WHAT THIS SCRIPT DOES ─────────────────────────────────────────────────────────────────────
# 1. Detects which server it's running on via detect_hosts() (MY_ID)
# 2. Configures sparse checkout to exclude other servers' credential files
# Each server only pulls its own master_host*.conf — never sees peer credentials
# 3. Pulls or clones latest scripts from Gitea
# 4. Sets executable permissions on all .sh files
#
# Supports --dry-run, --log, --status flags via common.sh parse_args.
# -----------------------------------------------------------------------------------------------
# ── SPARSE CHECKOUT ───────────────────────────────────────────────────────────────────────────
# Sparse checkout ensures each server only receives its own host conf:
# HOST1 pulls: master.conf + master_host1.conf + all scripts
# HOST1 skips: master_host2.conf, master_host3.conf etc.
# HOST2 pulls: master.conf + master_host2.conf + all scripts
# HOST2 skips: master_host1.conf, master_host3.conf etc.
#
# Adding a new server:
# Create master_host3.conf in the repo
# All existing servers automatically exclude it on next pull
# New server gets only its own conf ✅
#
# ── GITEA LOCATION DETECTION ──────────────────────────────────────────────────────────────────
# Detects where Gitea is running at runtime — works through failover:
# Gitea local → connects via local IP
# Gitea remote → connects via Tailscale IP
# Both fail → falls back to GITEA_DOMAIN if configured
#
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
# GITEA_CONTAINER — Docker container name for Gitea
# GITEA_REPO_PATH — repo path on Gitea (e.g. FailedProxy/Unraid_Scripts.git)
# GITEA_DOMAIN — public domain fallback (optional)
# TARGET_DIR — local path to clone/pull into
# GITEA_SSH_KEY — SSH key for Gitea authentication
# SSH_PORT — Gitea SSH port (often 221 or 222)
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
# git_pull_execute.sh — normal pull
# git_pull_execute.sh --dry-run — preview without making changes
# git_pull_execute.sh --log — verbose output
# git_pull_execute.sh --status — show config and exit
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# Root level script — sources from same directory, not parent
source "$SCRIPT_DIR/Master.conf"
source "$SCRIPT_DIR/common.sh"
# Root level script — load_config.sh is in the same directory
source "$SCRIPT_DIR/load_config.sh"
parse_args "$@"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Setup ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_GEAR Setup ━━━"
@@ -32,27 +62,30 @@ if [[ "$EUID" -ne 0 ]]; then
error "Must be run as root"
exit 1
fi
success "Running as root"
acquire_lock
# Detect which server we're on and where Gitea is running
# detect_hosts() sets MY_ID — needed for sparse checkout configuration
detect_hosts
# Check if Gitea container is running locally
# ==============================================================================================
# ━━━ Locate Gitea ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_CONTAINERS Locate Gitea ━━━"
if docker ps --format "{{.Names}}" 2>/dev/null | grep -q "^${GITEA_CONTAINER}$"; then
# Gitea is running on this server — use local IP
GITEA_IP=$(hostname -I | awk '{print $1}')
info "$ICON_CONTAINERS Gitea running locally — connecting via $GITEA_IP"
log "Gitea running locally — connecting via $GITEA_IP"
else
# Gitea is not running locally — find it on the remote server via Tailscale
info "$ICON_CONTAINERS Gitea not running locally — checking remote server"
# Gitea not running locally — find it on the remote server via Tailscale
log "Gitea not running locally — checking remote server"
GITEA_IP=$(tailscale ip -4 "$REMOTE_SERVER_NAME" 2>/dev/null)
if [[ -n "$GITEA_IP" ]]; then
info "$ICON_NET Gitea on $REMOTE_SERVER_NAME — connecting via Tailscale $GITEA_IP"
elif [[ -n "$GITEA_DOMAIN" ]]; then
# Tailscale failed — fall back to public domain
elif [[ -n "${GITEA_DOMAIN:-}" ]]; then
warn "Tailscale resolution failed — falling back to $GITEA_DOMAIN"
GITEA_IP="$GITEA_DOMAIN"
else
@@ -69,27 +102,85 @@ require_var TARGET_DIR
require_var GITEA_SSH_KEY
require_var SSH_PORT
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Status ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
echo ""
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
echo "$ICON_NET Repo: $REPO_SSH"
echo "$ICON_GEAR Target: $TARGET_DIR"
echo "$ICON_GEAR SSH Key: $GITEA_SSH_KEY"
echo "$ICON_GEAR SSH Port: $SSH_PORT"
echo "$ICON_NOTIFY Notifications: unRAID=${NOTIFY_UNRAID:-false} Discord=$([[ -n "${DISCORD_WEBHOOK:-}" ]] && echo enabled || echo disabled)"
echo "$ICON_GEAR Dry Run: $DRY_RUN"
echo "$ICON_NET Repo: $REPO_SSH"
echo "$ICON_GEAR Target: $TARGET_DIR"
echo "$ICON_GEAR SSH Key: $GITEA_SSH_KEY"
echo "$ICON_GEAR SSH Port: $SSH_PORT"
echo "$ICON_HOST My ID: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_HOST Remote ID: $REMOTE_ID ($REMOTE_SERVER_NAME)"
echo "$ICON_NOTIFY Notify: unRAID=${NOTIFY_UNRAID:-false} Discord=$([[ -n "${MY_DISCORD_WEBHOOK:-}" ]] && echo enabled || echo disabled)"
echo "$ICON_GEAR Dry Run: $DRY_RUN"
echo "━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
fi
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SYNC Git Sync ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Sparse Checkout Configuration ━━━
# ==============================================================================================
# Build the list of master_host*.conf files that belong to OTHER servers.
# This server pulls everything EXCEPT those files.
# MY_ID is set by detect_hosts() — e.g. "HOST1"
configure_sparse_checkout() {
local repo_dir="$1"
log "Configuring sparse checkout for $MY_ID..."
# Enable sparse checkout
git -C "$repo_dir" config core.sparseCheckout true 2>/dev/null
# Build exclusion list — all master_host*.conf files except MY_ID's
local sparse_file="$repo_dir/.git/info/sparse-checkout"
mkdir -p "$(dirname "$sparse_file")"
# Start with: pull everything
echo "/*" > "$sparse_file"
# Exclude each other server's conf file
# Find all master_host*.conf files present in the repo
local excluded=0
for conf_file in "$repo_dir"/master_host*.conf; do
[[ -f "$conf_file" ]] || continue
local conf_name
conf_name=$(basename "$conf_file")
# Determine which HOST ID owns this conf by grepping its hostname var
# Pattern: HOST1="unRAID-..." or HOST2="unRAID-..."
local conf_host_id
conf_host_id=$(grep -m1 -oP '^\s+HOST[0-9]+(?==)' "$conf_file" 2>/dev/null | tr -d ' ')
if [[ -z "$conf_host_id" ]]; then
log "Cannot determine HOST ID for $conf_name — including in pull (safe default)"
continue
fi
if [[ "$conf_host_id" != "$MY_ID" ]]; then
echo "!$conf_name" >> "$sparse_file"
log "Sparse checkout: excluding $conf_name (belongs to $conf_host_id)"
((excluded++))
else
log "Sparse checkout: including $conf_name (belongs to $MY_ID — this server)"
fi
done
if [[ "$excluded" -gt 0 ]]; then
info "$ICON_LOCK Sparse checkout: excluding $excluded peer conf file(s) — credentials protected"
else
log "Sparse checkout: no peer conf files to exclude (single server or first run)"
fi
}
# ==============================================================================================
# ━━━ Git Sync ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_SYNC Git Sync ━━━"
echo "$ICON_NET Repo: $REPO_SSH"
@@ -101,13 +192,21 @@ SYNC_SUCCESS=false
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would sync $REPO_SSH → $TARGET_DIR"
warn "DRY RUN — would configure sparse checkout for $MY_ID"
warn "DRY RUN — would exclude peer master_host*.conf files"
SYNC_SUCCESS=true
else
mkdir -p "$TARGET_DIR"
cd "$TARGET_DIR" || { error "Cannot cd into $TARGET_DIR"; exit 1; }
if [[ -d ".git" ]]; then
# ── Existing repository ──────────────────────────────────────────────
info "$ICON_SYNC Existing repository detected — updating"
# Configure sparse checkout BEFORE pull
# Uses conf files already present from last pull to determine exclusions
configure_sparse_checkout "$TARGET_DIR"
log "git reset --hard"
git reset --hard
@@ -125,9 +224,22 @@ else
fi
else
# ── Fresh clone ──────────────────────────────────────────────────────
info "$ICON_SYNC No repository found — cloning"
# Clone first — need the repo to exist before configuring sparse checkout
if GIT_SSH_COMMAND="ssh -i $GITEA_SSH_KEY -p $SSH_PORT" git clone "$REPO_SSH" .; then
success "Clone successful"
# Configure sparse checkout after clone
# Now all master_host*.conf files are present — can detect exclusions
configure_sparse_checkout "$TARGET_DIR"
# Apply sparse checkout — removes excluded files from working tree
info "Applying sparse checkout..."
git read-tree -mu HEAD
success "Sparse checkout applied — peer credentials removed from working tree"
SYNC_SUCCESS=true
else
error "Clone failed"
@@ -136,25 +248,25 @@ else
fi
fi
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Permissions ━━━
# -----------------------------------------------------------------------------------------------
# ── Permissions ──────────────────────────────────────────────────────────
echo ""
echo "━━━ $ICON_GEAR Permissions ━━━"
info "Setting executable permissions on all .sh files..."
log "Setting executable permissions on all .sh files..."
find "$TARGET_DIR" -type f -name "*.sh" -exec chmod +x {} \;
success "Permissions applied"
log "Permissions applied"
fi
END=$(date +%s)
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Summary ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
echo ""
echo "━━━━━ $ICON_SUMMARY GIT SYNC SUMMARY ━━━━━"
echo "$ICON_NET Repo: $REPO_SSH"
echo "$ICON_GEAR Target: $TARGET_DIR"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_LOCK Excluded: peer master_host*.conf files"
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
if [[ "$DRY_RUN" == true ]]; then
echo "$ICON_WARN Status: DRY RUN — no changes made"
load_config.sh
+97
View File
@@ -0,0 +1,97 @@
#!/bin/bash
# ==============================================================================================
# ================================= CONFIGURATION LOADER =======================================
# ==============================================================================================
# Single entry point for all configuration sourcing across the ecosystem.
# Every script sources this file instead of sourcing master.conf files directly.
#
# ── HOW IT WORKS ──────────────────────────────────────────────────────────────────────────────
# 1. Sources master.conf (shared config — thresholds, toggles, profiles, job lists)
# 2. Auto-discovers and sources all master_host*.conf files in the same directory
# Each host conf extends the shared profile arrays and adds host-specific identity
# 3. Sources common.sh (shared functions — detect_hosts, logging, notifications etc.)
#
# ── WHY THIS EXISTS ───────────────────────────────────────────────────────────────────────────
# Without this loader every script had to explicitly source each conf file:
# source master.conf
# source master_host1.conf
# source master_host2.conf
# source common.sh
#
# Adding a new server meant updating every script.
# With this loader — add master_host3.conf to the git repo and every server
# auto-discovers it on next git pull. Zero script changes required. Ever.
#
# ── ADDING A NEW SERVER ───────────────────────────────────────────────────────────────────────
# 1. Create master_host3.conf following the same structure as HOST1/HOST2
# 2. Commit and push to git repo
# 3. All servers pull it automatically — no other changes needed
#
# ── USAGE IN SCRIPTS ──────────────────────────────────────────────────────────────────────────
# Replace the three source lines at the top of every script with:
#
# SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# source "$SCRIPT_DIR/../load_config.sh"
#
# Scripts in subdirectories (Rsync/, Docker_Essentials/ etc.) use ../ to reach root.
# Scripts in root directory use ./ instead:
#
# SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# source "$SCRIPT_DIR/load_config.sh"
#
# ── SPARSE CHECKOUT NOTE ──────────────────────────────────────────────────────────────────────
# Sparse checkout controls which master_host*.conf files each server receives.
# HOST1 only pulls master_host1.conf — never HOST2's credentials.
# HOST2 only pulls master_host2.conf — never HOST1's credentials.
# This loader sources whatever conf files ARE present — sparse checkout handles the rest.
# Both servers pull all non-credential conf files (master.conf, common.sh, load_config.sh).
#
# ==============================================================================================
# ━━━ Locate config root ━━━
# load_config.sh always lives in the repo root.
# Scripts call it from subdirectories using ../ — resolve to the actual root.
LOAD_CONFIG_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# ━━━ Source shared config ━━━
# master.conf must be sourced first — it declares the shared PROFILE_* arrays
# that Host confs extend. Sourcing host confs before master.conf would fail.
if [[ ! -f "$LOAD_CONFIG_DIR/master.conf" ]]; then
echo "[FATAL] master.conf not found at $LOAD_CONFIG_DIR/master.conf" >&2
echo "[FATAL] Check TARGET_DIR and git pull status" >&2
exit 1
fi
source "$LOAD_CONFIG_DIR/master.conf"
# ━━━ Auto-discover and source all master_host*.conf files ━━━
# Sorted for consistent load order — HOST1 before HOST2 before HOST3 etc.
# Each host conf extends the shared PROFILE_* arrays and adds host-specific vars.
# Missing files are silently skipped — sparse checkout intentionally withholds some.
# At least one host conf must be present or the ecosystem has no identity to work with.
_host_confs_loaded=0
for _conf in $(ls "$LOAD_CONFIG_DIR"/master_host*.conf 2>/dev/null | sort); do
if [[ -f "$_conf" ]]; then
source "$_conf"
(( _host_confs_loaded++ ))
[[ "${ENABLE_LOGGING:-false}" == "true" ]] && \
echo "[LOG] Loaded host config: $(basename "$_conf")" >&2
fi
done
if [[ "$_host_confs_loaded" -eq 0 ]]; then
echo "[FATAL] No master_host*.conf files found in $LOAD_CONFIG_DIR" >&2
echo "[FATAL] At least one host conf required — check git pull and sparse checkout" >&2
exit 1
fi
# ━━━ Source shared functions ━━━
# common.sh sourced last — it calls detect_hosts() which needs HOST* vars to be set.
if [[ ! -f "$LOAD_CONFIG_DIR/common.sh" ]]; then
echo "[FATAL] common.sh not found at $LOAD_CONFIG_DIR/common.sh" >&2
exit 1
fi
source "$LOAD_CONFIG_DIR/common.sh"
# ━━━ Cleanup ━━━
unset _conf _host_confs_loaded LOAD_CONFIG_DIR
Master.conf → master.conf
+411 -620
View File
File diff suppressed because it is too large Load Diff
master_host1.conf
+646
View File
@@ -0,0 +1,646 @@
#!/bin/bash
# ==============================================================================================
# ========================== HOST1 CONFIGURATION — unRAID-Gmer4Lfe ============================
# ==============================================================================================
# HOST1-specific variables — credentials, container names, share paths, failover lists.
# Sourced after master.conf — values here extend shared profile arrays and add HOST1-specific
# identity, credentials, and container configuration.
#
# Sparse checkout (git) ensures HOST2 never receives this file.
# HOST2 never sees HOST1 credentials — clean separation at the file level.
#
# DO NOT put shared config here — thresholds, toggles, profiles belong in master.conf.
# DO NOT put HOST2 variables here — they belong in master_host2.conf.
#
# ── INDEX ─────────────────────────────────────────────────────────────────────────────────────
#
# ── IDENTITY & CONNECTIVITY ────────────────────────────────────────────────────────────────
# IDENTITY hostname, SSH key
# EMBY container name, URL, API key
# NOTIFICATIONS Discord webhook
# PARTNERSHIP auth containers, backup paths
#
# ── RSYNC ──────────────────────────────────────────────────────────────────────────────────
# DAILY SYNC SHARES media shares HOST1 owns and pushes to HOST2
# WEEKLY SYNC SHARES appdata shares synced weekly (Sunday 2:30am)
# CRITICAL SYNC SHARES appdata shares synced every 15 minutes
# BACKUP VERIFY shares for checksum verification against remote
# HOST1 RSYNC PROFILE host1-appdata profile for HOST1-specific appdata syncs
#
# ── DOCKER ─────────────────────────────────────────────────────────────────────────────────
# DOCKER DAILY RESTART containers restarted daily
# DOCKER WEEKLY RESTART containers restarted weekly
# DOCKER WATCHDOG memory limits, health URLs, required containers, ignore list
# DOCKER NETWORK CONNECT networks and containers for docker_network_connect.sh
#
# ── FAILOVER ───────────────────────────────────────────────────────────────────────────────
# DDNS DDNS containers managed by HOST1
# INTERNET LOSS containers stopped when internet is lost
# FAILOVER TIERS what HOST1 runs for HOST2 per tier
# TIER DELAYS how long HOST1 must be down before each tier activates on HOST2
# RSYNC WRITEBACK HOST1 appdata synced back on handback
#
# ── MEDIA ──────────────────────────────────────────────────────────────────────────────────
# MEDIA PERMISSIONS share list for media_shares_permissions.sh
# MEDIA CLEANER folder lists for media_cleaner.sh
#
# ── MONITORS ───────────────────────────────────────────────────────────────────────────────
# CERTIFICATE MONITOR domains checked for SSL expiry
# SMART HEALTH drives to skip in SMART monitoring
# ZFS REPORT pools to exclude from ZFS health report
#
# ── TRANSCODES ─────────────────────────────────────────────────────────────────────────────
# TRANSCODES ramdisk size, thresholds, SSD path, server array
#
# ── ARR STACK ──────────────────────────────────────────────────────────────────────────────
# DOWNLOADERS slskd, SABnzbd, qBittorrent credentials and URLs
# LIDARR URL, API key, path map
# SONARR URL, API key, path map
# RADARR URL, API key, path map
# ARR RECOVERY per-arr recovery toggles
#
# ==============================================================================================
# ==============================================================================================
# ── IDENTITY & CONNECTIVITY ───────────────────────────────────────────────────────────────────
# ==============================================================================================
# ━━━ Identity ━━━
# Hostname must match exact unRAID hostname AND Tailscale device name — case sensitive.
# Used by detect_hosts() in common.sh to identify this server as HOST1.
HOST1="unRAID-Gmer4Lfe"
# SSH key used for all server-to-server operations — rsync, failover container commands.
# Must be in /root/.ssh/ and authorised in HOST2's /root/.ssh/authorized_keys.
HOST1_SSH_KEY="/root/.ssh/Gmer4Lfe-rsync-key"
# ━━━ Emby ━━━
# Referenced by transcode_manager.sh, emby_session_report.sh, emby_database_repair.sh,
# weekly_sync_maintenance.sh, and HOST1_TRANSCODE_SERVERS below.
# API key: Emby Dashboard → API Keys → + New Key
HOST1_EMBY_CONTAINER="Emby"
HOST1_EMBY_URL="http://localhost:8096"
HOST1_EMBY_API_KEY="0c27448d93a7431f9ac63569f7655829"
# ━━━ Notifications ━━━
# Discord webhook — leave blank to disable.
# Per-host so HOST1 and HOST2 can post to different channels or only one server notifies.
HOST1_DISCORD_WEBHOOK=""
# ━━━ Partnership ━━━
# HOST1 is always the owner (source of truth) unless --transfer has been run.
# See README-Partnership.md and master.conf PARTNERSHIP section for full lifecycle docs.
# Auth containers reconfigured on onboard/offboard.
# Format: "ContainerName|WebUIPort"
# On onboard → WebUI pointed at owner's Tailscale IP (mirror clicks NPM, gets owner's auth)
# On offboard → WebUI pointed back at localhost
HOST1_PARTNERSHIP_AUTH_WEBUIS=(
"NginxProxyManager|81"
"Lldap-Gmer4Lfe|17170"
"Authelia|9091"
"Authelia-Secondary|9092"
)
# Paths HOST2 should collect during the grace window after offboard.
# Notified on offboard — no auto-deletion, HOST2 must collect manually within PARTNERSHIP_GRACE_HOURS.
HOST1_PARTNERSHIP_MIRROR_BACKUPS=(
# "/mnt/user/appdata-Failover/Jayred365-Emby"
)
# ==============================================================================================
# ── RSYNC ─────────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# ━━━ Daily Sync Shares ━━━
# Shares HOST1 owns and pushes to HOST2 every night (1am via daily_sync_maintenance.sh).
# HOST1 is the source of truth — HOST2 is the mirror.
# Never push a share both directions — one server always owns it.
# Uses DEFAULT_RSYNC_OPTS from master.conf — no profile needed.
# For shares needing container stops or custom options — add a profile in master.conf.
HOST1_DAILY_SYNC_SHARES=(
/mnt/user/Books
/mnt/user/Intros
/mnt/user/Kids_Movies
/mnt/user/Kids_Tv_Shows
/mnt/user/Movies
/mnt/user/Music
/mnt/user/Music_Videos
/mnt/user/Nextcloud
/mnt/user/stand-up_comedy
/mnt/user/Sports
/mnt/user/Tv_Shows
/mnt/user/Anime_Shows-Old
/mnt/user/Anime_Movies-Old
)
# Personal encrypted shares — synced for offsite backup, independent of media shares.
# ZFS encrypted at dataset level — remote receives encrypted blocks, cannot read content.
# See README-Rsync_Setup.md for ZFS encryption setup before uncommenting.
HOST1_PERSONAL_SHARES=(
# /mnt/user/HOST1-Personal # uncomment after creating encrypted dataset
)
# ━━━ Weekly Sync Shares ━━━
# Appdata shares synced during the weekly maintenance window (Sunday 2:30am).
# Containers stopped both sides before sync — full clean state guaranteed.
# Profiles drive container stops, excludes, and options — configured in master.conf RSYNC section.
# Order matters — Emby first (larger transfer), then Critical-Data (auth stack).
HOST1_WEEKLY_SYNC_SHARES=(
"/mnt/user/Media_Server/Emby" # emby profile — full clean mirror
"/mnt/user/appdata-Failover/Critical-Data" # critical-data profile — auth stack
)
# ━━━ Critical Sync Shares ━━━
# Appdata shares synced every 15 minutes by critical_sync_maintenance.sh.
# Format: "/path/to/share" or "/path/to/share|profile-name"
# Order matters — Critical-Data first (auth stack), then Emby dirty sync.
HOST1_CRITICAL_SYNC_SHARES=(
"/mnt/user/appdata-Failover/Critical-Data|critical-failover" # auth dirty sync — stays running
"/mnt/user/Media_Server/Emby|emby-failover" # Emby dirty sync — stays running
)
# ━━━ Backup Verify ━━━
# Shares verified by backup_verify.sh — random file checksum comparison against remote.
# Leave empty to use HOST1_DAILY_SYNC_SHARES automatically.
# Sample size and minimum file size defined in master.conf.
HOST1_BACKUP_VERIFY_SHARES=(
# leave empty to use HOST1_DAILY_SYNC_SHARES automatically
)
# ━━━ HOST1 Rsync Profile — host1-appdata ━━━
# HOST1-specific appdata sync profile — extends the shared PROFILE_* arrays in master.conf.
# Use for appdata unique to HOST1 (Organizrv2, VaultWarden, UptimeKuma etc.)
# Shared appdata (auth stack, Emby) use dedicated profiles defined in master.conf.
# Run manually: bash Rsync/rsync.sh /mnt/user/appdata-Failover/HOST1-Appdata --profile=host1-appdata
PROFILE_RSYNC_OPTS[host1-appdata]="-av --info=progress2 --bwlimit=${PROFILE_BW_LIMIT[host1-appdata]:-8000}"
PROFILE_BW_LIMIT[host1-appdata]=8000
PROFILE_RETRY_COUNT[host1-appdata]=3
PROFILE_SLEEP[host1-appdata]=300
PROFILE_CRITICAL_CONTAINER_NAMES[host1-appdata]="Organizrv2-Gmer4Lfe UptimeKuma-Gmer4Lfe VaultWarden-Gmer4Lfe"
PROFILE_DELAYED_CONTAINERS[host1-appdata]=""
PROFILE_CONTAINER_DELAY[host1-appdata]=5
PROFILE_EXCLUDE_DIRS[host1-appdata]="logs *.tmp"
# ==============================================================================================
# ── DOCKER ────────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# ━━━ Docker Daily Restart ━━━
# Containers restarted every day via DAILY_MAINTENANCE_SCRIPTS.
# Dispatcharr degrades over time without restart — daily is intentional, not just housekeeping.
# Order matters — auth stack first, then media services.
HOST1_DAILY_RESTART_CONTAINERS=(
"NginxProxyManager"
"Lldap-Gmer4Lfe"
"Authelia"
"Authelia-Secondary"
"Dispatcharr-Iptv-Users"
"Dispatcharr" # Live TV scheduler — degrades without daily restart
"Dispatcharr-Basic"
"ErsatzTV-Emby"
)
# ━━━ Docker Weekly Restart ━━━
# Less critical services restarted weekly via WEEKLY_MAINTENANCE_SCRIPTS (Sunday 2:30am).
# Containers already stopped for weekly sync — restart adds zero extra downtime.
HOST1_WEEKLY_RESTART_CONTAINERS=(
"NextCloud"
"Organizrv2-Gmer4Lfe"
"AdGuard-Home"
"Immich-Gmer4Lfe"
)
# ━━━ Docker Watchdog ━━━
# Per-HOST1 container configuration for docker_watchdog.sh.
# Shared thresholds and toggles live in master.conf.
# Memory hard limits in MB — immediate restart if exceeded.
# Set at "container is clearly broken" not "container is busy".
# 20GB=20480 18GB=18432 16GB=16384 12GB=12288 8GB=8192 4GB=4096 2GB=2048 1GB=1024
declare -A HOST1_WATCHDOG_CONTAINERS=(
["Emby"]=18432 # 18GB — large library + active transcodes
["LidaTube"]=6144 # 6GB — memory leak over time
["Tdarr"]=6144 # 6GB — encoding is memory intensive
["Code-Server"]=1024 # 1GB — should never need more
)
# HTTP health check URLs — checked every cycle, strike system before restart.
# Only add containers with a meaningful web interface to check.
declare -A HOST1_WATCHDOG_CONTAINER_URLS=(
["Emby"]="http://localhost:8096"
)
# Required containers — must always be running on HOST1.
# Strike system before restart — repeated failures go on skip list, auto-clears on recovery.
# Listed in dependency order — dependencies before dependents.
HOST1_WATCHDOG_REQUIRED_CONTAINERS=(
"NginxProxyManager"
"Lldap-Gmer4Lfe"
"Mariadb-Authelia"
"Mariadb-Authelia-Secondary"
"Redis-Authelia"
"Redis-Authelia-Secondary"
"Authelia"
"Authelia-Secondary"
)
# Containers to skip in Tier 2 global scan — legitimately stopped or frequently restarting.
# Watchdog leaves these alone entirely — no restart attempts, no crash loop tracking.
HOST1_WATCHDOG_SCAN_IGNORE=(
"DashGate"
"PIA-WG-Config-Generator"
"Aperture"
"Aperture-Kids"
"pgvector-18-Apeture-Kids"
"Pgvector18-Aperture"
)
# Dependency ordering — skip restarting a container if its dependency is also down.
# Prevents watchdog from restarting Authelia before Mariadb is back up.
# SPACE-SEPARATED STRINGS — converted to array at runtime.
declare -A HOST1_WATCHDOG_DEPENDENCIES=(
["Authelia"]="Mariadb-Authelia Redis-Authelia"
["Authelia-Secondary"]="Mariadb-Authelia Redis-Authelia-Secondary"
["NextCloud"]="Postgres-NextCloud"
)
# ━━━ Docker Network Connect ━━━
# Containers connected to custom networks at array start by docker_network_connect.sh.
# Networks created if they don't exist — idempotent, safe to re-run.
HOST1_NETWORK_CONNECT_CONTAINERS=(
"memcached"
"Npm-CrowdSec"
)
HOST1_NETWORK_CONNECT_NETWORKS=(
"high-availability"
)
# ==============================================================================================
# ── FAILOVER ──────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# ━━━ DDNS ━━━
# DDNS containers HOST1 manages — started/stopped by failover.sh per DDNS absolute rules:
# Internet loss → stop immediately
# Failover → HOST2 starts HOST1's DDNS as Tier 1 (before any other containers)
# Handback → stop HOST1's DDNS on HOST2 → rsync → start containers → start local DDNS last
HOST1_DDNS_CONTAINERS=(
"Gmer4Lfe.com"
)
# ━━━ Internet Loss ━━━
# Containers stopped immediately on HOST1 when internet connection is lost.
# Prevents external-facing services from operating without connectivity.
FAILOVER_HOST1_STOP_ON_NO_NET=(
"Gmer4Lfe.com"
)
# ━━━ Failover Tiers — HOST1 Runs for HOST2 ━━━
# Containers HOST1 starts when HOST2 goes down.
# Tier 1 is always immediate — vital services cannot wait.
# Higher tiers activate after HOST2_TIER*_DELAY minutes (set in master_host2.conf).
FAILOVER_HOST1_RUNS_FOR_HOST2_TIER1=(
"Gmer4Lfe.us"
"VaultWarden-Jayred365"
)
FAILOVER_HOST1_RUNS_FOR_HOST2_TIER2=(
# "container-placeholder"
)
FAILOVER_HOST1_RUNS_FOR_HOST2_TIER3=(
# "container-placeholder"
)
FAILOVER_HOST1_RUNS_FOR_HOST2_TIER4=(
# "container-placeholder"
)
# ━━━ Tier Delays — HOST1's Containers on HOST2 ━━━
# How long HOST1 must be down before each tier activates on HOST2 — in minutes.
# Tier 1 is always immediate — no delay var needed.
HOST1_TIER2_DELAY=240 # 4 hours — NextCloud, Immich
HOST1_TIER3_DELAY=720 # 12 hours — secondary services
HOST1_TIER4_DELAY=1440 # 24 hours — arrs + downloaders
# ━━━ Rsync Writeback — HOST1 Appdata Back on Handback ━━━
# Syncs HOST1 appdata BACK to HOST1 when it comes back online after a failover.
# Containers stopped before writeback — clean source, no competing writes.
#
# HOST1_TIER1_WRITEBACK_DELAY: short outages skip Tier 1 writeback — primary state
# is more reliable than dirty sync data for brief outages.
HOST1_TIER1_WRITEBACK_DELAY=60 # skip Emby writeback if outage under 1hr
# Tier 4 automatically syncs HOST1_DAILY_SYNC_SHARES — only list paths NOT in that array.
FAILOVER_HOST1_WRITEBACK_TIER1=(
"/mnt/user/Media_Server/Emby" # watch states built up during outage
)
FAILOVER_HOST1_WRITEBACK_TIER2=(
"/mnt/user/appdata-Failover/Important-Data" # NextCloud + Postgres — files added during outage
)
FAILOVER_HOST1_WRITEBACK_TIER3=(
# "location-placeholder"
)
FAILOVER_HOST1_WRITEBACK_TIER4=(
"/mnt/user/appdata-Failover/Arrs_Stack" # arr databases — downloads queued during outage
)
# ==============================================================================================
# ── MEDIA ─────────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# ━━━ Media Permissions ━━━
# Shares that media_shares_permissions.sh applies PERMISSIONS_MODE and PERMISSIONS_OWNER to.
# Runs first in DAILY_MAINTENANCE_SCRIPTS — arr cleanup depends on correct ownership.
HOST1_MEDIA_PERMISSION_SHARES=(
/mnt/user/Anime_Movies
/mnt/user/Anime_Movies-Old
/mnt/user/Anime_Shows
/mnt/user/Anime_Shows-Old
/mnt/user/appcache
/mnt/user/Books
/mnt/user/Downloads
/mnt/user/Games
/mnt/user/Intros
/mnt/user/Kids_Movies
/mnt/user/Kids_Tv_Shows
/mnt/user/Movie_Recordings
/mnt/user/Movies
/mnt/user/Music
/mnt/user/Music_Videos
/mnt/user/Photo
/mnt/user/Sports
/mnt/user/stand-up_comedy
/mnt/user/Temp_Storage
/mnt/user/Tv_Recordings
/mnt/user/Tv_Shows
/mnt/user/YouTube
)
# ━━━ Media Cleaner ━━━
# Folder lists for media_cleaner.sh — two profiles: anime and media.
# File patterns shared across all servers — defined in master.conf.
# Called via DAILY_MAINTENANCE_SCRIPTS. Run manually: Media/media_cleaner.sh anime|media
HOST1_ANIME_CLEAN_FOLDERS=(
/mnt/user/Anime_Movies
/mnt/user/Anime_Movies-Old
/mnt/user/Anime_Shows
/mnt/user/Anime_Shows-Old
)
HOST1_MEDIA_CLEAN_FOLDERS=(
/mnt/user/Kids_Movies
/mnt/user/Kids_Tv_Shows
/mnt/user/Movies
/mnt/user/Music
/mnt/user/Sports
/mnt/user/stand-up_comedy
/mnt/user/Tv_Shows
)
# ==============================================================================================
# ── MONITORS ──────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# ━━━ Certificate Monitor ━━━
# Domains checked via direct openssl connection — not relying on NPM's certificate state.
# Checks the actual certificate served, not what NPM thinks it has.
# Thresholds (CERT_WARN_DAYS, CERT_CRIT_DAYS) defined in master.conf.
HOST1_CERT_MONITOR_DOMAINS=(
"Gmer4Lfe.com"
"Gmer4Lfe.us"
)
# ━━━ SMART Health ━━━
# Drives skipped in SMART attribute monitoring — hardware is server-specific.
# Thresholds read from dynamix.cfg at runtime — fallbacks in master.conf.
HOST1_SMART_IGNORE_DRIVES=(
"sda" # boot USB — SMART not meaningful on flash drives
)
# ━━━ ZFS Report ━━━
# Pools excluded from the weekly ZFS health report — reduces noise from single-disk array pools.
# These are individual array disks formatted as ZFS — converting to XFS over time via unBalance.
# Pool health thresholds defined in master.conf.
HOST1_ZFS_REPORT_IGNORE_POOLS=(
"disk5"
"disk6"
"disk8"
"disk9"
"disk10"
)
# ==============================================================================================
# ── TRANSCODES ────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Ramdisk size ceiling — tmpfs only uses RAM actually needed, not the full size upfront.
# Real-world: 9 streams peaked at ~5.5GB — 8G gives comfortable headroom on 128GB RAM.
HOST1_RAMDISK_SIZE="8G"
# Usage thresholds — coupled to HOST1_RAMDISK_SIZE, adjust all three together if size changes.
# Hysteresis gap (6.8 - 5.5 = 1.3GB) prevents flip-flop between ramdisk and SSD.
HOST1_RAMDISK_WARN_GB=6.8 # flip to SSD when ramdisk usage reaches this
HOST1_RAMDISK_LOW_GB=5.5 # flip back to ramdisk when usage drops to this
# SSD fallback path — where transcodes land when ramdisk exceeds HOST1_RAMDISK_WARN_GB.
# Must be on cache pool — array disks too slow for active transcode writes.
HOST1_TRANSCODE_SSD="/mnt/cache/Temp_Storage/Emby/Transcodes/"
# Media servers sharing the ramdisk transcode space on HOST1.
# Format: "ContainerName|URL|APIKey|Type" — Type: emby | jellyfin | plex
# Entries with placeholder API keys are skipped automatically.
# ⚠️ Tdarr does NOT belong here — keep Tdarr on SSD, not ramdisk.
HOST1_TRANSCODE_SERVERS=(
"${HOST1_EMBY_CONTAINER}|${HOST1_EMBY_URL}|${HOST1_EMBY_API_KEY}|emby"
)
# ==============================================================================================
# ── ARR STACK ─────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Used by arr cleanup scripts and arrs_failed_stalled_recovery.sh.
# detect_hosts() selects HOST1 vars when running on HOST1.
#
# PATH MAPS — container path → host path translation.
# Arr stores file paths using container-internal paths — scripts need host paths to scan.
# Add one entry per root folder in arr Settings → Media Management → Root Folders.
# ━━━ Downloaders ━━━
# Used by downloaders_reset.sh — runs every 15min via CRITICAL_MAINTENANCE_SCRIPTS.
# Clears stuck states, purges old history, prepares each client for a clean cycle.
# slskd — clears stuck searches, dead transfers, purges expired failed imports.
# SLSKD_FAILED_IMPORTS_DIR: where Soularr moves albums Lidarr rejected.
HOST1_SLSKD_URL="http://localhost:8980"
HOST1_SLSKD_API_KEY="4bF9kL2mNpQrT7vWxYz1A3dEgHjKoRsU"
HOST1_SLSKD_FAILED_IMPORTS_DIR="/mnt/user/Temp_Storage/Slskd/completed/failed_imports"
# SABnzbd
HOST1_SABNZBD_URL="http://localhost:8180"
HOST1_SABNZBD_API_KEY="8bfefe41d83b4d50883e32859b55ca9a"
# qBittorrent — deleteFiles=false removes torrent from qBit but leaves files on disk.
# Radarr/Sonarr manage actual files independently.
HOST1_QBIT_URL="http://localhost:8080"
HOST1_QBIT_USERNAME="root"
HOST1_QBIT_PASSWORD="Stay0utD!ck"
# ━━━ Lidarr — HOST1 only ━━━
# HOST2 does not run Lidarr — HOST1_LIDARR_RECOVERY flag handles the exit cleanly.
HOST1_LIDARR_URL="http://localhost:8686"
HOST1_LIDARR_API_KEY="b2977e71ef074bc0a0529d9fcce3b2dc"
HOST1_LIDARR_MUSIC_ROOT="/mnt/user/Music-New"
declare -A HOST1_LIDARR_PATH_MAP=(
["/ext-music"]="/mnt/user/Music-New"
)
# ━━━ Sonarr ━━━
HOST1_SONARR_URL="http://localhost:8989"
HOST1_SONARR_API_KEY="d43a3ec6cf1549edb4af0cc63f98b2a9"
HOST1_SONARR_TV_ROOT="/mnt/user/Tv_Shows"
# Note: stand-up_comedy in both Sonarr + Radarr — TV specials and movie specials, one folder
declare -A HOST1_SONARR_PATH_MAP=(
["/tv"]="/mnt/user/Tv_Shows"
["/ext-standup-comedy"]="/mnt/user/stand-up_comedy"
["/kids tv"]="/mnt/user/Kids_Tv_Shows"
["/ext-anime-shows"]="/mnt/user/Anime_Shows-Old"
)
# ━━━ Radarr ━━━
HOST1_RADARR_URL="http://localhost:7878"
HOST1_RADARR_API_KEY="d43a3ec6cf1549edb4af0cc63f98b2a9"
HOST1_RADARR_MOVIES_ROOT="/mnt/user/Movies"
# Note: stand-up_comedy in both Radarr + Sonarr — movie specials and TV specials, one folder
declare -A HOST1_RADARR_PATH_MAP=(
["/movies"]="/mnt/user/Movies"
["/kids movies"]="/mnt/user/Kids_Movies"
["/ext-stand-up-comedy"]="/mnt/user/stand-up_comedy"
["/anime-movies"]="/mnt/user/Anime_Movies-Old"
)
# ━━━ Arr Recovery Toggles ━━━
# false = skip that arr on this host — exits cleanly without error
HOST1_SONARR_RECOVERY=true
HOST1_RADARR_RECOVERY=true
HOST1_LIDARR_RECOVERY=true # HOST1 only — exits cleanly on HOST2
# ==============================================================================================
# ── SYSTEM WATCHDOG ───────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Per-host check toggles and NIC config for system_watchdog.sh.
# Aliased by detect_hosts() — script uses unprefixed SYS_WATCHDOG_* names.
# HOST1: TR1950X 128GB — full media server, active transcoding, ZFS cache pools.
#
# Three-tier response — all critical checks enabled by default on HOST1:
# Tier 1 (bypass strikes, reboot now): docker daemon, rootfs full, kernel oops, FD, /boot
# Tier 2 (bypass strikes with OOM): RAM critical + OOM kills in cycle
# Tier 3 (standard strike system): everything else
#
# RAM tiers, OOM limits, and reboot loop settings in master.conf System Watchdog section.
# ━━━ Primary NIC ━━━
# Network interface for NIC state check — verify with: ip link show | grep "^[0-9]"
# Common values: eth0, bond0, br0, eno1
HOST1_SYS_WATCHDOG_NIC="eth0"
# ━━━ Tier 1 — Critical Checks ━━━
# These bypass the strike system — a single hit triggers immediate reboot.
# Disabling any of these is not recommended — they protect against acute system failure.
# Docker daemon unresponsive → try restart, reboot if restart fails.
# Without a working daemon docker_watchdog.sh is blind and containers cannot be managed.
HOST1_SYS_WATCHDOG_CHECK_DOCKER_DAEMON=true
# rootfs at critical threshold (ROOTFS_CRITICAL_PCT=99) → reboot immediately.
# At 99% rootfs writes fail silently — logs stop, Docker errors out, SSH may stop working.
# Standard 95% threshold still uses strike system — only 99%+ is critical tier.
HOST1_SYS_WATCHDOG_CHECK_ROOTFS=true
# Kernel BUG/Oops in dmesg delta since last cycle → reboot immediately.
# A kernel oops means the kernel ran with a corrupted state — stability is not guaranteed.
HOST1_SYS_WATCHDOG_CHECK_KERNEL_OOPS=true
# File descriptor exhaustion at FD_CRITICAL_PCT (95%) → reboot immediately.
# At 95% FD: new connections fail, Docker can't spawn processes, SSH drops.
HOST1_SYS_WATCHDOG_CHECK_FD=true
# /boot read-only detected → reboot immediately.
# Unexpected read-only /boot means state files and config writes are silently failing.
# Failover state, watchdog reboot log, and lock files all go stale silently.
HOST1_SYS_WATCHDOG_CHECK_BOOT=true
# ━━━ Tier 2 — Urgent OOM Check ━━━
# Bypass strikes when RAM is critically low AND OOM kill rate confirms active crisis.
# Both must be enabled for Tier 2 bypass to function — disable either to always use strikes.
# Track kernel OOM kills each cycle via /proc/vmstat oom_kill delta.
# Also provides diagnostic context in reboot messages (which processes were killed).
HOST1_SYS_WATCHDOG_CHECK_OOM=true
# Free RAM check — required for both Tier 2 bypass and RAM tier logic.
# Tiers: MEM_WARN_GB(10) → notify | MEM_SHUTDOWN_GB(6) → stop containers | MEM_GB(4) → strikes
HOST1_SYS_WATCHDOG_CHECK_RAM=true
# ━━━ Tier 3 — Standard Checks (strike system) ━━━
# Each check must fail SYS_WATCHDOG_STRIKE_LIMIT consecutive cycles before action is taken.
# Single spikes are ignored — sustained problems trigger reboot.
# /var/log filesystem usage above SYS_WATCHDOG_LOG_PCT.
# Log spam (Docker log storms, syslog loops) fills rootfs — indicates something broken.
HOST1_SYS_WATCHDOG_CHECK_LOG=true
# ZFS ARC memory pinned above SYS_WATCHDOG_ARC_PINNED_PCT after cache drop.
# Enabled on HOST1 — ZFS cache pools actively used. Disable on hosts without ZFS.
HOST1_SYS_WATCHDOG_CHECK_ARC=true
# CPU temperature above SYS_WATCHDOG_CPU_TEMP_MAX (95°C).
# Sustained high temp causes kernel throttling or panic. Requires lm-sensors.
HOST1_SYS_WATCHDOG_CHECK_CPU_TEMP=true
# Load average above SYS_WATCHDOG_LOAD_MULTIPLIER × core count.
# DISABLED on HOST1 — Tdarr and Emby cause legitimate sustained load spikes during encoding.
# Enable on idle servers or adjust SYS_WATCHDOG_LOAD_MULTIPLIER if load is always high.
HOST1_SYS_WATCHDOG_CHECK_LOAD=false
# Zombie process count above SYS_WATCHDOG_ZOMBIE_LIMIT (50).
# Large zombie counts indicate serious process management failure — something is stuck.
HOST1_SYS_WATCHDOG_CHECK_ZOMBIES=true
# Check docker_watchdog.sh persistent skip list — required containers on skip list.
# Cross-watchdog coordination: if docker_watchdog gave up, system_watchdog escalates.
# ENABLED — HOST1 fully built and operational, skip list is meaningful.
HOST1_SYS_WATCHDOG_CHECK_CONTAINERS=true
# /tmp filesystem usage above SYS_WATCHDOG_TMP_PCT with auto-clear attempt.
# Script tries to clear aged /tmp files first — only strikes if clear fails.
# Lock files, rsync temp files, and Docker ops use /tmp — 100% means lock failures.
HOST1_SYS_WATCHDOG_CHECK_TMP=true
# Array disk error count delta in /proc/mdstat — accumulating errors = disk failing now.
# Triggers on SYS_WATCHDOG_MDSTAT_ERROR_LIMIT new errors in one cycle.
HOST1_SYS_WATCHDOG_CHECK_MDSTAT=true
# Primary NIC operstate — detects NIC going down (physical or driver failure).
# Uses HOST1_SYS_WATCHDOG_NIC above. Strike system — brief flaps don't trigger reboot.
HOST1_SYS_WATCHDOG_CHECK_NETWORK=true
# sshd running check — attempts restart before escalating.
# sshd down = no remote access. Script tries rc.sshd start, notifies, strikes on failure.
HOST1_SYS_WATCHDOG_CHECK_SSHD=true
# Runaway process detection — single process above SYS_WATCHDOG_RUNAWAY_CPU_PCT sustained.
# DISABLED — Tdarr encoding and Emby transcoding legitimately peg CPU for extended periods.
# Enable only if HOST1 has no CPU-intensive workloads.
HOST1_SYS_WATCHDOG_CHECK_RUNAWAY=false
# ==============================================================================================
# ──────────────────────── End Of HOST1 Variables ──────────────────────────────────────────────
# ==============================================================================================
master_host2.conf
+545
View File
@@ -0,0 +1,545 @@
#!/bin/bash
# ==============================================================================================
# ========================== HOST2 CONFIGURATION — unRAID-Jayred365 ===========================
# ==============================================================================================
# HOST2-specific variables — credentials, container names, share paths, failover lists.
# Sourced after master.conf — values here extend shared profile arrays and add HOST2-specific
# identity, credentials, and container configuration.
#
# Sparse checkout (git) ensures HOST1 never receives this file.
# HOST1 never sees HOST2 credentials — clean separation at the file level.
#
# DO NOT put shared config here — thresholds, toggles, profiles belong in master.conf.
# DO NOT put HOST1 variables here — they belong in master_host1.conf.
#
# ── STATUS ────────────────────────────────────────────────────────────────────────────────────
# HOST2 is currently being rebuilt — most sections scaffolded, fill in when back online.
# When ready: set FAILOVER_ENABLED=true and DAILY_RSYNC_ENABLED=true in master.conf.
#
# ── INDEX ─────────────────────────────────────────────────────────────────────────────────────
#
# ── IDENTITY & CONNECTIVITY ────────────────────────────────────────────────────────────────
# IDENTITY hostname, SSH key
# EMBY container name, URL, API key
# NOTIFICATIONS Discord webhook
# PARTNERSHIP auth containers, backup paths
#
# ── RSYNC ──────────────────────────────────────────────────────────────────────────────────
# DAILY SYNC SHARES media shares HOST2 owns and pushes to HOST1
# WEEKLY SYNC SHARES appdata shares synced weekly (Sunday 2:30am)
# CRITICAL SYNC SHARES appdata shares synced every 15 minutes
# BACKUP VERIFY shares for checksum verification against remote
# HOST2 RSYNC PROFILE host2-appdata profile for HOST2-specific appdata syncs
#
# ── DOCKER ─────────────────────────────────────────────────────────────────────────────────
# DOCKER DAILY RESTART containers restarted daily
# DOCKER WEEKLY RESTART containers restarted weekly
# DOCKER WATCHDOG memory limits, health URLs, required containers, ignore list
# DOCKER NETWORK CONNECT networks and containers for docker_network_connect.sh
#
# ── FAILOVER ───────────────────────────────────────────────────────────────────────────────
# DDNS DDNS containers managed by HOST2
# INTERNET LOSS containers stopped when internet is lost
# FAILOVER TIERS what HOST2 runs for HOST1 per tier
# TIER DELAYS how long HOST2 must be down before each tier activates on HOST1
# RSYNC WRITEBACK HOST2 appdata synced back on handback
#
# ── MEDIA ──────────────────────────────────────────────────────────────────────────────────
# MEDIA PERMISSIONS share list for media_shares_permissions.sh
# MEDIA CLEANER folder lists for media_cleaner.sh
#
# ── MONITORS ───────────────────────────────────────────────────────────────────────────────
# CERTIFICATE MONITOR domains checked for SSL expiry
# SMART HEALTH drives to skip in SMART monitoring
# ZFS REPORT pools to exclude from ZFS health report
#
# ── TRANSCODES ─────────────────────────────────────────────────────────────────────────────
# TRANSCODES ramdisk size, thresholds, SSD path, server array
#
# ── ARR STACK ──────────────────────────────────────────────────────────────────────────────
# SONARR URL, API key, path map
# RADARR URL, API key, path map
# ARR RECOVERY per-arr recovery toggles (no Lidarr on HOST2)
#
# ==============================================================================================
# ==============================================================================================
# ── IDENTITY & CONNECTIVITY ───────────────────────────────────────────────────────────────────
# ==============================================================================================
# ━━━ Identity ━━━
# Hostname must match exact unRAID hostname AND Tailscale device name — case sensitive.
# Used by detect_hosts() in common.sh to identify this server as HOST2.
HOST2="unRAID-Jayred365"
# SSH key used for all server-to-server operations — rsync, failover container commands.
# Must be in /root/.ssh/ and authorised in HOST1's /root/.ssh/authorized_keys.
HOST2_SSH_KEY="/root/.ssh/Jayred365-rsync-key"
# ━━━ Emby ━━━
# Referenced by transcode_manager.sh, emby_session_report.sh, emby_database_repair.sh,
# weekly_sync_maintenance.sh, and HOST2_TRANSCODE_SERVERS below.
# API key: Emby Dashboard → API Keys → + New Key
HOST2_EMBY_CONTAINER="Emby-Jayred365"
HOST2_EMBY_URL="http://localhost:8096"
HOST2_EMBY_API_KEY="your-host2-emby-api-key"
# ━━━ Notifications ━━━
# Discord webhook — leave blank to disable.
# Per-host so HOST1 and HOST2 can post to different channels or only one server notifies.
HOST2_DISCORD_WEBHOOK=""
# ━━━ Partnership ━━━
# HOST2 is the mirror — HOST1 is always the owner unless --transfer has been run.
# See README-Partnership.md and master.conf PARTNERSHIP section for full lifecycle docs.
# Auth containers reconfigured on onboard/offboard.
# Format: "ContainerName|WebUIPort"
# On onboard → WebUI pointed at owner's Tailscale IP (mirror clicks NPM, gets owner's auth)
# On offboard → WebUI pointed back at localhost
HOST2_PARTNERSHIP_AUTH_WEBUIS=(
# fill in when HOST2 is back online
# "NginxProxyManager|81"
)
# Paths HOST1 should collect during the grace window after offboard.
# Notified on offboard — no auto-deletion, HOST1 must collect manually within PARTNERSHIP_GRACE_HOURS.
HOST2_PARTNERSHIP_MIRROR_BACKUPS=(
# fill in when HOST2 is back online
)
# ==============================================================================================
# ── RSYNC ─────────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# ━━━ Daily Sync Shares ━━━
# Shares HOST2 owns and pushes to HOST1 every night (1am via daily_sync_maintenance.sh).
# HOST2 is the source of truth — HOST1 is the mirror.
# Never push a share both directions — one server always owns it.
# Uses DEFAULT_RSYNC_OPTS from master.conf — no profile needed.
# For shares needing container stops or custom options — add a profile in master.conf.
HOST2_DAILY_SYNC_SHARES=(
/mnt/user/Anime_Movies
/mnt/user/Anime_Shows
)
# Personal encrypted shares — synced for offsite backup, independent of media shares.
# ZFS encrypted at dataset level — remote receives encrypted blocks, cannot read content.
# See README-Rsync_Setup.md for ZFS encryption setup before uncommenting.
HOST2_PERSONAL_SHARES=(
# /mnt/user/HOST2-Personal # uncomment after creating encrypted dataset
)
# ━━━ Weekly Sync Shares ━━━
# Appdata shares synced during the weekly maintenance window (Sunday 2:30am).
# Containers stopped both sides before sync — full clean state guaranteed.
# Profiles drive container stops, excludes, and options — configured in master.conf RSYNC section.
HOST2_WEEKLY_SYNC_SHARES=(
# fill in when HOST2 is back online
# "/mnt/user/Media_Server/Emby"
# "/mnt/user/appdata-Failover/Critical-Data"
)
# ━━━ Critical Sync Shares ━━━
# Appdata shares synced every 15 minutes by critical_sync_maintenance.sh.
# Format: "/path/to/share" or "/path/to/share|profile-name"
HOST2_CRITICAL_SYNC_SHARES=(
# fill in when HOST2 is back online
# "/mnt/user/appdata-Failover/Critical-Data|critical-failover"
# "/mnt/user/Media_Server/Emby|emby-failover"
)
# ━━━ Backup Verify ━━━
# Shares verified by backup_verify.sh — random file checksum comparison against remote.
# Leave empty to use HOST2_DAILY_SYNC_SHARES automatically.
# Sample size and minimum file size defined in master.conf.
HOST2_BACKUP_VERIFY_SHARES=(
# leave empty to use HOST2_DAILY_SYNC_SHARES automatically
)
# ━━━ HOST2 Rsync Profile — host2-appdata ━━━
# HOST2-specific appdata sync profile — extends the shared PROFILE_* arrays in master.conf.
# Use for appdata unique to HOST2.
# Shared appdata (auth stack, Emby) use dedicated profiles defined in master.conf.
# Run manually: bash Rsync/rsync.sh /mnt/user/appdata-Failover/HOST2-Appdata --profile=host2-appdata
PROFILE_RSYNC_OPTS[host2-appdata]="-av --info=progress2 --bwlimit=${PROFILE_BW_LIMIT[host2-appdata]:-8000}"
PROFILE_BW_LIMIT[host2-appdata]=8000
PROFILE_RETRY_COUNT[host2-appdata]=3
PROFILE_SLEEP[host2-appdata]=300
PROFILE_CRITICAL_CONTAINER_NAMES[host2-appdata]="" # fill in when HOST2 is back online
PROFILE_DELAYED_CONTAINERS[host2-appdata]=""
PROFILE_CONTAINER_DELAY[host2-appdata]=5
PROFILE_EXCLUDE_DIRS[host2-appdata]="logs *.tmp"
# ==============================================================================================
# ── DOCKER ────────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# ━━━ Docker Daily Restart ━━━
# Containers restarted every day via DAILY_MAINTENANCE_SCRIPTS.
# Fill in when HOST2 is back online — add containers that degrade without daily restart.
HOST2_DAILY_RESTART_CONTAINERS=(
"NginxProxyManager"
# add HOST2 daily restart containers here
)
# ━━━ Docker Weekly Restart ━━━
# Less critical services restarted weekly via WEEKLY_MAINTENANCE_SCRIPTS (Sunday 2:30am).
# Containers already stopped for weekly sync — restart adds zero extra downtime.
HOST2_WEEKLY_RESTART_CONTAINERS=(
# add HOST2 weekly restart containers here
)
# ━━━ Docker Watchdog ━━━
# Per-HOST2 container configuration for docker_watchdog.sh.
# Shared thresholds and toggles live in master.conf.
# Memory hard limits in MB — immediate restart if exceeded.
# Set at "container is clearly broken" not "container is busy".
# 20GB=20480 16GB=16384 12GB=12288 10GB=10240 8GB=8192 4GB=4096 2GB=2048 1GB=1024
declare -A HOST2_WATCHDOG_CONTAINERS=(
["Emby"]=16384 # fill in correct limit when HOST2 is back online
)
# HTTP health check URLs — checked every cycle, strike system before restart.
# Only add containers with a meaningful web interface to check.
declare -A HOST2_WATCHDOG_CONTAINER_URLS=(
["Emby"]="http://localhost:8096"
)
# Required containers — must always be running on HOST2.
# Strike system before restart — repeated failures go on skip list, auto-clears on recovery.
# Listed in dependency order — dependencies before dependents.
HOST2_WATCHDOG_REQUIRED_CONTAINERS=(
"NginxProxyManager"
# add HOST2 required containers here when back online
)
# Containers to skip in Tier 2 global scan — legitimately stopped or frequently restarting.
# Watchdog leaves these alone entirely — no restart attempts, no crash loop tracking.
HOST2_WATCHDOG_SCAN_IGNORE=(
# add HOST2 scan ignore containers here when back online
)
# Dependency ordering — skip restarting a container if its dependency is also down.
# Prevents watchdog from restarting dependent services before their dependencies are up.
# SPACE-SEPARATED STRINGS — converted to array at runtime.
declare -A HOST2_WATCHDOG_DEPENDENCIES=(
# add HOST2 dependencies here when containers are defined
# ["Authelia"]="Mariadb-Authelia Redis-Authelia"
)
# ━━━ Docker Network Connect ━━━
# Containers connected to custom networks at array start by docker_network_connect.sh.
# Networks created if they don't exist — idempotent, safe to re-run.
HOST2_NETWORK_CONNECT_CONTAINERS=(
# fill in when HOST2 is back online
)
HOST2_NETWORK_CONNECT_NETWORKS=(
# fill in when HOST2 is back online
)
# ==============================================================================================
# ── FAILOVER ──────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# ━━━ DDNS ━━━
# DDNS containers HOST2 manages — started/stopped by failover.sh per DDNS absolute rules:
# Internet loss → stop immediately
# Failover → HOST1 starts HOST2's DDNS as Tier 1 (before any other containers)
# Handback → stop HOST2's DDNS on HOST1 → rsync → start containers → start local DDNS last
HOST2_DDNS_CONTAINERS=(
"Gmer4Lfe.us"
)
# ━━━ Internet Loss ━━━
# Containers stopped immediately on HOST2 when internet connection is lost.
# Prevents external-facing services from operating without connectivity.
FAILOVER_HOST2_STOP_ON_NO_NET=(
"Gmer4Lfe.us"
)
# ━━━ Failover Tiers — HOST2 Runs for HOST1 ━━━
# Containers HOST2 starts when HOST1 goes down.
# Tier 1 is always immediate — vital services cannot wait.
# Higher tiers activate after HOST1_TIER*_DELAY minutes (set in master_host1.conf).
FAILOVER_HOST2_RUNS_FOR_HOST1_TIER1=(
"Gmer4Lfe.com"
"Emby"
"VaultWarden-Gmer4Lfe"
"Dispatcharr"
"Dispatcharr-Basic"
"Dispatcharr-Iptv-Users"
"ErsatzTV-Emby"
)
FAILOVER_HOST2_RUNS_FOR_HOST1_TIER2=(
"Postgres-NextCloud"
"NextCloud"
"PostgreSQL_Immich"
"Immich-Gmer4Lfe"
)
FAILOVER_HOST2_RUNS_FOR_HOST1_TIER3=(
"Gitea"
)
FAILOVER_HOST2_RUNS_FOR_HOST1_TIER4=(
"Sonarr"
"Radarr"
"Lidarr"
"Readarr"
"Prowlarr"
"Bazarr"
"SABnzbd-Gmer4Lfe"
"Qbittorrent-Gmer4Lfe"
"LidaTube"
"Pinchflat"
"ChannelTube"
)
# ━━━ Tier Delays — HOST2's Containers on HOST1 ━━━
# How long HOST2 must be down before each tier activates on HOST1 — in minutes.
# Tier 1 is always immediate — no delay var needed.
HOST2_TIER2_DELAY=240 # 4 hours — productivity services
HOST2_TIER3_DELAY=720 # 12 hours — secondary services
HOST2_TIER4_DELAY=1440 # 24 hours — arrs + downloaders
# ━━━ Rsync Writeback — HOST2 Appdata Back on Handback ━━━
# Syncs HOST2 appdata BACK to HOST2 when it comes back online after a failover.
# Containers stopped before writeback — clean source, no competing writes.
#
# HOST2_TIER1_WRITEBACK_DELAY: short outages skip Tier 1 writeback — primary state
# is more reliable than dirty sync data for brief outages.
HOST2_TIER1_WRITEBACK_DELAY=60 # skip writeback if outage under 1hr
# Tier 4 automatically syncs HOST2_DAILY_SYNC_SHARES — only list paths NOT in that array.
FAILOVER_HOST2_WRITEBACK_TIER1=(
# "/mnt/user/appdata-Failover/Jayred365-Emby"
)
FAILOVER_HOST2_WRITEBACK_TIER2=(
# "/mnt/user/appdata-Failover/Jayred365-Important"
)
FAILOVER_HOST2_WRITEBACK_TIER3=(
# "location-placeholder"
)
FAILOVER_HOST2_WRITEBACK_TIER4=(
"/mnt/user/appdata-Failover/Arrs_Stack" # arr databases — downloads queued during outage
)
# ==============================================================================================
# ── MEDIA ─────────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# ━━━ Media Permissions ━━━
# Shares that media_shares_permissions.sh applies PERMISSIONS_MODE and PERMISSIONS_OWNER to.
# Runs first in DAILY_MAINTENANCE_SCRIPTS — arr cleanup depends on correct ownership.
HOST2_MEDIA_PERMISSION_SHARES=(
/mnt/user/Anime_Movies
/mnt/user/Anime_Shows
)
# ━━━ Media Cleaner ━━━
# Folder lists for media_cleaner.sh — two profiles: anime and media.
# File patterns shared across all servers — defined in master.conf.
# Called via DAILY_MAINTENANCE_SCRIPTS. Run manually: Media/media_cleaner.sh anime|media
HOST2_ANIME_CLEAN_FOLDERS=(
/mnt/user/Anime_Movies
/mnt/user/Anime_Shows
)
HOST2_MEDIA_CLEAN_FOLDERS=(
# fill in when HOST2 is back online
)
# ==============================================================================================
# ── MONITORS ──────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# ━━━ Certificate Monitor ━━━
# Domains checked via direct openssl connection — not relying on NPM's certificate state.
# Checks the actual certificate served, not what NPM thinks it has.
# Thresholds (CERT_WARN_DAYS, CERT_CRIT_DAYS) defined in master.conf.
HOST2_CERT_MONITOR_DOMAINS=(
# fill in when HOST2 is back online
)
# ━━━ SMART Health ━━━
# Drives skipped in SMART attribute monitoring — hardware is server-specific.
# Thresholds read from dynamix.cfg at runtime — fallbacks in master.conf.
HOST2_SMART_IGNORE_DRIVES=(
"sda" # boot USB — SMART not meaningful on flash drives
)
# ━━━ ZFS Report ━━━
# Pools excluded from the weekly ZFS health report — reduces noise from single-disk array pools.
# Pool health thresholds defined in master.conf.
HOST2_ZFS_REPORT_IGNORE_POOLS=(
# fill in when HOST2 is back online
)
# ==============================================================================================
# ── TRANSCODES ────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Ramdisk size ceiling — tmpfs only uses RAM actually needed, not the full size upfront.
# Adjust HOST2_RAMDISK_WARN_GB and HOST2_RAMDISK_LOW_GB together if this changes.
HOST2_RAMDISK_SIZE="8G"
# Usage thresholds — coupled to HOST2_RAMDISK_SIZE, adjust all three together if size changes.
# Hysteresis gap (6.8 - 5.5 = 1.3GB) prevents flip-flop between ramdisk and SSD.
HOST2_RAMDISK_WARN_GB=6.8 # flip to SSD when ramdisk usage reaches this
HOST2_RAMDISK_LOW_GB=5.5 # flip back to ramdisk when usage drops to this
# SSD fallback path — where transcodes land when ramdisk exceeds HOST2_RAMDISK_WARN_GB.
# Must be on cache pool — array disks too slow for active transcode writes.
HOST2_TRANSCODE_SSD="/mnt/cache/Temp_Storage/Emby/Transcodes/"
# Media servers sharing the ramdisk transcode space on HOST2.
# Format: "ContainerName|URL|APIKey|Type" — Type: emby | jellyfin | plex
# Entries with placeholder API keys are skipped automatically.
# ⚠️ Tdarr does NOT belong here — keep Tdarr on SSD, not ramdisk.
HOST2_TRANSCODE_SERVERS=(
"${HOST2_EMBY_CONTAINER}|${HOST2_EMBY_URL}|${HOST2_EMBY_API_KEY}|emby"
)
# ==============================================================================================
# ── ARR STACK ─────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Used by arr cleanup scripts and arrs_failed_stalled_recovery.sh.
# detect_hosts() selects HOST2 vars when running on HOST2.
# Lidarr does not run on HOST2 — HOST1_LIDARR_RECOVERY flag handles the exit cleanly.
#
# PATH MAPS — container path → host path translation.
# Arr stores file paths using container-internal paths — scripts need host paths to scan.
# Add one entry per root folder in arr Settings → Media Management → Root Folders.
# ━━━ Sonarr ━━━
HOST2_SONARR_URL="http://localhost:8989"
HOST2_SONARR_API_KEY="130decd3db5b4c25afad64864cd03f9f"
HOST2_SONARR_TV_ROOT="/mnt/user/Anime_Shows"
declare -A HOST2_SONARR_PATH_MAP=(
# fill in when HOST2 is back online
# ["/tv"]="/mnt/user/Anime_Shows"
)
# ━━━ Radarr ━━━
HOST2_RADARR_URL="http://localhost:7878"
HOST2_RADARR_API_KEY="d43a3ec6cf1549edb4af0cc63f98b2a9"
HOST2_RADARR_MOVIES_ROOT="/mnt/user/Anime_Movies"
declare -A HOST2_RADARR_PATH_MAP=(
# fill in when HOST2 is back online
# ["/anime-movies"]="/mnt/user/Anime_Movies"
)
# ━━━ Arr Recovery Toggles ━━━
# false = skip that arr on this host — exits cleanly without error
HOST2_SONARR_RECOVERY=true
HOST2_RADARR_RECOVERY=true
# HOST2_LIDARR_RECOVERY not set — Lidarr does not run on HOST2
# ==============================================================================================
# ──────────────────────── End Of HOST2 Variables ──────────────────────────────────────────────
# ==============================================================================================
# ==============================================================================================
# ── SYSTEM WATCHDOG ───────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Per-host check toggles and NIC config for system_watchdog.sh.
# Aliased by detect_hosts() — script uses unprefixed SYS_WATCHDOG_* names.
# HOST2: i5 10th gen 64GB — being rebuilt, lighter workload, no ZFS cache pools.
#
# Conservative defaults during rebuild — re-enable checks as HOST2 stabilises.
# Three-tier response — all critical checks enabled regardless of rebuild state:
# Tier 1 (bypass strikes, reboot now): docker daemon, rootfs full, kernel oops, FD, /boot
# Tier 2 (bypass strikes with OOM): RAM critical + OOM kills in cycle
# Tier 3 (standard strike system): selectively disabled during rebuild
#
# RAM tiers, OOM limits, and reboot loop settings in master.conf System Watchdog section.
# ━━━ Primary NIC ━━━
# Network interface for NIC state check — verify with: ip link show | grep "^[0-9]"
# Common values: eth0, bond0, br0, eno1
HOST2_SYS_WATCHDOG_NIC="eth0"
# ━━━ Tier 1 — Critical Checks ━━━
# All critical checks always enabled — these protect against acute failure regardless of
# rebuild state. Disabling any is not recommended.
# Docker daemon unresponsive → try restart, reboot if restart fails.
HOST2_SYS_WATCHDOG_CHECK_DOCKER_DAEMON=true
# rootfs at critical threshold (ROOTFS_CRITICAL_PCT=99) → reboot immediately.
HOST2_SYS_WATCHDOG_CHECK_ROOTFS=true
# Kernel BUG/Oops in dmesg delta since last cycle → reboot immediately.
HOST2_SYS_WATCHDOG_CHECK_KERNEL_OOPS=true
# File descriptor exhaustion at FD_CRITICAL_PCT (95%) → reboot immediately.
HOST2_SYS_WATCHDOG_CHECK_FD=true
# /boot read-only detected → reboot immediately.
HOST2_SYS_WATCHDOG_CHECK_BOOT=true
# ━━━ Tier 2 — Urgent OOM Check ━━━
# Both must be enabled for Tier 2 bypass to function.
# Track kernel OOM kills each cycle via /proc/vmstat oom_kill delta.
HOST2_SYS_WATCHDOG_CHECK_OOM=true
# Free RAM check — 64GB RAM on HOST2, tiers adjusted relative to HOST1.
# Update master.conf SYS_WATCHDOG_MEM_* thresholds if HOST2 needs different values.
# Currently inheriting shared master.conf values — may want lower thresholds on 64GB.
HOST2_SYS_WATCHDOG_CHECK_RAM=true
# ━━━ Tier 3 — Standard Checks (strike system) ━━━
# Several checks disabled during rebuild — enable progressively as HOST2 stabilises.
# Each check must fail SYS_WATCHDOG_STRIKE_LIMIT consecutive cycles before action.
# /var/log filesystem usage above SYS_WATCHDOG_LOG_PCT.
HOST2_SYS_WATCHDOG_CHECK_LOG=true
# ZFS ARC memory check.
# DISABLED — HOST2 has no ZFS cache pools. Enable if ZFS pools are added later.
HOST2_SYS_WATCHDOG_CHECK_ARC=false
# CPU temperature above SYS_WATCHDOG_CPU_TEMP_MAX (95°C).
HOST2_SYS_WATCHDOG_CHECK_CPU_TEMP=true
# Load average above SYS_WATCHDOG_LOAD_MULTIPLIER × core count.
# DISABLED — rebuild operations cause legitimate load spikes. Enable after rebuild.
HOST2_SYS_WATCHDOG_CHECK_LOAD=false
# Zombie process count above SYS_WATCHDOG_ZOMBIE_LIMIT (50).
HOST2_SYS_WATCHDOG_CHECK_ZOMBIES=true
# docker_watchdog.sh persistent skip list check.
# DISABLED during rebuild — skip list may be unreliable mid-rebuild, avoid false reboots.
# Enable once HOST2 is fully operational and docker_watchdog.sh is running stably.
HOST2_SYS_WATCHDOG_CHECK_CONTAINERS=false
# /tmp filesystem usage with auto-clear attempt.
HOST2_SYS_WATCHDOG_CHECK_TMP=true
# Array disk error count delta in /proc/mdstat.
HOST2_SYS_WATCHDOG_CHECK_MDSTAT=true
# Primary NIC operstate — uses HOST2_SYS_WATCHDOG_NIC above.
HOST2_SYS_WATCHDOG_CHECK_NETWORK=true
# sshd running check — restart attempt before escalating.
HOST2_SYS_WATCHDOG_CHECK_SSHD=true
# Runaway process detection.
# DISABLED — rebuild workloads may legitimately peg CPU. Enable after rebuild.
HOST2_SYS_WATCHDOG_CHECK_RUNAWAY=false
# ==============================================================================================
# ──────────────────────── End Of HOST2 Variables ──────────────────────────────────────────────
# ==============================================================================================
safe_master.conf
-1518
View File
File diff suppressed because it is too large Load Diff
unRAID_Essentials/README-Unraid_Essentials.md
+750 -262
View File
File diff suppressed because it is too large Load Diff
unRAID_Essentials/clear_logs.sh
+222 -102
View File
@@ -1,134 +1,254 @@
#!/bin/bash
# -----------------------------------------------------------------------------------------------
# --------------------------------- Clear Logs Script ------------------------------------------
# -----------------------------------------------------------------------------------------------
# Clears unRAID system logs and Docker container logs safely.
# Log file paths are configured in Master.conf under LOG_FILES.
# Supports --dry-run to preview what would be cleared without making changes.
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ================================= Clear Logs =================================================
# ==============================================================================================
# Clears system and Docker container logs to prevent rootfs fill over time.
# Runs weekly via WEEKLY_MAINTENANCE_SCRIPTS — Sunday 2:30am.
# Uses size thresholds — only clears logs that have grown large enough to matter.
#
# ── WHAT IT CLEARS ────────────────────────────────────────────────────────────────────────────
# System logs — LOG_FILES from master.conf (/var/log/syslog, messages, dmesg)
# Cleared if size exceeds LOG_MIN_SIZE_MB
# These grow continuously — weekly clearing keeps rootfs healthy
#
# Docker logs — /var/lib/docker/containers/**/*-json.log
# Cleared only if individual container log exceeds LOG_DOCKER_MAX_MB
# Active containers (Emby, SABnzbd) grow fastest — 100MB+ easily
# Inactive containers not cleared — their logs are typically small
#
# ── SIZE THRESHOLD APPROACH ───────────────────────────────────────────────────────────────────
# Truncating everything blindly destroys useful diagnostic context.
# A 2MB log is not worth clearing — it contains useful recent history.
# A 500MB log is consuming rootfs and contains mostly noise — clear it.
#
# LOG_MIN_SIZE_MB — system logs under this size are left alone
# LOG_DOCKER_MAX_MB — Docker logs under this size are left alone
#
# ── WHY NOT LOGROTATE ─────────────────────────────────────────────────────────────────────────
# unRAID writes to tmpfs (/var/log) — logrotate's compress + archive approach
# would consume even more tmpfs space. Truncation (: > file) keeps the file
# descriptor open and valid while emptying content — safe for running services.
#
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
# acquire_lock — prevents concurrent runs corrupting logs
# Root check — truncating system logs requires root
# Size thresholds — only clears logs that have grown large enough
# Byte tracking — reports MB freed for weekly digest
# validate_unraid — notify validated before use
# Silent on clean — small logs = nothing to clear = no output ✅
#
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
# LOG_FILES — system log paths to check and clear
# LOG_MIN_SIZE_MB — minimum system log size before clearing (default 10MB)
# LOG_DOCKER_MAX_MB — clear Docker log only if above this size (default 100MB)
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
# clear_logs.sh — normal run (threshold-based clearing)
# clear_logs.sh --dry-run — show what would be cleared and sizes
# clear_logs.sh --status — show current log sizes
# clear_logs.sh --log — verbose output per file
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../Master.conf"
source "$SCRIPT_DIR/../common.sh"
source "$SCRIPT_DIR/../load_config.sh"
parse_args "$@"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Setup ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_GEAR Setup ━━━"
# ROOT CHECK
# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
error "Must be run as root"
error "Must be run as root — truncating system logs requires root"
exit 1
fi
success "Running as root"
validate_unraid_cmd \
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
"" "" \
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Status ━━━
# -----------------------------------------------------------------------------------------------
acquire_lock
detect_hosts
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be cleared"
# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
echo ""
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
echo "$ICON_HEALTH System Logs: ${LOG_FILES[*]}"
echo "$ICON_CONTAINERS Docker Logs: /var/lib/docker/containers"
echo "$ICON_GEAR Dry Run: $DRY_RUN"
echo "━━━━━ $ICON_SUMMARY LOG STATUS ━━━━━"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_GEAR Min system log: ${LOG_MIN_SIZE_MB:-10}MB before clearing"
echo "$ICON_GEAR Max Docker log: ${LOG_DOCKER_MAX_MB:-100}MB before clearing"
echo ""
echo "━━━ System Logs ━━━"
for f in "${LOG_FILES[@]}"; do
if [[ -f "$f" ]]; then
size=$(du -sh "$f" 2>/dev/null | cut -f1)
size_mb=$(du -sm "$f" 2>/dev/null | cut -f1)
threshold="${LOG_MIN_SIZE_MB:-10}"
if [[ "${size_mb:-0}" -ge "$threshold" ]]; then
echo " $ICON_WARN $f — $size (above ${threshold}MB threshold — would clear)"
else
echo " $ICON_SUCCESS $f — $size (under threshold)"
fi
else
echo " $ICON_SKIP $f — not found"
fi
done
echo ""
echo "━━━ Docker Logs (top 10 by size) ━━━"
if [[ -d /var/lib/docker/containers ]]; then
find /var/lib/docker/containers/ -name "*-json.log" 2>/dev/null | \
while IFS= read -r logfile; do
size_mb=$(du -sm "$logfile" 2>/dev/null | cut -f1)
container_id=$(basename "$(dirname "$logfile")" | cut -c1-12)
container_name=$(docker inspect --format '{{.Name}}' "$container_id" \
2>/dev/null | tr -d '/' || echo "$container_id")
echo "${size_mb:-0} $container_name $logfile"
done | sort -rn | head -10 | \
while read -r size_mb name logfile; do
threshold="${LOG_DOCKER_MAX_MB:-100}"
if [[ "$size_mb" -ge "$threshold" ]]; then
echo " $ICON_WARN ${size_mb}MB — $name (above ${threshold}MB — would clear)"
else
echo " $ICON_SUCCESS ${size_mb}MB — $name"
fi
done
else
echo " Docker directory not found"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
fi
# -----------------------------------------------------------------------------------------------
# FUNCTIONS
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Clear Logs ━━━
# ==============================================================================================
START=$(date +%s)
SYS_CLEARED=0
SYS_SKIPPED=0
SYS_BYTES=0
DOCKER_CLEARED=0
DOCKER_SKIPPED=0
DOCKER_BYTES=0
FAILED=()
# Clears a single log file if it exists.
# Skips with a warning if the file is not found.
clear_file() {
local file="$1"
# ── System Logs ───────────────────────────────────────────────────────────────────────────────
for logfile in "${LOG_FILES[@]}"; do
if [[ ! -f "$logfile" ]]; then
log "$logfile — not found, skipping"
continue
fi
if [[ ! -f "$file" ]]; then
warn "Not found: $file — skipping"
return
size_bytes=$(stat -c%s "$logfile" 2>/dev/null || echo 0)
size_mb=$(( size_bytes / 1048576 ))
size_h=$(du -sh "$logfile" 2>/dev/null | cut -f1)
threshold="${LOG_MIN_SIZE_MB:-10}"
if [[ "$size_mb" -lt "$threshold" ]]; then
log "$logfile — ${size_h} (under ${threshold}MB — skipping)"
(( SYS_SKIPPED++ ))
continue
fi
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would clear: $file"
warn "DRY RUN — would clear: $logfile (${size_h})"
(( SYS_CLEARED++ ))
SYS_BYTES=$(( SYS_BYTES + size_bytes ))
continue
fi
if : > "$logfile" 2>/dev/null; then
log "Cleared: $logfile (freed ${size_h})"
(( SYS_CLEARED++ ))
SYS_BYTES=$(( SYS_BYTES + size_bytes ))
else
: > "$file"
success "Cleared: $file"
error "Failed to clear: $logfile"
FAILED+=("$logfile")
fi
}
# Finds and clears all Docker container json log files.
# Skips gracefully if Docker directory or log files are not found.
clear_docker_logs() {
if [[ ! -d /var/lib/docker/containers ]]; then
warn "$ICON_CONTAINERS Docker directory not found — skipping"
return
fi
local files
files=$(find /var/lib/docker/containers/ -name "*-json.log" 2>/dev/null || true)
if [[ -z "$files" ]]; then
warn "$ICON_CONTAINERS No Docker logs found — skipping"
return
fi
while IFS= read -r file; do
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would clear Docker log: $file"
else
: > "$file"
success "Cleared Docker log: $(basename "$(dirname "$file")")"
fi
done <<< "$files"
}
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_HEALTH Clear Logs ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_HEALTH Clear Logs ━━━"
echo "$ICON_HEALTH System Logs: ${LOG_FILES[*]}"
echo "$ICON_CONTAINERS Docker Logs: enabled"
echo "$ICON_GEAR Dry Run: $DRY_RUN"
echo ""
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
START=$(date +%s)
CLEAR_FAILED=false
for logfile in "${LOG_FILES[@]}"; do
clear_file "$logfile" || CLEAR_FAILED=true
done
echo ""
echo "━━━ $ICON_CONTAINERS Docker ━━━"
clear_docker_logs
# ── Docker Logs ───────────────────────────────────────────────────────────────────────────────
if [[ ! -d /var/lib/docker/containers ]]; then
log "Docker containers directory not found — skipping Docker log clear"
else
while IFS= read -r logfile; do
[[ -z "$logfile" ]] && continue
size_bytes=$(stat -c%s "$logfile" 2>/dev/null || echo 0)
size_mb=$(( size_bytes / 1048576 ))
size_h=$(du -sh "$logfile" 2>/dev/null | cut -f1)
threshold="${LOG_DOCKER_MAX_MB:-100}"
# Get container name for display
container_id=$(basename "$(dirname "$logfile")" | cut -c1-12)
container_name=$(docker inspect --format '{{.Name}}' "$container_id" \
2>/dev/null | tr -d '/' || echo "$container_id")
if [[ "$size_mb" -lt "$threshold" ]]; then
log "Docker $container_name — ${size_h} (under ${threshold}MB — skipping)"
(( DOCKER_SKIPPED++ ))
continue
fi
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would clear Docker log: $container_name (${size_h})"
(( DOCKER_CLEARED++ ))
DOCKER_BYTES=$(( DOCKER_BYTES + size_bytes ))
continue
fi
if : > "$logfile" 2>/dev/null; then
log "Cleared Docker log: $container_name (freed ${size_h})"
(( DOCKER_CLEARED++ ))
DOCKER_BYTES=$(( DOCKER_BYTES + size_bytes ))
else
error "Failed to clear Docker log: $container_name"
FAILED+=("docker:$container_name")
fi
done < <(find /var/lib/docker/containers/ -name "*-json.log" 2>/dev/null)
fi
END=$(date +%s)
TOTAL_BYTES=$(( SYS_BYTES + DOCKER_BYTES ))
TOTAL_FREED_H=$(awk "BEGIN {printf \"%.1fMB\", $TOTAL_BYTES / 1048576}")
TOTAL_CLEARED=$(( SYS_CLEARED + DOCKER_CLEARED ))
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Summary ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━━━ $ICON_SUMMARY LOG CLEANER SUMMARY ━━━━━"
echo "$ICON_HEALTH System Logs: ${LOG_FILES[*]}"
echo "$ICON_CONTAINERS Docker Logs: $([[ "$DRY_RUN" == true ]] && echo "skipped (dry run)" || echo "cleared")"
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
if [[ "$DRY_RUN" == true ]]; then
echo "$ICON_WARN Status: DRY RUN — no changes made"
elif [[ "$CLEAR_FAILED" == true ]]; then
echo "$ICON_ERROR Status: $ICON_ERROR SOME LOGS FAILED TO CLEAR"
notify "Log clear completed with errors on $(hostname)" "Clear Logs" "warning"
# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
if [[ "$TOTAL_CLEARED" -gt 0 || ${#FAILED[@]} -gt 0 ]]; then
echo ""
echo "━━━━━ $ICON_SUMMARY LOG CLEANER SUMMARY ━━━━━"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_HEALTH System cleared: $SYS_CLEARED file(s)"
echo "$ICON_CONTAINERS Docker cleared: $DOCKER_CLEARED file(s)"
echo "$ICON_HEALTH Total freed: $TOTAL_FREED_H"
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
[[ "$SYS_SKIPPED" -gt 0 || "$DOCKER_SKIPPED" -gt 0 ]] && \
log "Skipped: ${SYS_SKIPPED} system + ${DOCKER_SKIPPED} Docker (under threshold)"
echo ""
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — no files cleared"
elif [[ ${#FAILED[@]} -gt 0 ]]; then
echo "$ICON_ERROR Status: SOME FILES FAILED — ${FAILED[*]}"
notify "Log clear failed on $(hostname) ($MY_ID) — ${FAILED[*]}" \
"Clear Logs" "warning"
else
log "$ICON_DONE Status: done — ${TOTAL_FREED_H} freed"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
else
echo "$ICON_DONE Status: $ICON_SUCCESS DONE"
notify "Logs cleared successfully on $(hostname)" "Clear Logs" "normal"
# All logs under threshold — completely silent
log "All logs under threshold — nothing to clear"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
[[ ${#FAILED[@]} -gt 0 ]] && exit 1
exit 0
unRAID_Essentials/docker_syslog_filter.sh
+173 -94
View File
@@ -1,125 +1,204 @@
#!/bin/bash
# -----------------------------------------------------------------------------------------------
# --------------------------------- Docker Syslog Filter ---------------------------------------
# -----------------------------------------------------------------------------------------------
# Suppresses noisy Docker veth/docker0 syslog messages on unRAID boot.
# Creates an rsyslog filter file and restarts the rsyslog service.
# Filter file path is configured in Master.conf under FILTER_FILE.
# Supports --dry-run to preview what would be done without making changes.
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ============================= Docker Syslog Filter ===========================================
# ==============================================================================================
# Suppresses noisy Docker veth/docker0 interface messages from syslog.
# Run once at array start via ARRAY_START_SCRIPTS in master.conf.
# Idempotent — completely silent when filter is already correct.
#
# ── WHY THIS EXISTS ───────────────────────────────────────────────────────────────────────────
# Every time Docker creates or destroys a container network interface it logs messages like:
# kernel: veth2a3b4c5: renamed from eth0
# kernel: docker0: port 1(veth2a3b4c5) entered blocking state
# kernel: docker0: port 1(veth2a3b4c5) entered forwarding state
#
# On a busy server creating and restarting many containers these fill syslog rapidly —
# hundreds of entries per minute on container restarts, completely masking real events.
# The filter tells rsyslog to drop these before they reach the log file.
#
# ── HOW IT WORKS ──────────────────────────────────────────────────────────────────────────────
# Creates /etc/rsyslog.d/ignore-docker-veth.conf (FILTER_FILE in master.conf).
# rsyslog processes .conf files in /etc/rsyslog.d/ automatically on startup.
# Filter uses rsyslog's RainerScript to match messages containing "veth" or "docker0"
# and calls stop — the message is dropped before reaching any output target.
#
# ── IDEMPOTENT DESIGN ─────────────────────────────────────────────────────────────────────────
# On every array start: checks if filter file already exists with correct content.
# If already correct → completely silent — no rsyslog restart, no output.
# Only writes + restarts rsyslog if filter is missing or content has changed.
# This prevents unnecessary rsyslog restarts on every boot.
#
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
# Root check — writing to /etc/rsyslog.d/ requires root
# acquire_lock — prevents concurrent runs at array start
# Idempotent check — only restarts rsyslog when filter actually changed
# Directory creation — mkdir -p /etc/rsyslog.d/ before writing
# rsyslog verify — checks rsyslog running after restart
# validate_unraid — notify validated before use
# Silent on success — runs every boot, no noise when already correct
#
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
# FILTER_FILE — path for rsyslog drop filter (default /etc/rsyslog.d/ignore-docker-veth.conf)
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
# docker_syslog_filter.sh — normal run (idempotent)
# docker_syslog_filter.sh --dry-run — show what would change
# docker_syslog_filter.sh --status — show filter file state and rsyslog status
# docker_syslog_filter.sh --log — verbose output
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../Master.conf"
source "$SCRIPT_DIR/../common.sh"
source "$SCRIPT_DIR/../load_config.sh"
parse_args "$@"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Setup ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_GEAR Setup ━━━"
# Expected filter content — used for idempotent check
EXPECTED_FILTER='if ($msg contains "veth" or $msg contains "docker0") then {
stop
}'
# ROOT CHECK
# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
error "Must be run as root"
error "Must be run as root — writing to /etc/rsyslog.d/ requires root"
exit 1
fi
success "Running as root"
validate_unraid_cmd \
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
"" "" \
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Status ━━━
# -----------------------------------------------------------------------------------------------
acquire_lock
detect_hosts
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
echo ""
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
echo "$ICON_HEALTH Filter File: $FILTER_FILE"
echo "$ICON_CONTAINERS Targets: veth, docker0"
echo "$ICON_GEAR Dry Run: $DRY_RUN"
echo "━━━━━ $ICON_SUMMARY SYSLOG FILTER STATUS ━━━━━"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_HEALTH Filter file: $FILTER_FILE"
echo ""
if [[ -f "$FILTER_FILE" ]]; then
echo " Filter file: EXISTS"
echo ""
echo " Current content:"
while IFS= read -r line; do
echo " $line"
done < "$FILTER_FILE"
echo ""
if [[ "$(cat "$FILTER_FILE" 2>/dev/null)" == "$EXPECTED_FILTER" ]]; then
echo " $ICON_SUCCESS Content: correct ✅"
else
echo " $ICON_WARN Content: differs from expected — would be rewritten"
fi
else
echo " Filter file: NOT FOUND — would be created"
fi
echo ""
echo "━━━ rsyslog Status ━━━"
if pgrep -x rsyslogd >/dev/null 2>&1; then
echo " rsyslogd: running ✅"
else
echo " rsyslogd: NOT running"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
fi
# -----------------------------------------------------------------------------------------------
# FUNCTIONS
# -----------------------------------------------------------------------------------------------
# Creates the rsyslog filter file that suppresses veth and docker0 noise.
# Filter is written to FILTER_FILE defined in Master.conf.
create_filter() {
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would create filter file: $FILTER_FILE"
return
fi
info "Writing rsyslog filter file: $FILTER_FILE"
cat <<'EOF' > "$FILTER_FILE"
if ($msg contains "veth" or $msg contains "docker0") then {
stop
}
EOF
success "Filter file written"
}
# Restarts the rsyslog service to apply the new filter.
# Uses unRAID's native rc.rsyslogd script.
restart_rsyslog() {
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would restart rsyslog service"
return 0
fi
info "Restarting rsyslog..."
if /etc/rc.d/rc.rsyslogd restart; then
success "rsyslog restarted"
return 0
else
error "Failed to restart rsyslog"
return 1
fi
}
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_HEALTH Syslog Filter ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_HEALTH Syslog Filter ━━━"
echo "$ICON_HEALTH Filter File: $FILTER_FILE"
echo "$ICON_CONTAINERS Targets: veth / docker0"
echo "$ICON_GEAR Dry Run: $DRY_RUN"
echo ""
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
# ==============================================================================================
# ━━━ Idempotent Check ━━━
# ==============================================================================================
# Already correct — completely silent
if [[ -f "$FILTER_FILE" ]] && \
[[ "$(cat "$FILTER_FILE" 2>/dev/null)" == "$EXPECTED_FILTER" ]]; then
log "Filter already correct — no changes needed"
exit 0
fi
# ==============================================================================================
# ━━━ Apply Filter ━━━
# ==============================================================================================
START=$(date +%s)
create_filter
log "Filter file missing or outdated — applying..."
RSYSLOG_OK=true
restart_rsyslog || RSYSLOG_OK=false
# Ensure rsyslog.d directory exists
if [[ "$DRY_RUN" == false ]]; then
mkdir -p "$(dirname "$FILTER_FILE")" || {
error "Failed to create directory: $(dirname "$FILTER_FILE")"
exit 1
}
fi
# Write filter file
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would write filter to: $FILTER_FILE"
warn "Content:"
echo "$EXPECTED_FILTER" | while IFS= read -r line; do
echo " $line"
done
else
echo "$EXPECTED_FILTER" > "$FILTER_FILE" || {
error "Failed to write filter file: $FILTER_FILE"
notify "Syslog filter write failed on $(hostname) ($MY_ID)" \
"Syslog Filter" "warning"
exit 1
}
log "Filter file written: $FILTER_FILE"
fi
# Restart rsyslog to apply
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would restart rsyslog"
else
log "Restarting rsyslog..."
if /etc/rc.d/rc.rsyslogd restart >/dev/null 2>&1; then
sleep 2
# Verify rsyslog actually running after restart
if pgrep -x rsyslogd >/dev/null 2>&1; then
log "rsyslog restarted and running ✅"
else
error "rsyslog not running after restart"
notify "rsyslog failed to start after filter update on $(hostname) ($MY_ID)" \
"Syslog Filter" "warning"
exit 1
fi
else
error "rsyslog restart command failed"
notify "rsyslog restart failed on $(hostname) ($MY_ID) — filter may not be active" \
"Syslog Filter" "warning"
exit 1
fi
fi
END=$(date +%s)
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Summary ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
echo ""
echo "━━━━━ $ICON_SUMMARY SYSLOG FILTER SUMMARY ━━━━━"
echo "$ICON_HEALTH Filter File: $FILTER_FILE"
echo "$ICON_CONTAINERS Targets: veth / docker0"
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_HEALTH Filter file: $FILTER_FILE"
echo "$ICON_HEALTH Targets: veth / docker0"
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
echo ""
if [[ "$DRY_RUN" == true ]]; then
echo "$ICON_WARN Status: DRY RUN — no changes made"
elif [[ "$RSYSLOG_OK" == false ]]; then
echo "$ICON_ERROR Status: $ICON_ERROR RSYSLOG RESTART FAILED"
notify "rsyslog restart failed on $(hostname) — syslog filter may not be active" "Syslog Filter" "warning"
warn "DRY RUN — no changes made"
else
echo "$ICON_DONE Status: $ICON_SUCCESS DONE"
notify "Syslog filter applied on $(hostname)" "Syslog Filter" "normal"
log "$ICON_DONE Status: done — Docker veth noise suppressed ✅"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
unRAID_Essentials/inotify_tuning.sh
+159 -72
View File
@@ -1,100 +1,177 @@
#!/bin/bash
# -----------------------------------------------------------------------------------------------
# --------------------------------- inotify Tuning --------------------------------------------
# -----------------------------------------------------------------------------------------------
# Increases Linux inotify limits to prevent "too many open files" and inotify exhaustion.
# Run once at array start via ARRAY_START_SCRIPTS in Master.conf.
# ==============================================================================================
# ================================= inotify Tuning ============================================
# ==============================================================================================
# Raises Linux inotify limits at array start to prevent exhaustion across the container stack.
# Run once at array start via ARRAY_START_SCRIPTS in master.conf.
# Settings are lost on reboot — this script reapplies them on every array start.
#
# Why this matters:
# Each Docker container that watches files (Sonarr, Radarr, Lidarr, NextCloud etc.)
# consumes inotify instances and watches. unRAID defaults are very low — with many
# containers running you can exhaust the limit silently, causing containers to miss
# file events (new downloads not detected, library not updated etc.)
# ── THREE INOTIFY LIMITS ──────────────────────────────────────────────────────────────────────
# max_user_instances — max number of independent inotify file descriptor objects per user
# Each container that calls inotify_init() consumes one instance
# Default 128 — exhausted quickly with 20+ active containers
#
# max_user_instances = max number of inotify instances per user (default: 128)
# max_user_watches = max number of files/dirs watched per instance (default: 8192)
# max_queued_events = max events queued before dropping (default: 16384)
# max_user_watches — SHARED budget across ALL users and containers on the system
# Each watched file or directory costs one watch from this pool
# Default 8192 — VSCode alone can need 50K-200K for large workspaces
#
# These settings are lost on reboot — this script reapplies them at every array start.
# All values configurable in Master.conf under unRAID Essentials.
# -----------------------------------------------------------------------------------------------
# max_queued_events — max events buffered before kernel starts dropping them
# Low value = events silently lost during high-activity periods
# Default 16384 — sufficient for most setups
#
# ── WHY VSCODE THROWS "UNABLE TO WATCH FOR FILE CHANGES" ─────────────────────────────────────
# VSCode (and Code-Server in Docker) opens one inotify watch per file in the workspace.
# A typical project with node_modules can easily have 100K-200K files.
# All containers on the host share max_user_watches — the combined usage of:
# Sonarr, Radarr, Lidarr, Emby, Nextcloud, Code-Server, AdGuard, all other arrs
# easily exceeds 524288 (512K) watches on a busy server.
# Raising to 1048576 (1M) gives sufficient headroom — safe on 128GB RAM (~128MB kernel use).
#
# ── STARTUP ORDER MATTERS ─────────────────────────────────────────────────────────────────────
# inotify_tuning.sh must run BEFORE containers that watch files start.
# In ARRAY_START_SCRIPTS order: inotify_tuning.sh first, then container-starting scripts.
# If Code-Server starts before limits are raised it inherits the old (low) limits.
# Code-Server restart fixes this: limits are kernel-wide, not process-bound at start.
# So if Code-Server is already running: docker restart Code-Server after this script runs.
#
# ── CONSUMERS ON THIS STACK ───────────────────────────────────────────────────────────────────
# Emby — watches all media library paths (1 watch per folder)
# Sonarr — watches TV_Shows folder tree
# Radarr — watches Movies folder tree
# Lidarr — watches Music folder tree
# Nextcloud — watches data directory for changes
# Code-Server — watches entire workspace (can be 50K-200K with node_modules)
# AdGuard Home — watches config directory
# + all other containers using inotify internally
#
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
# acquire_lock — prevents duplicate runs at array start
# Root check — sysctl writes require root
# validate_unraid — notify validated before use
# Silent on success — runs every boot, no noise when already correct
# Only warns on changes or failures
#
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
# INOTIFY_MAX_INSTANCES — default 1024
# INOTIFY_MAX_WATCHES — default 1048576 (1M)
# INOTIFY_MAX_QUEUED_EVENTS — default 32768
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
# inotify_tuning.sh — normal run (apply settings)
# inotify_tuning.sh --dry-run — show what would change
# inotify_tuning.sh --status — show current vs target values and top consumers
# inotify_tuning.sh --log — verbose output
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../Master.conf"
source "$SCRIPT_DIR/../common.sh"
source "$SCRIPT_DIR/../load_config.sh"
parse_args "$@"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Setup ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_GEAR inotify Tuning ━━━"
# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
error "Must be run as root"
error "Must be run as root — sysctl writes require root"
exit 1
fi
success "Running as root"
validate_unraid_cmd \
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
"" "" \
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
# -----------------------------------------------------------------------------------------------
# ━━━ Current Values ━━━
# -----------------------------------------------------------------------------------------------
CURRENT_INSTANCES=$(sysctl -n fs.inotify.max_user_instances 2>/dev/null || echo "?")
CURRENT_WATCHES=$(sysctl -n fs.inotify.max_user_watches 2>/dev/null || echo "?")
CURRENT_EVENTS=$(sysctl -n fs.inotify.max_queued_events 2>/dev/null || echo "?")
acquire_lock
info "Current: instances=$CURRENT_INSTANCES watches=$CURRENT_WATCHES queued=$CURRENT_EVENTS"
info "Target: instances=${INOTIFY_MAX_INSTANCES} watches=${INOTIFY_MAX_WATCHES} queued=${INOTIFY_MAX_QUEUED_EVENTS}"
detect_hosts
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
echo ""
echo "━━━━━ $ICON_SUMMARY INOTIFY STATUS ━━━━━"
echo " max_user_instances: $CURRENT_INSTANCES (target: ${INOTIFY_MAX_INSTANCES})"
echo " max_user_watches: $CURRENT_WATCHES (target: ${INOTIFY_MAX_WATCHES})"
echo " max_queued_events: $CURRENT_EVENTS (target: ${INOTIFY_MAX_QUEUED_EVENTS})"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo ""
echo " Active inotify instances in use:"
find /proc/*/fd -lname "anon_inode:inotify" 2>/dev/null | \
awk -F/ '{print $3}' | sort -u | while read -r pid; do
cmd=$(cat /proc/$pid/comm 2>/dev/null || echo "?")
echo " PID $pid ($cmd)"
done | head -20
echo "━━━ Kernel Limits ━━━"
CURRENT_INSTANCES=$(sysctl -n fs.inotify.max_user_instances 2>/dev/null || echo "?")
CURRENT_WATCHES=$(sysctl -n fs.inotify.max_user_watches 2>/dev/null || echo "?")
CURRENT_EVENTS=$(sysctl -n fs.inotify.max_queued_events 2>/dev/null || echo "?")
for label in "max_user_instances current=$CURRENT_INSTANCES target=$INOTIFY_MAX_INSTANCES" \
"max_user_watches current=$CURRENT_WATCHES target=$INOTIFY_MAX_WATCHES" \
"max_queued_events current=$CURRENT_EVENTS target=$INOTIFY_MAX_QUEUED_EVENTS"; do
echo " $label"
done
echo ""
echo "━━━ Active Instances ━━━"
USED_INSTANCES=$(find /proc/*/fd -lname 'anon_inode:inotify' 2>/dev/null | wc -l)
USED_INSTANCES="${USED_INSTANCES//[^0-9]/}"
echo " Instances in use: ${USED_INSTANCES:-0} / $CURRENT_INSTANCES"
if [[ "$CURRENT_INSTANCES" -gt 0 ]]; then
PCT=$(( ${USED_INSTANCES:-0} * 100 / CURRENT_INSTANCES ))
echo " Utilisation: ${PCT}%"
fi
echo ""
echo "━━━ Top Consumers ━━━"
find /proc/*/fd -lname 'anon_inode:inotify' 2>/dev/null | \
awk -F/ '{print $3}' | sort | uniq -c | sort -rn | head -10 | \
while read -r count pid; do
cmd=$(cat /proc/"$pid"/comm 2>/dev/null || echo "?")
cgroup=$(cat /proc/"$pid"/cgroup 2>/dev/null | \
grep docker | grep -o '[a-f0-9]\{12\}' | head -1 || echo "")
if [[ -n "$cgroup" ]]; then
label="[docker:${cgroup}] $cmd"
else
label="[host] $cmd"
fi
echo " ${count} instances — $label (PID $pid)"
done | head -10
echo ""
echo "━━━ VSCode / Code-Server ━━━"
echo " If VSCode shows 'unable to watch for file changes':"
echo " 1. Verify max_user_watches target is set high enough"
echo " 2. Check total watches used: cat /proc/sys/fs/inotify/max_user_watches"
echo " 3. After any limit change: docker restart Code-Server"
echo " (running containers inherit limits at start, not dynamically)"
echo "━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
fi
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Apply Settings ━━━
# -----------------------------------------------------------------------------------------------
echo ""
# ==============================================================================================
CHANGED=0
FAILED=0
apply_sysctl() {
local key="$1"
local value="$2"
local key="$1" value="$2"
local current
current=$(sysctl -n "$key" 2>/dev/null || echo 0)
if [[ "$current" -eq "$value" ]]; then
success "$key = $value (already set)"
return
log "$key = $value (already correct)"
return 0
fi
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would set $key = $value (currently $current)"
return
return 0
fi
if sysctl -w "${key}=${value}" >/dev/null 2>&1; then
success "$key = $value (was $current)"
((CHANGED++))
warn "Set $key = $value (was $current)"
(( CHANGED++ ))
else
error "Failed to set $key = $value"
((FAILED++))
(( FAILED++ ))
fi
}
@@ -102,25 +179,35 @@ apply_sysctl "fs.inotify.max_user_instances" "$INOTIFY_MAX_INSTANCES"
apply_sysctl "fs.inotify.max_user_watches" "$INOTIFY_MAX_WATCHES"
apply_sysctl "fs.inotify.max_queued_events" "$INOTIFY_MAX_QUEUED_EVENTS"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Summary ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━━━ $ICON_SUMMARY INOTIFY TUNING SUMMARY ━━━━━"
echo " max_user_instances: $(sysctl -n fs.inotify.max_user_instances 2>/dev/null)"
echo " max_user_watches: $(sysctl -n fs.inotify.max_user_watches 2>/dev/null)"
echo " max_queued_events: $(sysctl -n fs.inotify.max_queued_events 2>/dev/null)"
echo ""
if [[ "$DRY_RUN" == true ]]; then
echo "$ICON_WARN Status: DRY RUN — no changes made"
elif [[ "$FAILED" -gt 0 ]]; then
echo "$ICON_ERROR Status: $FAILED setting(s) failed"
notify "inotify tuning failed on $(hostname) — $FAILED setting(s) could not be applied" "inotify Tuning" "warning"
# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
if [[ "$FAILED" -gt 0 ]]; then
echo ""
echo "━━━━━ $ICON_SUMMARY INOTIFY TUNING SUMMARY ━━━━━"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_ERROR $FAILED setting(s) failed to apply"
notify "inotify tuning failed on $(hostname) ($MY_ID) — $FAILED setting(s) could not be applied" \
"inotify Tuning" "warning"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
exit 1
elif [[ "$CHANGED" -gt 0 ]]; then
echo "$ICON_DONE Status: $ICON_SUCCESS $CHANGED setting(s) applied"
echo ""
echo "━━━━━ $ICON_SUMMARY INOTIFY TUNING SUMMARY ━━━━━"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo " max_user_instances: $(sysctl -n fs.inotify.max_user_instances 2>/dev/null)"
echo " max_user_watches: $(sysctl -n fs.inotify.max_user_watches 2>/dev/null)"
echo " max_queued_events: $(sysctl -n fs.inotify.max_queued_events 2>/dev/null)"
echo ""
warn "$CHANGED setting(s) updated"
if [[ "$CHANGED" -gt 0 ]]; then
warn "If Code-Server is running: docker restart Code-Server"
warn "Running containers inherit limits at start — restart picks up new values"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
else
echo "$ICON_DONE Status: $ICON_SUCCESS All settings already correct"
# Already correct — completely silent (runs every boot)
log "inotify limits already correct — no changes needed"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
unRAID_Essentials/mover_stop.sh
+129 -83
View File
@@ -1,116 +1,162 @@
#!/bin/bash
# -----------------------------------------------------------------------------------------------
# --------------------------------- Mover Stop Script ------------------------------------------
# -----------------------------------------------------------------------------------------------
# Safely stops the unRAID mover process with a user warning before halting.
# Timeout before stopping is configured in Master.conf under MOVER_STOP_TIMEOUT.
# Supports --dry-run to preview what would happen without making changes.
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ================================= Mover Stop =================================================
# ==============================================================================================
# Safely stops the unRAID mover process with a warning before halting.
# Warns all logged-in users via wall message, waits the configured timeout, then stops.
#
# ── WHEN TO USE ───────────────────────────────────────────────────────────────────────────────
# - Before a planned reboot when mover is running mid-cycle
# - Before disk replacement or array operations that need mover stopped
# - Before rsync — mover and rsync simultaneously moving the same files causes corruption
# - Called automatically by maintenance scripts that need the mover stopped first
#
# ── STOP SEQUENCE ─────────────────────────────────────────────────────────────────────────────
# 1. Check if mover is running — exit cleanly if not
# 2. Broadcast wall warning to all logged-in users
# 3. Wait MOVER_STOP_TIMEOUT seconds (default 30) — gives active sessions a chance to note it
# 4. Send SIGTERM — mover can complete its current file operation before exiting
# 5. Wait 5 seconds for graceful exit
# 6. Verify stopped — if still running send SIGKILL (force)
# 7. Final verify — error if still running after SIGKILL
#
# ── SIGTERM vs SIGKILL ────────────────────────────────────────────────────────────────────────
# SIGTERM first — allows mover to finish the file it is currently moving (no partial files).
# SIGKILL only as fallback — forces immediate stop (may leave partial files on cache or array).
#
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
# acquire_lock — prevents concurrent stop attempts racing each other
# Root check — pkill on emhttp processes requires root
# validate_unraid — notify validated before use
# SIGTERM → verify → SIGKILL sequence — graceful then forced
# Final verify — confirms mover actually stopped
# Silent on clean — mover not running = log() only, no output ✅
#
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
# MOVER_STOP_TIMEOUT — seconds to warn users before stopping (default 30)
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
# mover_stop.sh — stop mover with configured timeout
# mover_stop.sh --dry-run — show what would happen
# mover_stop.sh --status — show mover state
# mover_stop.sh --log — verbose output
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../Master.conf"
source "$SCRIPT_DIR/../common.sh"
source "$SCRIPT_DIR/../load_config.sh"
parse_args "$@"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Setup ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_GEAR Setup ━━━"
# ROOT CHECK
# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
error "Must be run as root"
error "Must be run as root — pkill on emhttp processes requires root"
exit 1
fi
success "Running as root"
validate_unraid_cmd \
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
"" "" \
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
# Validate MOVER_STOP_TIMEOUT is a valid integer before using it
validate_int MOVER_STOP_TIMEOUT "$MOVER_STOP_TIMEOUT"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Status ━━━
# -----------------------------------------------------------------------------------------------
acquire_lock
detect_hosts
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
echo ""
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
echo "$ICON_MOVER Timeout: ${MOVER_STOP_TIMEOUT}s"
echo "$ICON_GEAR Dry Run: $DRY_RUN"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_MOVER Timeout: ${MOVER_STOP_TIMEOUT}s"
echo "$ICON_GEAR Dry Run: $DRY_RUN"
echo ""
if pgrep -f "emhttp.*Mover" >/dev/null 2>&1; then
MOVER_PID=$(pgrep -f "emhttp.*Mover" | head -1)
MOVER_START=$(ps -o lstart= -p "$MOVER_PID" 2>/dev/null | xargs)
echo " $ICON_MOVER Mover: RUNNING (PID $MOVER_PID)"
[[ -n "$MOVER_START" ]] && echo " $ICON_TIME Started: $MOVER_START"
else
echo " $ICON_MOVER Mover: not running"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
fi
# -----------------------------------------------------------------------------------------------
# FUNCTIONS
# -----------------------------------------------------------------------------------------------
# Returns 0 if the unRAID mover process is currently running, 1 if not.
check_mover_running() {
pgrep -f "emhttp.*Mover" >/dev/null 2>&1
}
# Broadcasts a wall message to all logged in users warning mover is stopping.
notify_users() {
warn "Notifying users — mover stopping in ${MOVER_STOP_TIMEOUT}s"
wall "$ICON_WARN unRAID Mover will stop in ${MOVER_STOP_TIMEOUT} second(s)."
}
# Sends SIGTERM to the mover process via pkill.
# Skips if dry run is active.
stop_mover() {
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would stop unRAID Mover process"
return
fi
info "Stopping unRAID Mover..."
if pkill -f "emhttp.*Mover"; then
success "Mover stopped"
else
warn "Could not stop mover — may have already stopped"
fi
}
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_MOVER Mover Stop ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_MOVER Mover Stop ━━━"
echo "$ICON_MOVER Timeout: ${MOVER_STOP_TIMEOUT}s"
echo "$ICON_GEAR Dry Run: $DRY_RUN"
echo ""
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
# ==============================================================================================
# ━━━ Mover Stop ━━━
# ==============================================================================================
START=$(date +%s)
if check_mover_running; then
info "$ICON_MOVER Mover is running"
notify_users
info "Waiting ${MOVER_STOP_TIMEOUT}s before stopping..."
if ! pgrep -f "emhttp.*Mover" >/dev/null 2>&1; then
log "Mover is not running — nothing to do"
exit 0
fi
MOVER_PID=$(pgrep -f "emhttp.*Mover" | head -1)
warn "Mover is running (PID $MOVER_PID) — stopping in ${MOVER_STOP_TIMEOUT}s"
# ── Warn users via wall ───────────────────────────────────────────────────────────────────────
if [[ "$DRY_RUN" == false ]]; then
wall "$ICON_WARN $MY_ID ($LOCAL_SERVER_NAME) — unRAID Mover stopping in ${MOVER_STOP_TIMEOUT}s"
log "Wall message sent — waiting ${MOVER_STOP_TIMEOUT}s..."
sleep "$MOVER_STOP_TIMEOUT"
stop_mover
else
info "$ICON_MOVER Mover is not running — nothing to do"
warn "DRY RUN — would send wall warning and wait ${MOVER_STOP_TIMEOUT}s"
fi
# ── SIGTERM — graceful stop ───────────────────────────────────────────────────────────────────
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would send SIGTERM to mover (PID $MOVER_PID)"
else
log "Sending SIGTERM to mover (PID $MOVER_PID)..."
pkill -TERM -f "emhttp.*Mover" 2>/dev/null || true
sleep 5
# Verify stopped after SIGTERM
if ! pgrep -f "emhttp.*Mover" >/dev/null 2>&1; then
warn "Mover stopped cleanly (SIGTERM) ✅"
else
# ── SIGKILL — forced stop ─────────────────────────────────────────────────────────────
warn "Mover still running after SIGTERM — sending SIGKILL (may leave partial files)"
pkill -KILL -f "emhttp.*Mover" 2>/dev/null || true
sleep 2
# Final verify
if pgrep -f "emhttp.*Mover" >/dev/null 2>&1; then
error "Mover still running after SIGKILL — manual intervention needed"
notify "Mover stop failed on $(hostname) ($MY_ID) — process unkillable" \
"Mover Stop" "warning"
exit 1
else
warn "Mover force-stopped (SIGKILL) — check for partial files on cache"
fi
fi
fi
END=$(date +%s)
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Summary ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
echo ""
echo "━━━━━ $ICON_SUMMARY MOVER STOP SUMMARY ━━━━━"
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
if check_mover_running; then
echo "$ICON_ERROR Status: $ICON_ERROR STILL RUNNING"
notify "Mover stop failed — mover still running on $(hostname)" "Mover Stop" "warning"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_MOVER Timeout: ${MOVER_STOP_TIMEOUT}s"
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
echo ""
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — no changes made"
else
echo "$ICON_DONE Status: $ICON_SUCCESS STOPPED / NOT RUNNING"
notify "Mover stopped on $(hostname)" "Mover Stop" "normal"
log "$ICON_DONE Status: done — mover stopped ✅"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
unRAID_Essentials/php_fpm_max_children.sh
+176 -111
View File
@@ -1,141 +1,206 @@
#!/bin/bash
# -----------------------------------------------------------------------------------------------
# ----------------------------- PHP-FPM Max Children Script ------------------------------------
# -----------------------------------------------------------------------------------------------
# Persistently sets PHP-FPM pm.max_children on unRAID.
# Config file path and max children value are set in Master.conf.
# Supports --dry-run to preview what would be changed without making changes.
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ============================= PHP-FPM Max Children ===========================================
# ==============================================================================================
# Persistently sets PHP-FPM pm.max_children on unRAID to prevent WebGUI slowdowns.
# Run once at array start via ARRAY_START_SCRIPTS in master.conf.
# Idempotent — completely silent when value is already correct.
#
# ── WHY THIS EXISTS ───────────────────────────────────────────────────────────────────────────
# unRAID's WebGUI runs through PHP-FPM. The default pm.max_children is very low (4-8).
# Under load — multiple users, Docker operations, heavy dashboard usage — all PHP workers
# saturate and new requests queue. The WebGUI becomes slow or unresponsive.
#
# pm.max_children controls how many PHP worker processes can run simultaneously.
# Raising it allows the WebGUI to handle more concurrent requests without queuing.
# Too high: wastes RAM. Too low: WebGUI slowdowns.
# PHP_MAX_CHILDREN=250 is appropriate for 128GB — ~2MB per worker = ~500MB total.
#
# ── WHY IDEMPOTENT ────────────────────────────────────────────────────────────────────────────
# This runs at every array start. If the value is already correct there is nothing to do —
# no config write, no PHP-FPM restart. Restarting PHP-FPM unnecessarily disrupts active
# WebGUI sessions and is annoying on every boot.
#
# ── APPLY SEQUENCE ────────────────────────────────────────────────────────────────────────────
# 1. Read current pm.max_children from PHP_CONF
# 2. If already at target → exit silently (idempotent)
# 3. Verify sed pattern matches before writing
# 4. Apply sed replacement
# 5. Restart PHP-FPM via rc.php-fpm
# 6. Verify PHP-FPM process running after restart
# 7. Verify config file reflects target value
#
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
# Root check — writing to system config requires root
# acquire_lock — prevents concurrent runs at array start
# Idempotent check — only restarts PHP-FPM when value actually changes
# Pattern match check — verifies sed found pm.max_children before writing
# Process verify — confirms PHP-FPM running after restart
# Config verify — reads back config to confirm value applied
# validate_unraid — notify validated before use
# Silent on correct — runs every boot, no noise when already set ✅
#
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
# PHP_MAX_CHILDREN — target pm.max_children value (default 250)
# PHP_CONF — path to PHP-FPM www.conf (default /etc/php83/php-fpm.d/www.conf)
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
# php_fpm_max_children.sh — normal run (idempotent)
# php_fpm_max_children.sh --dry-run — show what would change
# php_fpm_max_children.sh --status — show current vs target and process state
# php_fpm_max_children.sh --log — verbose output
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../Master.conf"
source "$SCRIPT_DIR/../common.sh"
source "$SCRIPT_DIR/../load_config.sh"
parse_args "$@"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Setup ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_GEAR Setup ━━━"
# ROOT CHECK
# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
error "Must be run as root"
error "Must be run as root — writing system config requires root"
exit 1
fi
success "Running as root"
validate_unraid_cmd \
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
"" "" \
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
# VALIDATION
validate_int PHP_MAX_CHILDREN "$PHP_MAX_CHILDREN"
require_var PHP_CONF
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Status ━━━
# -----------------------------------------------------------------------------------------------
acquire_lock
detect_hosts
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
echo ""
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
echo "$ICON_GEAR Config File: $PHP_CONF"
echo "$ICON_PHP Max Children: $PHP_MAX_CHILDREN"
echo "$ICON_GEAR Dry Run: $DRY_RUN"
echo "━━━━━ $ICON_SUMMARY PHP-FPM STATUS ━━━━━"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_GEAR Config file: $PHP_CONF"
echo "$ICON_PHP Target: pm.max_children = $PHP_MAX_CHILDREN"
echo ""
if [[ -f "$PHP_CONF" ]]; then
CURRENT_VAL=$(grep -E "^pm\.max_children" "$PHP_CONF" 2>/dev/null | \
awk '{print $NF}')
if [[ "${CURRENT_VAL:-0}" -eq "$PHP_MAX_CHILDREN" ]]; then
echo " $ICON_SUCCESS Current: pm.max_children = $CURRENT_VAL (correct ✅)"
else
echo " $ICON_WARN Current: pm.max_children = ${CURRENT_VAL:-not set} (would update)"
fi
else
echo " $ICON_ERROR Config file not found: $PHP_CONF"
fi
echo ""
if pgrep -f "php-fpm" >/dev/null 2>&1; then
FPM_COUNT=$(pgrep -fc "php-fpm" 2>/dev/null || echo "?")
echo " $ICON_SUCCESS PHP-FPM: running ($FPM_COUNT worker(s))"
else
echo " $ICON_ERROR PHP-FPM: NOT running"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
fi
# -----------------------------------------------------------------------------------------------
# FUNCTIONS
# -----------------------------------------------------------------------------------------------
# Applies pm.max_children to the PHP-FPM config file and restarts the service.
# Verifies the value was applied correctly after restart.
# Skips all changes if dry run is active.
apply_php_max_children() {
local target="pm.max_children = $PHP_MAX_CHILDREN"
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would set pm.max_children = $PHP_MAX_CHILDREN in $PHP_CONF"
warn "DRY RUN — would restart PHP-FPM service"
return 0
fi
# Verify config file exists before attempting changes
if [[ ! -f "$PHP_CONF" ]]; then
error "PHP config file not found: $PHP_CONF"
return 1
fi
info "Applying pm.max_children = $PHP_MAX_CHILDREN..."
if ! sed -i "s/^pm\.max_children.*/$target/" "$PHP_CONF"; then
error "Failed to update PHP config: $PHP_CONF"
return 1
fi
success "Config updated"
info "Restarting PHP-FPM..."
if ! /etc/rc.d/rc.php-fpm restart; then
error "PHP-FPM restart failed"
return 1
fi
success "PHP-FPM restarted"
# Verify the value was applied correctly
local current
current=$(grep -E "^pm\.max_children" "$PHP_CONF" || true)
if [[ -n "$current" ]]; then
success "Verified: $current"
logger "Userscript: PHP-FPM updated → $current"
else
warn "Could not verify configuration value — check $PHP_CONF manually"
fi
return 0
}
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_PHP PHP-FPM Config ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_PHP PHP-FPM Config ━━━"
echo "$ICON_GEAR Config File: $PHP_CONF"
echo "$ICON_PHP Max Children: $PHP_MAX_CHILDREN"
echo "$ICON_GEAR Dry Run: $DRY_RUN"
echo ""
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
# ==============================================================================================
# ━━━ PHP-FPM Config ━━━
# ==============================================================================================
START=$(date +%s)
PHP_SUCCESS=false
apply_php_max_children && PHP_SUCCESS=true
if [[ ! -f "$PHP_CONF" ]]; then
error "PHP config file not found: $PHP_CONF"
notify "PHP-FPM config not found on $(hostname) ($MY_ID) — $PHP_CONF missing" \
"PHP-FPM" "warning"
exit 1
fi
# ── Idempotent check ─────────────────────────────────────────────────────────────────────────
CURRENT_VAL=$(grep -E "^pm\.max_children" "$PHP_CONF" 2>/dev/null | awk '{print $NF}')
if [[ "${CURRENT_VAL:-0}" -eq "$PHP_MAX_CHILDREN" ]]; then
log "pm.max_children already $PHP_MAX_CHILDREN — no changes needed"
exit 0
fi
warn "pm.max_children: ${CURRENT_VAL:-not set} → $PHP_MAX_CHILDREN"
# ── Verify pattern exists before writing ─────────────────────────────────────────────────────
if ! grep -qE "^pm\.max_children" "$PHP_CONF" 2>/dev/null; then
error "pm.max_children not found in $PHP_CONF — cannot apply"
error "Add 'pm.max_children = $PHP_MAX_CHILDREN' to $PHP_CONF manually"
notify "PHP-FPM pm.max_children not found in config on $(hostname) ($MY_ID)" \
"PHP-FPM" "warning"
exit 1
fi
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would set pm.max_children = $PHP_MAX_CHILDREN in $PHP_CONF"
warn "DRY RUN — would restart PHP-FPM"
exit 0
fi
# ── Apply setting ─────────────────────────────────────────────────────────────────────────────
log "Applying pm.max_children = $PHP_MAX_CHILDREN..."
if ! sed -i "s/^pm\.max_children.*/pm.max_children = $PHP_MAX_CHILDREN/" "$PHP_CONF"; then
error "Failed to update $PHP_CONF"
notify "PHP-FPM config update failed on $(hostname) ($MY_ID)" \
"PHP-FPM" "warning"
exit 1
fi
log "Config updated"
# ── Restart PHP-FPM ──────────────────────────────────────────────────────────────────────────
log "Restarting PHP-FPM..."
if ! /etc/rc.d/rc.php-fpm restart >/dev/null 2>&1; then
error "PHP-FPM restart command failed"
notify "PHP-FPM restart failed on $(hostname) ($MY_ID)" \
"PHP-FPM" "warning"
exit 1
fi
sleep 3 # Allow PHP-FPM workers to initialise
# ── Verify process running ────────────────────────────────────────────────────────────────────
if ! pgrep -f "php-fpm" >/dev/null 2>&1; then
error "PHP-FPM not running after restart — WebGUI may be broken"
notify "PHP-FPM failed to start after config update on $(hostname) ($MY_ID)" \
"PHP-FPM" "warning"
exit 1
fi
# ── Verify config reflects target ────────────────────────────────────────────────────────────
APPLIED_VAL=$(grep -E "^pm\.max_children" "$PHP_CONF" 2>/dev/null | awk '{print $NF}')
if [[ "${APPLIED_VAL:-0}" -ne "$PHP_MAX_CHILDREN" ]]; then
warn "Config reads pm.max_children = ${APPLIED_VAL:-unknown} — expected $PHP_MAX_CHILDREN"
warn "Check $PHP_CONF manually"
else
log "Verified: pm.max_children = $APPLIED_VAL ✅"
fi
END=$(date +%s)
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Summary ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
echo ""
echo "━━━━━ $ICON_SUMMARY PHP-FPM SUMMARY ━━━━━"
echo "$ICON_GEAR Config File: $PHP_CONF"
echo "$ICON_PHP Max Children: $PHP_MAX_CHILDREN"
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
if [[ "$DRY_RUN" == true ]]; then
echo "$ICON_WARN Status: DRY RUN — no changes made"
elif [[ "$PHP_SUCCESS" == true ]]; then
echo "$ICON_DONE Status: $ICON_SUCCESS DONE"
notify "PHP-FPM max_children set to $PHP_MAX_CHILDREN on $(hostname)" "PHP-FPM" "normal"
else
echo "$ICON_ERROR Status: $ICON_ERROR FAILED"
notify "PHP-FPM config update failed on $(hostname)" "PHP-FPM" "warning"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_GEAR Config file: $PHP_CONF"
echo "$ICON_PHP Applied: pm.max_children = $PHP_MAX_CHILDREN"
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
echo ""
log "$ICON_DONE Status: done ✅"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
[[ "$PHP_SUCCESS" == false ]] && exit 1
exit 0
unRAID_Essentials/rsync_stop.sh
+260 -199
View File
@@ -1,135 +1,184 @@
#!/bin/bash
# -----------------------------------------------------------------------------------------------
# --------------------------------- Rsync Stop Script ------------------------------------------
# -----------------------------------------------------------------------------------------------
# Stops rsync intelligently — auto-detects what's running and acts accordingly.
# ==============================================================================================
# ================================= Rsync Stop =================================================
# ==============================================================================================
# Stops rsync intelligently on both local and remote servers.
# Auto-detects orchestrators and chooses the safest stop strategy automatically.
#
# Default behavior (just run it):
# Detects if an orchestrator (daily/weekly) is running
# If yes → kills rsync subprocess only
# orchestrator sees rsync died → moves to next share or exits cleanly
# If no → kills rsync processes directly (solo rsync.sh run)
# Cleans stale lock files
# Recovers any containers left stopped by interrupted rsync
# ── TWO MODES ─────────────────────────────────────────────────────────────────────────────────
# Default (smart):
# Detects if an orchestrator (daily/weekly/critical sync) is running
# If orchestrator found → kills rsync subprocess only
# Orchestrator sees rsync died → moves to next share or exits cleanly
# If no orchestrator → kills rsync directly (standalone rsync.sh run)
# Cleans stale lock files after kill
# Recovers containers left stopped by interrupted rsync (local only)
#
# --full-stop flag (nuclear):
# Kills orchestrator first → then rsync
# Use when: you want everything dead immediately
# daily/weekly loop will NOT continue to next share
# --full-stop (nuclear):
# Kills orchestrator first → then kills rsync
# Orchestrator will NOT continue to next share
# Use when: you need everything dead immediately
#
# Both local and remote are handled in one run.
# Remote containers left as-is — docker_watchdog.sh handles remote recovery.
# ── REMOTE HANDLING ───────────────────────────────────────────────────────────────────────────
# Both local and remote handled in one run via SSH.
# Remote containers left as-is — docker_watchdog.sh handles remote container recovery.
# If remote unreachable → skips remote cleanly, logs warning.
#
# Flags:
# (none) ← smart mode — auto-detects, rsync-only if orchestrator running
# --full-stop ← nuclear — kill orchestrator + rsync
# --dry-run ← preview without changes
# -----------------------------------------------------------------------------------------------
# ── ORCHESTRATOR DETECTION ────────────────────────────────────────────────────────────────────
# detect_rsync_parent() scans all lock files to find which running process
# has rsync as a descendant. No hardcoded list — works for any orchestrator.
# Returns: "script_name:parent_pid" if found, empty if rsync running standalone.
#
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
# Root check — pkill and docker require root
# acquire_lock — prevents concurrent stop attempts racing
# DOCKER_TIMEOUT — all docker calls protected against hung daemon
# SSH_TIMEOUT — all remote SSH calls timeout-protected
# SIGTERM → SIGKILL — graceful then forced for orchestrators
# Container recovery — restarts local containers left stopped by killed rsync
# validate_unraid_cmd — notify validated before use
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
# rsync_stop.sh — smart stop (auto-detect)
# rsync_stop.sh --full-stop — kill orchestrator + rsync
# rsync_stop.sh --rsync-only — skip container recovery (called by other scripts)
# rsync_stop.sh --dry-run — preview without changes
# rsync_stop.sh --status — show what's currently running
# rsync_stop.sh --full-stop --dry-run — preview full stop
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../Master.conf"
source "$SCRIPT_DIR/../common.sh"
source "$SCRIPT_DIR/../load_config.sh"
# Check for --full-stop before parse_args
DOCKER_TIMEOUT=15
SSH_TIMEOUT=15
# ── Parse special flags before parse_args ─────────────────────────────────────────────────────
FULL_STOP=false
RSYNC_ONLY_MODE=false
FILTERED_ARGS=()
for arg in "$@"; do
if [[ "$arg" == "--full-stop" ]]; then
FULL_STOP=true
else
FILTERED_ARGS+=("$arg")
fi
case "$arg" in
--full-stop) FULL_STOP=true ;;
--rsync-only) RSYNC_ONLY_MODE=true ;;
*) FILTERED_ARGS+=("$arg") ;;
esac
done
parse_args "${FILTERED_ARGS[@]}"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Setup ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_GEAR Setup ━━━"
# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
error "Must be run as root — pkill and docker require root"
exit 1
fi
validate_unraid_cmd \
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
"" "" \
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
acquire_lock
detect_hosts
resolve_remote_ip
REMOTE_REACHABLE=true
if ! ping -c1 -W3 "$REMOTE_SERVER" &>/dev/null; then
warn "$ICON_PING Remote $REMOTE_SERVER_NAME unreachable — will skip remote"
REMOTE_REACHABLE=false
# Remote reachability
REMOTE_REACHABLE=false
if timeout "$SSH_TIMEOUT" ping -c1 -W3 "$REMOTE_SERVER" &>/dev/null; then
REMOTE_REACHABLE=true
log "$REMOTE_SERVER_NAME reachable ✅"
else
info "$ICON_PING $REMOTE_SERVER_NAME reachable"
warn "$REMOTE_SERVER_NAME unreachable — remote operations will be skipped"
fi
[[ "$SHOW_STATUS" == true ]] && show_status && exit 0
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
[[ "$FULL_STOP" == true ]] && warn "FULL STOP mode — orchestrator + rsync will be killed"
# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
echo ""
echo "━━━━━ $ICON_SUMMARY RSYNC STOP STATUS ━━━━━"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_NET Remote: $REMOTE_ID ($REMOTE_SERVER_NAME)"
echo ""
LOCAL_PIDS=$(pgrep -x rsync 2>/dev/null | tr '\n' ' ')
echo " $ICON_SYNC Local rsync PIDs: ${LOCAL_PIDS:-none}"
for lockfile in "$LOCK_DIR"/*.lock; do
[[ -f "$lockfile" ]] || continue
content=$(cat "$lockfile" 2>/dev/null)
pid="${content%%:*}"
name="${content##*:}"
[[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null && \
echo " $ICON_RUNNING Lock: $name (PID $pid)"
done
if [[ "$REMOTE_REACHABLE" == true ]]; then
REMOTE_PIDS=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
-o ConnectTimeout="$SSH_TIMEOUT" \
root@"$REMOTE_SERVER" "pgrep -x rsync || true" 2>/dev/null | tr '\n' ' ')
echo " $ICON_SYNC Remote rsync PIDs: ${REMOTE_PIDS:-none}"
else
echo " $ICON_WARN Remote: unreachable"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
fi
# ==============================================================================================
# ── ORCHESTRATOR DETECTION ────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Scans lock files to find which running process has rsync as a descendant.
# No hardcoded script names — detects any orchestrator automatically.
# -----------------------------------------------------------------------------------------------
# ━━━ Auto-detect orchestrators ━━━
# Check if daily or weekly is running on local and remote
# This determines default behavior
# -----------------------------------------------------------------------------------------------
# -----------------------------------------------------------------------------------------------
# detect_rsync_parent — scans all lock files, finds which running process has rsync as a child
# No hardcoded list — works for any orchestrator automatically
#
# Returns: "script_name:parent_pid" if found, empty if rsync running standalone
# -----------------------------------------------------------------------------------------------
detect_rsync_parent() {
local found=""
# Get all rsync PIDs running locally
local rsync_pids
rsync_pids=$(pgrep -x rsync 2>/dev/null || true)
[[ -z "$rsync_pids" ]] && echo "" && return
# Scan all lock files in LOCK_DIR
for lockfile in "$LOCK_DIR"/*.lock; do
[[ -f "$lockfile" ]] || continue
local content pid locked_name
content=$(cat "$lockfile" 2>/dev/null)
pid="${content%%:*}"
locked_name="${content##*:}"
# Skip if PID dead or is itself a rsync lock
[[ -z "$pid" ]] && continue
! kill -0 "$pid" 2>/dev/null && continue
[[ "$locked_name" == rsync_* ]] && continue
# Check if any rsync PID is a child of this lock's PID
local children
children=$(cat /proc/"$pid"/task/"$pid"/children 2>/dev/null || \
tr ' ' '\n' < /proc/"$pid"/children 2>/dev/null || true)
# Walk the child tree — rsync may be a grandchild (bash → rsync.sh → rsync)
local all_descendants
all_descendants=$(pgrep -P "$pid" 2>/dev/null || true)
# Check if any rsync PID is in the descendants
while IFS= read -r rsync_pid; do
[[ -z "$rsync_pid" ]] && continue
local ppid
ppid=$(awk '/^PPid:/{print $2}' /proc/"$rsync_pid"/status 2>/dev/null || echo "")
if echo "$all_descendants" | grep -qw "$rsync_pid" 2>/dev/null || \
[[ "$(cat /proc/"$rsync_pid"/status 2>/dev/null | awk '/^PPid:/{print $2}')" == "$pid" ]]; then
found="$locked_name:$pid"
break 2
[[ "$ppid" == "$pid" ]]; then
echo "${locked_name}:${pid}"
return
fi
done <<< "$rsync_pids"
done
echo "$found"
echo ""
}
detect_rsync_parent_remote() {
[[ "$REMOTE_REACHABLE" != true ]] && echo "" && return
# Run the same logic on remote via SSH
local found
found=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" bash << 'REMOTE_SCRIPT'
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
-o ConnectTimeout="$SSH_TIMEOUT" \
root@"$REMOTE_SERVER" bash << 'REMOTE_SCRIPT' 2>/dev/null
LOCK_DIR="/tmp/unraid_locks"
rsync_pids=$(pgrep -x rsync 2>/dev/null || true)
[[ -z "$rsync_pids" ]] && exit 0
for lockfile in "$LOCK_DIR"/*.lock; do
[[ -f "$lockfile" ]] || continue
content=$(cat "$lockfile" 2>/dev/null)
@@ -138,21 +187,18 @@ for lockfile in "$LOCK_DIR"/*.lock; do
[[ -z "$pid" ]] && continue
! kill -0 "$pid" 2>/dev/null && continue
[[ "$locked_name" == rsync_* ]] && continue
all_descendants=$(pgrep -P "$pid" 2>/dev/null || true)
while IFS= read -r rsync_pid; do
[[ -z "$rsync_pid" ]] && continue
ppid=$(awk '/^PPid:/{print $2}' /proc/"$rsync_pid"/status 2>/dev/null)
ppid=$(awk '/^PPid:/{print $2}' /proc/"$rsync_pid"/status 2>/dev/null || echo "")
if echo "$all_descendants" | grep -qw "$rsync_pid" 2>/dev/null || \
[[ "$ppid" == "$pid" ]]; then
echo "$locked_name:$pid"
echo "${locked_name}:${pid}"
exit 0
fi
done <<< "$rsync_pids"
done
REMOTE_SCRIPT
2>/dev/null)
echo "$found"
}
LOCAL_ORCH=$(detect_rsync_parent)
@@ -162,27 +208,26 @@ REMOTE_ORCH=""
# Determine mode
if [[ "$FULL_STOP" == true ]]; then
MODE="full-stop"
info "Mode: FULL STOP — orchestrator + rsync will be killed"
elif [[ -n "$LOCAL_ORCH" ]] || [[ -n "$REMOTE_ORCH" ]]; then
MODE="rsync-only"
[[ -n "$LOCAL_ORCH" ]] && info "Detected local orchestrator: ${LOCAL_ORCH%%:*} — rsync-only mode"
[[ -n "$REMOTE_ORCH" ]] && info "Detected remote orchestrator: ${REMOTE_ORCH%%:*} — rsync-only mode"
info "Orchestrator will continue after rsync is killed"
info "Use --full-stop to also kill the orchestrator"
[[ -n "$LOCAL_ORCH" ]] && \
warn "Local orchestrator detected: ${LOCAL_ORCH%%:*} — rsync-only mode"
[[ -n "$REMOTE_ORCH" ]] && \
warn "Remote orchestrator detected: ${REMOTE_ORCH%%:*} — rsync-only mode"
warn "Use --full-stop to also kill the orchestrator"
else
MODE="rsync-only"
info "No orchestrator detected — killing rsync directly"
log "No orchestrator detected — killing rsync directly"
fi
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_STOP Kill Orchestrators (full-stop only) ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ── Kill Orchestrators (full-stop only) ───────────────────────────────────────────────────────
# ==============================================================================================
ORCHESTRATORS_KILLED=()
REMOTE_ORCHESTRATORS_KILLED=()
kill_orchestrator() {
local script_name="$1"
local pid="$2"
local script_name="$1" pid="$2"
local lockfile="$LOCK_DIR/${script_name}.lock"
if [[ "$DRY_RUN" == true ]]; then
@@ -196,7 +241,7 @@ kill_orchestrator() {
sleep 1
if ! kill -0 "$pid" 2>/dev/null; then
success "$script_name stopped ✅"
warn "$script_name stopped (PID $pid) ✅"
rm -f "$lockfile"
return 0
else
@@ -207,70 +252,62 @@ kill_orchestrator() {
if [[ "$MODE" == "full-stop" ]]; then
echo ""
echo "━━━ $ICON_STOP Orchestrators ━━━"
echo "━━━ $ICON_STOP Kill Orchestrators ━━━"
# Local
if [[ -n "$LOCAL_ORCH" ]]; then
name="${LOCAL_ORCH%%:*}"
pid="${LOCAL_ORCH##*:}"
info "Killing local: $name (PID $pid)"
if kill_orchestrator "$name" "$pid"; then
ORCHESTRATORS_KILLED+=("$name")
fi
local_name="${LOCAL_ORCH%%:*}"
local_pid="${LOCAL_ORCH##*:}"
warn "Killing local: $local_name (PID $local_pid)"
kill_orchestrator "$local_name" "$local_pid" && \
ORCHESTRATORS_KILLED+=("$local_name")
else
info "No local orchestrator running"
log "No local orchestrator running"
fi
# Remote
if [[ "$REMOTE_REACHABLE" == true ]] && [[ -n "$REMOTE_ORCH" ]]; then
name="${REMOTE_ORCH%%:*}"
pid="${REMOTE_ORCH##*:}"
lockfile="$LOCK_DIR/${name}.lock"
info "Killing remote: $name (PID $pid)"
remote_name="${REMOTE_ORCH%%:*}"
remote_pid="${REMOTE_ORCH##*:}"
remote_lock="$LOCK_DIR/${remote_name}.lock"
warn "Killing remote: $remote_name (PID $remote_pid)"
if [[ "$DRY_RUN" == false ]]; then
ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
"kill -TERM '$pid' 2>/dev/null; sleep 2; \
kill -0 '$pid' 2>/dev/null && kill -KILL '$pid' 2>/dev/null; \
rm -f '$lockfile'" 2>/dev/null
success "Remote $name stopped ✅"
REMOTE_ORCHESTRATORS_KILLED+=("$name")
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
-o ConnectTimeout="$SSH_TIMEOUT" \
root@"$REMOTE_SERVER" \
"kill -TERM '$remote_pid' 2>/dev/null; sleep 2; \
kill -0 '$remote_pid' 2>/dev/null && kill -KILL '$remote_pid' 2>/dev/null; \
rm -f '$remote_lock'" 2>/dev/null
warn "Remote $remote_name stopped ✅"
REMOTE_ORCHESTRATORS_KILLED+=("$remote_name")
else
warn "DRY RUN — would kill remote $name (PID $pid)"
warn "DRY RUN — would kill remote $remote_name (PID $remote_pid)"
fi
elif [[ "$REMOTE_REACHABLE" == true ]]; then
info "No remote orchestrator running"
log "No remote orchestrator running"
fi
# Wait for subprocesses to settle
if [[ ${#ORCHESTRATORS_KILLED[@]} -gt 0 ]] || \
[[ ${#REMOTE_ORCHESTRATORS_KILLED[@]} -gt 0 ]]; then
info "Waiting 3s for subprocesses to settle..."
sleep 3
fi
[[ ${#ORCHESTRATORS_KILLED[@]} -gt 0 || \
${#REMOTE_ORCHESTRATORS_KILLED[@]} -gt 0 ]] && sleep 3
fi
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_STOP Local Rsync ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Local Rsync ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_STOP Local Rsync ━━━"
LOCAL_KILLED=false
LOCAL_PIDS=$(pgrep -x rsync || true)
LOCAL_PIDS=$(pgrep -x rsync 2>/dev/null || true)
if [[ -z "$LOCAL_PIDS" ]]; then
info "No rsync processes running locally"
log "No rsync processes running locally"
else
info "Found PIDs: $(echo "$LOCAL_PIDS" | tr '\n' ' ')"
warn "Found local rsync PIDs: $(echo "$LOCAL_PIDS" | tr '\n' ' ')"
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would kill local rsync"
else
if pkill -x rsync; then
success "Local rsync killed ✅"
LOCAL_KILLED=true
else
warn "pkill non-zero — may have already exited"
fi
pkill -x rsync 2>/dev/null && LOCAL_KILLED=true || \
warn "pkill returned non-zero — rsync may have already exited"
[[ "$LOCAL_KILLED" == true ]] && warn "Local rsync killed ✅"
fi
fi
@@ -280,61 +317,65 @@ for lockfile in "$LOCK_DIR"/rsync_*.lock; do
content=$(cat "$lockfile" 2>/dev/null)
pid="${content%%:*}"
if [[ -n "$pid" ]] && ! kill -0 "$pid" 2>/dev/null; then
info "Cleaning stale lock: $(basename "$lockfile")"
log "Cleaning stale lock: $(basename "$lockfile")"
[[ "$DRY_RUN" == false ]] && rm -f "$lockfile"
fi
done
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_STOP Remote Rsync ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Remote Rsync ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_STOP Remote Rsync ($REMOTE_SERVER_NAME) ━━━"
echo "━━━ $ICON_STOP Remote Rsync — $REMOTE_SERVER_NAME ━━━"
REMOTE_KILLED=false
if [[ "$REMOTE_REACHABLE" == false ]]; then
warn "Skipping — $REMOTE_SERVER_NAME unreachable"
else
REMOTE_PIDS=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
"pgrep -x rsync || true" 2>/dev/null || true)
REMOTE_PIDS=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
-o ConnectTimeout="$SSH_TIMEOUT" \
root@"$REMOTE_SERVER" "pgrep -x rsync || true" 2>/dev/null || true)
if [[ -z "$REMOTE_PIDS" ]]; then
info "No rsync running on $REMOTE_SERVER_NAME"
log "No rsync running on $REMOTE_SERVER_NAME"
else
info "Found PIDs on $REMOTE_SERVER_NAME: $(echo "$REMOTE_PIDS" | tr '\n' ' ')"
warn "Found remote rsync PIDs: $(echo "$REMOTE_PIDS" | tr '\n' ' ')"
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would kill remote rsync"
else
if ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" "pkill -x rsync || true" 2>/dev/null; then
success "Remote rsync killed ✅"
REMOTE_KILLED=true
else
warn "Remote pkill non-zero — may have already exited"
fi
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
-o ConnectTimeout="$SSH_TIMEOUT" \
root@"$REMOTE_SERVER" "pkill -x rsync || true" 2>/dev/null && \
REMOTE_KILLED=true || \
warn "Remote pkill returned non-zero — rsync may have already exited"
[[ "$REMOTE_KILLED" == true ]] && warn "Remote rsync killed ✅"
fi
fi
fi
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_START $ICON_CONTAINERS Container Recovery ━━━
# Restart containers left stopped by interrupted rsync
# Only runs if something was actually killed locally
# Remote containers left as-is — docker_watchdog.sh handles remote
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Container Recovery ━━━
# ==============================================================================================
# Restart local containers left stopped by interrupted rsync.
# Remote containers left for docker_watchdog.sh to recover.
# Skipped with --rsync-only flag (called by other scripts that handle recovery themselves).
CONTAINERS_RESTARTED=()
CONTAINERS_FAILED=()
if [[ "$RSYNC_ONLY_MODE" == false ]] && \
{ [[ "$LOCAL_KILLED" == true ]] || [[ ${#ORCHESTRATORS_KILLED[@]} -gt 0 ]]; }; then
if [[ "$LOCAL_KILLED" == true ]] || [[ ${#ORCHESTRATORS_KILLED[@]} -gt 0 ]]; then
echo ""
echo "━━━ $ICON_START $ICON_CONTAINERS Container Recovery ━━━"
info "Checking all profile containers..."
echo "━━━ $ICON_START Container Recovery ━━━"
log "Checking profile containers for recovery..."
declare -A SEEN
ALL_CONTAINERS=()
for profile_containers in "${PROFILE_CRITICAL_CONTAINER_NAMES[@]}"; do
for profile_containers in "${PROFILE_CRITICAL_CONTAINER_NAMES[@]:-}"; do
read -r -a container_list <<< "$profile_containers"
for c in "${container_list[@]}"; do
for c in "${container_list[@]:-}"; do
[[ -z "$c" ]] && continue
if [[ -z "${SEEN[$c]:-}" ]]; then
SEEN[$c]=1
@@ -344,69 +385,89 @@ if [[ "$LOCAL_KILLED" == true ]] || [[ ${#ORCHESTRATORS_KILLED[@]} -gt 0 ]]; the
done
if [[ ${#ALL_CONTAINERS[@]} -eq 0 ]]; then
info "No containers defined — skipping recovery"
log "No profile containers defined — skipping recovery"
else
for c in "${ALL_CONTAINERS[@]}"; do
STATUS=$(docker inspect -f '{{.State.Running}}' "$c" 2>/dev/null || echo "unknown")
if [[ "$STATUS" == "true" ]]; then
info "$ICON_RUNNING $c — running ✅"
elif [[ "$STATUS" == "false" ]]; then
warn "$ICON_NOT_RUNNING $c — stopped, restarting..."
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would restart $c"
else
if docker start "$c" >/dev/null 2>&1; then
success "$c restarted ✅"
CONTAINERS_RESTARTED+=("$c")
STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
'{{.State.Running}}' "$c" 2>/dev/null || echo "unknown")
case "$STATUS" in
true)
log "$c — running ✅"
;;
false)
warn "$c — stopped — restarting..."
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would restart $c"
else
error "Failed to restart $c"
if timeout "$DOCKER_TIMEOUT" docker start "$c" >/dev/null 2>&1; then
warn "$c restarted ✅"
CONTAINERS_RESTARTED+=("$c")
else
error "Failed to restart $c"
CONTAINERS_FAILED+=("$c")
fi
fi
fi
else
info "$c not found on this host — skipping"
fi
;;
*)
log "$c not found locally — skipping"
;;
esac
done
fi
fi
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Summary ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
echo ""
echo "━━━━━ $ICON_SUMMARY RSYNC STOP SUMMARY ━━━━━"
echo " Mode: $MODE"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_GEAR Mode: $MODE"
echo ""
echo "$ICON_HOST Local ($LOCAL_SERVER_NAME):"
echo "$ICON_HOST Local ($MY_ID):"
[[ ${#ORCHESTRATORS_KILLED[@]} -gt 0 ]] && \
echo " $ICON_STOPPED Orchestrators killed: ${ORCHESTRATORS_KILLED[*]}"
[[ "$LOCAL_KILLED" == true ]] && \
echo " $ICON_STOPPED Rsync killed" || \
echo " $ICON_SUCCESS No rsync was running"
warn " Orchestrators killed: ${ORCHESTRATORS_KILLED[*]}"
if [[ "$LOCAL_KILLED" == true ]]; then
warn " Rsync killed ✅"
else
log " No rsync was running"
fi
echo "$ICON_NET Remote ($REMOTE_SERVER_NAME):"
echo "$ICON_NET Remote ($REMOTE_ID — $REMOTE_SERVER_NAME):"
if [[ "$REMOTE_REACHABLE" == false ]]; then
echo " $ICON_WARN Unreachable — skipped"
warn " Unreachable — skipped"
else
[[ ${#REMOTE_ORCHESTRATORS_KILLED[@]} -gt 0 ]] && \
echo " $ICON_STOPPED Orchestrators killed: ${REMOTE_ORCHESTRATORS_KILLED[*]}"
[[ "$REMOTE_KILLED" == true ]] && \
echo " $ICON_STOPPED Rsync killed" || \
echo " $ICON_SUCCESS No rsync was running"
warn " Orchestrators killed: ${REMOTE_ORCHESTRATORS_KILLED[*]}"
if [[ "$REMOTE_KILLED" == true ]]; then
warn " Rsync killed ✅"
else
log " No rsync was running"
fi
fi
[[ ${#CONTAINERS_RESTARTED[@]} -gt 0 ]] && \
echo "$ICON_CONTAINERS Containers recovered: ${CONTAINERS_RESTARTED[*]}"
warn "$ICON_CONTAINERS Containers recovered: ${CONTAINERS_RESTARTED[*]}"
[[ ${#CONTAINERS_FAILED[@]} -gt 0 ]] && \
echo "$ICON_ERROR Containers failed to restart: ${CONTAINERS_FAILED[*]}"
echo ""
if [[ "$DRY_RUN" == true ]]; then
echo "$ICON_WARN Status: DRY RUN — no changes made"
warn "DRY RUN — no changes made"
else
echo "$ICON_DONE Status: $ICON_SUCCESS DONE"
log "$ICON_DONE Status: done ✅"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
if [[ "$LOCAL_KILLED" == true ]] || [[ "$REMOTE_KILLED" == true ]] || \
[[ ${#ORCHESTRATORS_KILLED[@]} -gt 0 ]]; then
notify "Rsync stopped on $(hostname) — mode: $MODE — ${#CONTAINERS_RESTARTED[@]} containers recovered" \
"Rsync Stop" "warning"
# Notify if anything was actually killed or failed
if [[ "$DRY_RUN" == false ]]; then
if [[ ${#CONTAINERS_FAILED[@]} -gt 0 ]]; then
notify "Rsync stop on $(hostname) ($MY_ID) — containers failed to restart: ${CONTAINERS_FAILED[*]}" \
"Rsync Stop" "warning"
elif [[ "$LOCAL_KILLED" == true || "$REMOTE_KILLED" == true || \
${#ORCHESTRATORS_KILLED[@]} -gt 0 ]]; then
notify "Rsync stopped on $(hostname) ($MY_ID) — mode: $MODE${CONTAINERS_RESTARTED:+ — recovered: ${CONTAINERS_RESTARTED[*]}}" \
"Rsync Stop" "warning"
fi
fi
unRAID_Essentials/server_reboot.sh
+253 -124
View File
@@ -1,159 +1,288 @@
#!/bin/bash
# -----------------------------------------------------------------------------------------------
# --------------------------------- Server Reboot Script ---------------------------------------
# -----------------------------------------------------------------------------------------------
# Gracefully reboots the unRAID server with a configurable user warning delay.
# Stops Docker and VM Manager cleanly before issuing reboot.
# Reboot delay is configured in Master.conf under REBOOT_SLEEP.
# Supports --dry-run to walk through the sequence without actually rebooting.
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ================================= Server Reboot ==============================================
# ==============================================================================================
# Gracefully reboots the unRAID server with full pre-flight checks and clean shutdown sequence.
# Warns all users, checks for active processes, stops services, syncs disks, then reboots.
#
# ── SHUTDOWN SEQUENCE ─────────────────────────────────────────────────────────────────────────
# 1. Pre-flight warnings — rsync, mover, active Emby sessions (warn not block)
# 2. Wall message to all logged-in terminal users
# 3. unRAID notification to dashboard
# 4. Wait REBOOT_SLEEP seconds (default 30) — gives users time to save work
# 5. Gracefully shutdown VMs (virsh shutdown each, then wait)
# 6. Stop libvirt (VM Manager)
# 7. Stop Docker service
# 8. Sync filesystem buffers to disk
# 9. Reboot
#
# ── PRE-FLIGHT WARNINGS ───────────────────────────────────────────────────────────────────────
# The following are warnings only — they do not block the reboot. You called this script,
# so you know what you're doing. The warnings give you context before the countdown starts.
# - rsync running → partial files possible if mid-transfer
# - mover running → files may be left on cache or array mid-move
# - Emby sessions → active streams/transcodes will be interrupted
#
# ── VM GRACEFUL SHUTDOWN ──────────────────────────────────────────────────────────────────────
# virsh shutdown sends ACPI power button signal to each VM — same as pressing power button.
# VM gets a chance to flush its own buffers and shutdown cleanly.
# Waits REBOOT_VM_WAIT seconds (default 30) for VMs to shut down before stopping libvirt.
# If VMs don't shut down in time libvirt stops anyway — system reboot takes priority.
#
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
# detect_hosts() sets MY_ID — used in wall message, notification, and summary.
# Critical on a two-server setup — wall and notifications show WHICH server is rebooting.
#
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
# Root check — reboot requires root
# acquire_lock — prevents concurrent reboot calls
# detect_hosts() — MY_ID in all user-facing messages
# validate_unraid_cmd — notify validated before use
# Graceful VM shutdown — VMs get clean ACPI signal before libvirt stops
# sync before reboot — filesystem buffers flushed to disk
#
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
# REBOOT_SLEEP — seconds to warn users before starting shutdown sequence (default 30)
# REBOOT_VM_WAIT — seconds to wait for VMs to shut down gracefully (default 30)
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
# server_reboot.sh — reboot with 30s warning
# server_reboot.sh --dry-run — walk through sequence without rebooting
# server_reboot.sh --status — show running processes that would be affected
# server_reboot.sh --reason="maintenance" — log reason for reboot
# server_reboot.sh --log — verbose output
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../Master.conf"
source "$SCRIPT_DIR/../common.sh"
source "$SCRIPT_DIR/../load_config.sh"
parse_args "$@"
# ── Parse --reason flag before parse_args ─────────────────────────────────────────────────────
REBOOT_REASON="manual"
FILTERED_ARGS=()
for arg in "$@"; do
case "$arg" in
--reason=*) REBOOT_REASON="${arg#--reason=}" ;;
*) FILTERED_ARGS+=("$arg") ;;
esac
done
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Setup ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_GEAR Setup ━━━"
parse_args "${FILTERED_ARGS[@]}"
# ROOT CHECK
# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
error "Must be run as root"
error "Must be run as root — reboot requires root"
exit 1
fi
success "Running as root"
validate_unraid_cmd \
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
"" "" \
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
# VALIDATION
validate_int REBOOT_SLEEP "$REBOOT_SLEEP"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Status ━━━
# -----------------------------------------------------------------------------------------------
acquire_lock
detect_hosts
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made, no reboot will occur"
# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
echo ""
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
echo "$ICON_REBOOT Delay: ${REBOOT_SLEEP}s"
echo "$ICON_GEAR Dry Run: $DRY_RUN"
echo "━━━━━ $ICON_SUMMARY REBOOT STATUS ━━━━━"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_REBOOT Delay: ${REBOOT_SLEEP}s"
echo "$ICON_GEAR VM wait: ${REBOOT_VM_WAIT:-30}s"
echo "$ICON_GEAR Reason: $REBOOT_REASON"
echo ""
echo "━━━ Active Processes ━━━"
pgrep -x rsync >/dev/null 2>&1 && \
warn " rsync: RUNNING — partial files if rebooted now" || \
log " rsync: not running"
pgrep -f "emhttp.*Mover" >/dev/null 2>&1 && \
warn " mover: RUNNING — files may be left mid-move" || \
log " mover: not running"
if command -v virsh >/dev/null 2>&1; then
VM_COUNT=$(virsh list --name 2>/dev/null | grep -c "." || echo 0)
[[ "$VM_COUNT" -gt 0 ]] && \
warn " VMs: $VM_COUNT running — will be gracefully shut down" || \
log " VMs: none running"
fi
if command -v docker >/dev/null 2>&1; then
CONTAINER_COUNT=$(docker ps -q 2>/dev/null | wc -l || echo 0)
log " Docker: $CONTAINER_COUNT container(s) running"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
fi
# -----------------------------------------------------------------------------------------------
# FUNCTIONS
# -----------------------------------------------------------------------------------------------
# Broadcasts a wall message warning all logged in users of the upcoming reboot.
notify_users() {
warn "Notifying users — reboot in ${REBOOT_SLEEP}s"
wall "$ICON_WARN unRAID server will reboot in ${REBOOT_SLEEP} second(s). Save your work."
}
# Stops the Docker service cleanly.
# Warns but continues if Docker is already stopped or fails — shutdown must proceed.
stop_docker() {
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would stop Docker service"
return
fi
info "Stopping Docker service..."
if /etc/rc.d/rc.docker stop; then
success "Docker stopped"
else
warn "Docker stop failed or already stopped — continuing"
fi
}
# Stops the VM Manager (libvirt) cleanly.
# Warns but continues if libvirt is already stopped or fails — shutdown must proceed.
stop_vm_manager() {
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would stop VM Manager (libvirt)"
return
fi
info "Stopping VM Manager..."
if /etc/rc.d/rc.libvirt stop; then
success "VM Manager stopped"
else
warn "VM Manager stop failed or already stopped — continuing"
fi
}
# Flushes filesystem buffers to disk before reboot.
sync_disks() {
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would sync filesystem buffers"
return
fi
info "Syncing disks..."
if sync; then
success "Disk sync complete"
else
warn "Sync returned an error — continuing"
fi
}
# Issues the system reboot command.
# System will not return from this call unless dry-run is active.
reboot_system() {
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would reboot system now"
return
fi
echo ""
echo "$ICON_REBOOT Rebooting system NOW..."
/sbin/reboot
}
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_REBOOT Reboot Sequence ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_REBOOT Reboot Sequence ━━━"
echo "$ICON_REBOOT Delay: ${REBOOT_SLEEP}s"
echo "$ICON_GEAR Dry Run: $DRY_RUN"
# ==============================================================================================
# ━━━ Pre-flight Warnings ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_SHIELD Pre-flight ━━━"
WARNINGS=()
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
# rsync check — partial files if killed mid-transfer
if pgrep -x rsync >/dev/null 2>&1; then
RSYNC_PIDS=$(pgrep -x rsync | tr '\n' ' ')
warn "rsync is running (PIDs: $RSYNC_PIDS) — partial files possible"
warn "Consider: rsync_stop.sh before rebooting"
WARNINGS+=("rsync running")
fi
# mover check — files may be left mid-move
if pgrep -f "emhttp.*Mover" >/dev/null 2>&1; then
warn "Mover is running — files may be left mid-move on cache or array"
warn "Consider: mover_stop.sh before rebooting"
WARNINGS+=("mover running")
fi
# Emby sessions check — active streams interrupted
if [[ -n "${EMBY_URL:-}" ]] && [[ -n "${EMBY_API_KEY:-}" ]]; then
ACTIVE_STREAMS=$(curl -sf --max-time 5 \
-H "X-Emby-Token: $EMBY_API_KEY" \
"${EMBY_URL}/Sessions" 2>/dev/null | \
grep -c "NowPlayingItem" 2>/dev/null || echo 0)
ACTIVE_STREAMS="${ACTIVE_STREAMS//[^0-9]/}"
if [[ "${ACTIVE_STREAMS:-0}" -gt 0 ]]; then
warn "$ACTIVE_STREAMS active Emby stream(s) — will be interrupted"
WARNINGS+=("${ACTIVE_STREAMS} Emby sessions")
fi
fi
if [[ ${#WARNINGS[@]} -eq 0 ]]; then
log "Pre-flight clean — no active processes to warn about"
else
warn "Proceeding with reboot despite warnings — ${WARNINGS[*]}"
fi
# ==============================================================================================
# ━━━ Notify and Wait ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_REBOOT Reboot Sequence — $MY_ID ━━━"
echo " Reason: $REBOOT_REASON"
echo " Delay: ${REBOOT_SLEEP}s"
echo " Dry Run: $DRY_RUN"
echo ""
START=$(date +%s)
if [[ "$REBOOT_SLEEP" -gt 0 ]]; then
notify_users
info "Waiting ${REBOOT_SLEEP}s before shutdown sequence..."
sleep "$REBOOT_SLEEP"
# Wall message — terminal users
wall "$ICON_WARN $MY_ID ($LOCAL_SERVER_NAME) rebooting in ${REBOOT_SLEEP}s — reason: $REBOOT_REASON. Save your work now."
# unRAID notification — dashboard
if [[ "$DRY_RUN" == false ]]; then
notify "$MY_ID ($LOCAL_SERVER_NAME) rebooting in ${REBOOT_SLEEP}s — reason: $REBOOT_REASON${WARNINGS:+ — warnings: ${WARNINGS[*]}}" \
"Server Reboot" "warning"
fi
warn "Waiting ${REBOOT_SLEEP}s before shutdown sequence..."
if [[ "$DRY_RUN" == false ]]; then
sleep "$REBOOT_SLEEP"
else
warn "DRY RUN — skipping sleep"
fi
fi
stop_docker
stop_vm_manager
sync_disks
reboot_system
# ==============================================================================================
# ━━━ Graceful VM Shutdown ━━━
# ==============================================================================================
if command -v virsh >/dev/null 2>&1; then
VM_LIST=$(virsh list --name 2>/dev/null | grep -v "^$" || true)
if [[ -n "$VM_LIST" ]]; then
echo ""
echo "━━━ $ICON_GEAR Graceful VM Shutdown ━━━"
while IFS= read -r vm; do
[[ -z "$vm" ]] && continue
warn "Sending ACPI shutdown to VM: $vm"
if [[ "$DRY_RUN" == false ]]; then
virsh shutdown "$vm" >/dev/null 2>&1 || true
else
warn "DRY RUN — would virsh shutdown $vm"
fi
done <<< "$VM_LIST"
# NOTE: system will not reach here unless --dry-run is active
if [[ "$DRY_RUN" == false ]]; then
VM_WAIT="${REBOOT_VM_WAIT:-30}"
log "Waiting ${VM_WAIT}s for VMs to shut down..."
sleep "$VM_WAIT"
fi
fi
fi
# ==============================================================================================
# ━━━ Stop VM Manager ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_GEAR Stop VM Manager ━━━"
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would stop VM Manager (libvirt)"
else
if /etc/rc.d/rc.libvirt stop >/dev/null 2>&1; then
warn "VM Manager stopped ✅"
else
warn "VM Manager stop returned non-zero — may already be stopped"
fi
fi
# ==============================================================================================
# ━━━ Stop Docker ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_CONTAINERS Stop Docker ━━━"
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would stop Docker service"
else
if /etc/rc.d/rc.docker stop >/dev/null 2>&1; then
warn "Docker stopped ✅"
else
warn "Docker stop returned non-zero — may already be stopped"
fi
fi
# ==============================================================================================
# ━━━ Sync Disks ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_DISK Sync Disks ━━━"
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would sync filesystem buffers"
else
sync
log "Filesystem buffers flushed ✅"
fi
# ==============================================================================================
# ━━━ Reboot ━━━
# ==============================================================================================
END=$(date +%s)
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Summary ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━━━ $ICON_SUMMARY REBOOT SUMMARY ━━━━━"
echo "$ICON_REBOOT Delay: ${REBOOT_SLEEP}s"
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_REBOOT Reason: $REBOOT_REASON"
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
[[ ${#WARNINGS[@]} -gt 0 ]] && warn "Warnings: ${WARNINGS[*]}"
echo ""
if [[ "$DRY_RUN" == true ]]; then
echo "$ICON_WARN Status: DRY RUN — no reboot executed"
warn "DRY RUN — sequence complete, no reboot executed"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
else
echo "$ICON_REBOOT Status: $ICON_WARN SYSTEM SHOULD BE REBOOTING"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
warn "$ICON_REBOOT Rebooting $MY_ID now..."
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
/sbin/reboot
fi
unRAID_Essentials/system_watchdog.sh
+611 -167
View File
File diff suppressed because it is too large Load Diff
unRAID_Essentials/user_scripts_stop.sh
+197 -92
View File
@@ -1,126 +1,231 @@
#!/bin/bash
# -----------------------------------------------------------------------------------------------
# --------------------------------- User Script Stop -------------------------------------------
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ============================= User Scripts Stop ==============================================
# ==============================================================================================
# Stops all running User Script processes spawned by the unRAID User Scripts plugin.
# Identifies processes by their /tmp/user.scripts path signature.
# Supports --dry-run to preview what would be killed without making changes.
# -----------------------------------------------------------------------------------------------
# Shows script names not just PIDs — you know what's being stopped.
#
# ── WHEN TO USE ───────────────────────────────────────────────────────────────────────────────
# - Before a planned reboot when scripts are running mid-cycle
# - When a script is stuck and won't respond to the Abort button in the UI
# - Called automatically by server_reboot.sh as part of shutdown sequence
# - Emergency stop of all background ecosystem scripts
#
# ── HOW IT IDENTIFIES PROCESSES ───────────────────────────────────────────────────────────────
# Scans /proc/*/cmdline for processes whose command line contains "/tmp/user.scripts".
# The unRAID User Scripts plugin stages all scripts in /tmp/user.scripts/ before execution.
# This is more reliable than process name matching which can vary.
#
# ── STOP SEQUENCE PER PROCESS ─────────────────────────────────────────────────────────────────
# 1. Send SIGTERM — allows script to trap and clean up gracefully
# 2. Wait 5 seconds
# 3. Check if still running → SIGKILL (force) if SIGTERM ignored
# 4. Verify dead after SIGKILL
#
# ── SELF-EXCLUSION ────────────────────────────────────────────────────────────────────────────
# If this script itself is run via the User Scripts plugin it would find its own PID.
# Self-exclusion prevents this script from killing itself mid-execution.
#
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
# Root check — kill requires root for other users' processes
# acquire_lock — prevents concurrent stop attempts
# Self-exclusion — never kills its own process tree
# SIGTERM → SIGKILL — graceful then forced
# Verify after kill — confirms processes are actually dead
# validate_unraid_cmd — notify validated before use
# Silent when clean — no processes running = log() only ✅
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
# user_scripts_stop.sh — stop all user scripts
# user_scripts_stop.sh --dry-run — show what would be stopped
# user_scripts_stop.sh --status — show currently running user scripts
# user_scripts_stop.sh --log — verbose output
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../Master.conf"
source "$SCRIPT_DIR/../common.sh"
source "$SCRIPT_DIR/../load_config.sh"
parse_args "$@"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Setup ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_GEAR Setup ━━━"
MY_PID=$$
MY_PPID=$PPID
# ROOT CHECK
# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
error "Must be run as root"
error "Must be run as root — kill requires root for other users' processes"
exit 1
fi
success "Running as root"
validate_unraid_cmd \
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
"" "" \
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Status ━━━
# -----------------------------------------------------------------------------------------------
acquire_lock
detect_hosts
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no processes will be killed"
# ==============================================================================================
# ── HELPER FUNCTIONS ──────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Get script name from PID — extracts meaningful name from /tmp/user.scripts path
get_script_name() {
local pid="$1"
local cmdline
cmdline=$(tr '\0' ' ' < /proc/"$pid"/cmdline 2>/dev/null || echo "")
# Extract the script filename from the /tmp/user.scripts/... path
echo "$cmdline" | grep -o '/tmp/user\.scripts[^ ]*' | \
awk -F/ '{print $NF}' | head -1 || echo "pid-$pid"
}
# Get all user script PIDs — excludes self and own parent process tree
get_user_script_pids() {
local -a pids=()
while IFS= read -r pid; do
[[ -z "$pid" ]] && continue
# Self-exclusion — don't kill our own process or parent
[[ "$pid" == "$MY_PID" ]] && continue
[[ "$pid" == "$MY_PPID" ]] && continue
pids+=("$pid")
done < <(
for dir in /proc/[0-9]*/cmdline; do
pid="${dir%/cmdline}"
pid="${pid#/proc/}"
if grep -ql '/tmp/user\.scripts' "$dir" 2>/dev/null; then
echo "$pid"
fi
done
)
printf '%s\n' "${pids[@]}"
}
# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
echo ""
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
echo "$ICON_PLUGIN Target: /tmp/user.scripts processes"
echo "$ICON_GEAR Dry Run: $DRY_RUN"
echo "━━━━━ $ICON_SUMMARY USER SCRIPTS STATUS ━━━━━"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo ""
mapfile -t PIDS < <(get_user_script_pids)
if [[ ${#PIDS[@]} -eq 0 ]]; then
log "No User Script processes running"
else
echo " ${#PIDS[@]} User Script process(es) running:"
for pid in "${PIDS[@]}"; do
name=$(get_script_name "$pid")
elapsed=$(ps -o etimes= -p "$pid" 2>/dev/null | tr -d ' ')
runtime=$(format_duration "${elapsed:-0}")
echo " $ICON_RUNNING PID $pid — $name (${runtime})"
done
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
fi
# -----------------------------------------------------------------------------------------------
# FUNCTIONS
# -----------------------------------------------------------------------------------------------
# Returns PIDs of all processes running under /tmp/user.scripts
# These are processes spawned by the unRAID User Scripts plugin.
get_user_script_pids() {
/usr/bin/ps -eo pid,cmd | grep "/tmp/user.scripts" | grep -v grep | awk '{print $1}'
}
# Kills all running User Script processes one by one.
# Reports each PID killed or skipped in dry run mode.
stop_user_scripts() {
local pids
pids=$(get_user_script_pids)
if [[ -z "$pids" ]]; then
info "$ICON_PLUGIN No running User Script processes found — nothing to do"
return
fi
local count=0
for pid in $pids; do
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would kill User Script PID $pid"
else
info "Killing User Script PID $pid..."
if kill "$pid" 2>/dev/null; then
success "Killed PID $pid"
else
warn "Could not kill PID $pid — may have already exited"
fi
fi
count=$((count + 1))
done
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would have targeted $count process(es)"
else
info "$count process(es) targeted"
fi
}
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_PLUGIN User Script Stop ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ User Scripts Stop ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_PLUGIN User Script Stop ━━━"
echo "$ICON_PLUGIN Target: User Scripts Plugin processes"
echo "$ICON_GEAR Dry Run: $DRY_RUN"
echo ""
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
echo "━━━ $ICON_PLUGIN User Scripts Stop — $MY_ID ━━━"
START=$(date +%s)
stop_user_scripts
mapfile -t PIDS < <(get_user_script_pids)
KILLED=()
FAILED=()
SKIPPED=()
if [[ ${#PIDS[@]} -eq 0 ]]; then
log "No User Script processes running — nothing to do"
else
warn "${#PIDS[@]} User Script process(es) found"
echo ""
for pid in "${PIDS[@]}"; do
name=$(get_script_name "$pid")
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would stop: $name (PID $pid)"
SKIPPED+=("$name")
continue
fi
# Verify still running before trying to kill
if ! kill -0 "$pid" 2>/dev/null; then
log "$name (PID $pid) — already exited"
continue
fi
# SIGTERM — graceful stop
log "Sending SIGTERM to $name (PID $pid)..."
kill -TERM "$pid" 2>/dev/null || true
sleep 5
# Check if stopped after SIGTERM
if ! kill -0 "$pid" 2>/dev/null; then
warn "Stopped: $name (PID $pid) ✅"
KILLED+=("$name")
continue
fi
# SIGKILL — forced stop
warn "$name still running after SIGTERM — sending SIGKILL"
kill -KILL "$pid" 2>/dev/null || true
sleep 2
# Final verify
if ! kill -0 "$pid" 2>/dev/null; then
warn "Force-stopped: $name (PID $pid) ✅"
KILLED+=("$name")
else
error "Failed to kill: $name (PID $pid)"
FAILED+=("$name")
fi
done
fi
END=$(date +%s)
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Summary ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
echo ""
echo "━━━━━ $ICON_SUMMARY USER SCRIPTS STOP SUMMARY ━━━━━"
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
echo ""
if [[ "$DRY_RUN" == true ]]; then
echo "$ICON_WARN Status: DRY RUN — no processes killed"
if [[ ${#PIDS[@]} -eq 0 ]]; then
log "No processes were running"
elif [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would have stopped ${#SKIPPED[@]} process(es): ${SKIPPED[*]}"
else
REMAINING=$(get_user_script_pids)
if [[ -z "$REMAINING" ]]; then
echo "$ICON_DONE Status: $ICON_SUCCESS ALL PROCESSES STOPPED"
notify "User Scripts stopped on $(hostname)" "User Script Stop" "warning"
else
echo "$ICON_WARN Status: $ICON_WARN SOME PROCESSES MAY STILL BE RUNNING"
notify "User Script stop completed but some processes may still be running on $(hostname)" "User Script Stop" "warning"
fi
[[ ${#KILLED[@]} -gt 0 ]] && warn "Stopped (${#KILLED[@]}): ${KILLED[*]}"
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed (${#FAILED[@]}): ${FAILED[*]}"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo ""
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — no changes made"
elif [[ ${#FAILED[@]} -gt 0 ]]; then
echo "$ICON_ERROR Status: SOME PROCESSES COULD NOT BE KILLED"
notify "User Scripts stop failed on $(hostname) ($MY_ID) — unkillable: ${FAILED[*]}" \
"User Scripts Stop" "warning"
else
log "$ICON_DONE Status: done ✅"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
[[ ${#FAILED[@]} -gt 0 ]] && exit 1
exit 0
unRAID_Essentials/webgui_restart.sh
+207 -138
View File
@@ -1,198 +1,267 @@
#!/bin/bash
# -----------------------------------------------------------------------------------------------
# --------------------------------- WebGUI Watchdog --------------------------------------------
# -----------------------------------------------------------------------------------------------
# Monitors unRAID's WebGUI and restarts it if unresponsive.
# Uses an escalating restart strategy — tries nginx first, then emhttp if needed.
# emhttp is the unRAID management daemon — restarting it is more disruptive than nginx
# but recovers cleanly. Notification sent on any restart so you know what happened.
# ==============================================================================================
# ================================= WebGUI Watchdog ============================================
# ==============================================================================================
# Monitors the unRAID WebGUI and restarts services if unresponsive.
# Uses a three-step escalating strategy — lightest fix first, heaviest last.
# Run every 5-10 minutes via User Scripts plugin.
# Silent when healthy — only produces output when something needs fixing.
#
# Escalation path:
# Check WebGUI → unresponsive → restart nginx → recheck
# Still unresponsive → restart emhttp → recheck
# Still unresponsive → notify warning, manual intervention needed
# ── ESCALATION PATH ───────────────────────────────────────────────────────────────────────────
# Check WebGUI → responding → log() + exit 0 (completely silent ✅)
#
# Run every 5-10 minutes via cron/User Scripts plugin.
# All configuration in Master.conf under WebGUI Watchdog section.
# Supports --dry-run to show what would be restarted without acting.
# -----------------------------------------------------------------------------------------------
# Not responding:
# Step 1 — Restart nginx
# Lightest fix — handles most transient WebGUI failures
# nginx crash, worker stuck, connection timeout
# Wait WEBGUI_NGINX_WAIT seconds → recheck
#
# Step 2 — Restart php-fpm
# WebGUI runs through PHP-FPM — worker exhaustion causes silent failure
# php-fpm workers saturated → new requests queue → WebGUI appears frozen
# system_tuning_monitor.sh tracks usage — this recovers it
# Wait WEBGUI_PHP_WAIT seconds → recheck
#
# Step 3 — Restart emhttp
# Heaviest fix — emhttp is the unRAID management daemon
# Array, Docker, shares stay running — only WebGUI management restarts
# Takes longer to recover — WEBGUI_EMHTTP_WAIT gives it time
# Wait WEBGUI_EMHTTP_WAIT seconds → recheck
#
# All three failed → notify warning, manual intervention needed → exit 1
#
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
# detect_hosts() sets MY_ID — used in all notifications and summary.
# Critical on two-server setup — which server's WebGUI failed?
#
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
# acquire_lock — prevents concurrent runs double-restarting services
# detect_hosts() — MY_ID in all notifications
# Process verify — pgrep check after each service restart
# Silent healthy — completely silent on healthy cycle ✅
# validate_unraid_cmd — notify validated before use
#
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
# WEBGUI_URL — URL to check (default http://localhost)
# WEBGUI_TIMEOUT — curl timeout in seconds (default 5)
# WEBGUI_NGINX_WAIT — seconds after nginx restart before rechecking (default 15)
# WEBGUI_PHP_WAIT — seconds after php-fpm restart before rechecking (default 10)
# WEBGUI_EMHTTP_WAIT — seconds after emhttp restart before rechecking (default 30)
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
# webgui_restart.sh — check and recover if needed
# webgui_restart.sh --dry-run — show what would be restarted
# webgui_restart.sh --status — show current WebGUI and service states
# webgui_restart.sh --log — verbose output
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../Master.conf"
source "$SCRIPT_DIR/../common.sh"
source "$SCRIPT_DIR/../load_config.sh"
parse_args "$@"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Setup ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_GEAR Setup ━━━"
# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
error "Must be run as root"
exit 1
fi
success "Running as root"
validate_unraid_cmd \
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
"" "" \
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Status ━━━
# -----------------------------------------------------------------------------------------------
acquire_lock
detect_hosts
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no services will be restarted"
# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
echo ""
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
echo "$ICON_WEBGUI URL: $WEBGUI_URL"
echo "$ICON_WEBGUI Curl timeout: ${WEBGUI_TIMEOUT}s"
echo "$ICON_WEBGUI Nginx wait: ${WEBGUI_NGINX_WAIT}s"
echo "$ICON_WEBGUI emhttp wait: ${WEBGUI_EMHTTP_WAIT}s"
echo "$ICON_GEAR Dry Run: $DRY_RUN"
echo "━━━━━ $ICON_SUMMARY WEBGUI WATCHDOG STATUS ━━━━━"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_WEBGUI URL: $WEBGUI_URL"
echo "$ICON_WEBGUI Timeouts: curl=${WEBGUI_TIMEOUT}s nginx=${WEBGUI_NGINX_WAIT}s php=${WEBGUI_PHP_WAIT:-10}s emhttp=${WEBGUI_EMHTTP_WAIT}s"
echo ""
# Show current state
if curl -sf --max-time "$WEBGUI_TIMEOUT" "$WEBGUI_URL" >/dev/null 2>&1; then
echo "$ICON_WEBGUI WebGUI: $ICON_RUNNING responding"
echo " $ICON_SUCCESS WebGUI: responding ✅"
else
echo "$ICON_WEBGUI WebGUI: $ICON_NOT_RUNNING not responding"
echo " $ICON_ERROR WebGUI: NOT responding"
fi
pgrep -x nginx >/dev/null 2>&1 && \
echo " $ICON_SUCCESS nginx: running ✅" || \
echo " $ICON_ERROR nginx: NOT running"
pgrep -f "php-fpm" >/dev/null 2>&1 && \
FPM_COUNT=$(pgrep -fc "php-fpm" 2>/dev/null || echo "?") && \
echo " $ICON_SUCCESS php-fpm: running ($FPM_COUNT workers) ✅" || \
echo " $ICON_ERROR php-fpm: NOT running"
pgrep -x emhttp >/dev/null 2>&1 && \
echo " $ICON_SUCCESS emhttp: running ✅" || \
echo " $ICON_ERROR emhttp: NOT running"
echo "━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
fi
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no services will be restarted"
# -----------------------------------------------------------------------------------------------
# FUNCTIONS
# -----------------------------------------------------------------------------------------------
# Check if WebGUI is responding
# ==============================================================================================
# ── CHECK AND ESCALATE ────────────────────────────────────────────────────────────────────────
# ==============================================================================================
check_webgui() {
curl -sf --max-time "$WEBGUI_TIMEOUT" "$WEBGUI_URL" >/dev/null 2>&1
}
# Restart nginx — lightweight fix, try first
restart_nginx() {
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would restart nginx"
return 0
fi
info "$ICON_WEBGUI Restarting nginx..."
if /etc/rc.d/rc.nginx restart >/dev/null 2>&1; then
success "nginx restarted"
return 0
else
error "nginx restart failed"
return 1
fi
}
# Restart emhttp — heavier fix, escalate if nginx didn't help
# emhttp drives the array, Docker management, shares — recovers cleanly but takes longer
restart_emhttp() {
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would restart emhttp"
return 0
fi
info "$ICON_WEBGUI Restarting emhttp..."
if /etc/rc.d/rc.emhttp restart >/dev/null 2>&1; then
success "emhttp restarted"
return 0
else
error "emhttp restart failed"
return 1
fi
}
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_WEBGUI WebGUI Watchdog ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_WEBGUI WebGUI Watchdog — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
echo "$ICON_WEBGUI URL: $WEBGUI_URL"
echo ""
START=$(date +%s)
RECOVERY_ACTION=""
RECOVERY_OK=false
# Initial check
info "Checking WebGUI..."
log "WebGUI check — $WEBGUI_URL"
# ── Healthy — completely silent ───────────────────────────────────────────────────────────────
if check_webgui; then
success "$ICON_WEBGUI WebGUI is responding — nothing to do"
echo ""
echo "━━━━━ $ICON_SUMMARY WEBGUI WATCHDOG SUMMARY ━━━━━"
echo "$ICON_WEBGUI Status: $ICON_RUNNING HEALTHY"
echo "$ICON_TIME Duration: $(format_duration $(($(date +%s) - START)))"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
log "WebGUI responding — healthy ✅"
exit 0
fi
# WebGUI not responding — begin escalation
warn "$ICON_WEBGUI WebGUI is not responding at $WEBGUI_URL"
# ── Step 1: Restart nginx ──
# ── Not responding — begin escalation ────────────────────────────────────────────────────────
echo ""
echo "━━━ $ICON_WEBGUI Step 1 — Nginx Restart ━━━"
echo "━━━ $ICON_WEBGUI WebGUI Watchdog — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME)"
echo ""
warn "WebGUI not responding at $WEBGUI_URL — beginning escalation"
restart_nginx
# ── Step 1 — nginx restart ────────────────────────────────────────────────────────────────────
echo ""
echo "━━━ Step 1 — nginx Restart ━━━"
if [[ "$DRY_RUN" == false ]]; then
info "Waiting ${WEBGUI_NGINX_WAIT}s for nginx to recover..."
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would restart nginx"
else
warn "Restarting nginx..."
if /etc/rc.d/rc.nginx restart >/dev/null 2>&1; then
# Verify nginx actually running
sleep 2
if pgrep -x nginx >/dev/null 2>&1; then
warn "nginx restarted ✅"
else
error "nginx not running after restart command"
fi
else
error "nginx restart command failed"
fi
log "Waiting ${WEBGUI_NGINX_WAIT}s for nginx to recover..."
sleep "$WEBGUI_NGINX_WAIT"
if check_webgui; then
success "$ICON_WEBGUI WebGUI recovered after nginx restart"
notify "WebGUI recovered on $(hostname) after nginx restart" "WebGUI Watchdog" "warning"
echo ""
echo "━━━━━ $ICON_SUMMARY WEBGUI WATCHDOG SUMMARY ━━━━━"
echo "$ICON_WEBGUI Status: $ICON_SUCCESS RECOVERED via nginx restart"
echo "$ICON_TIME Duration: $(format_duration $(($(date +%s) - START)))"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
RECOVERY_ACTION="nginx restart"
RECOVERY_OK=true
fi
fi
# ── Step 2 — php-fpm restart ──────────────────────────────────────────────────────────────────
if [[ "$RECOVERY_OK" == false ]]; then
echo ""
echo "━━━ Step 2 — php-fpm Restart ━━━"
warn "WebGUI still not responding — restarting php-fpm"
warn "WebGUI may be frozen due to worker exhaustion (check system_tuning_monitor.sh)"
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would restart php-fpm"
else
if /etc/rc.d/rc.php-fpm restart >/dev/null 2>&1; then
sleep 2
if pgrep -f "php-fpm" >/dev/null 2>&1; then
warn "php-fpm restarted ✅"
else
error "php-fpm not running after restart command"
fi
else
error "php-fpm restart command failed"
fi
warn "WebGUI still not responding after nginx restart — escalating to emhttp"
log "Waiting ${WEBGUI_PHP_WAIT:-10}s for php-fpm to recover..."
sleep "${WEBGUI_PHP_WAIT:-10}"
if check_webgui; then
RECOVERY_ACTION="php-fpm restart"
RECOVERY_OK=true
fi
fi
fi
# ── Step 2: Restart emhttp ──
echo ""
echo "━━━ $ICON_WEBGUI Step 2 — emhttp Restart ━━━"
warn "Restarting emhttp — this is the unRAID management daemon"
warn "Array, Docker management and shares remain running but WebGUI will be briefly unavailable"
restart_emhttp
# ── Step 3 — emhttp restart ───────────────────────────────────────────────────────────────────
if [[ "$RECOVERY_OK" == false ]]; then
echo ""
echo "━━━ Step 3 — emhttp Restart ━━━"
warn "WebGUI still not responding — restarting emhttp (unRAID management daemon)"
warn "Array, Docker, and shares remain running — WebGUI management will briefly restart"
if [[ "$DRY_RUN" == false ]]; then
info "Waiting ${WEBGUI_EMHTTP_WAIT}s for emhttp to recover..."
sleep "$WEBGUI_EMHTTP_WAIT"
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would restart emhttp"
else
if /etc/rc.d/rc.emhttp restart >/dev/null 2>&1; then
sleep 2
if pgrep -x emhttp >/dev/null 2>&1; then
warn "emhttp restarted ✅"
else
error "emhttp not running after restart command"
fi
else
error "emhttp restart command failed"
fi
if check_webgui; then
success "$ICON_WEBGUI WebGUI recovered after emhttp restart"
notify "WebGUI recovered on $(hostname) after emhttp restart — check system health" "WebGUI Watchdog" "warning"
log "Waiting ${WEBGUI_EMHTTP_WAIT}s for emhttp to recover..."
sleep "$WEBGUI_EMHTTP_WAIT"
echo ""
echo "━━━━━ $ICON_SUMMARY WEBGUI WATCHDOG SUMMARY ━━━━━"
echo "$ICON_WEBGUI Status: $ICON_SUCCESS RECOVERED via emhttp restart"
echo "$ICON_TIME Duration: $(format_duration $(($(date +%s) - START)))"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
if check_webgui; then
RECOVERY_ACTION="emhttp restart"
RECOVERY_OK=true
fi
fi
fi
# ── Both restarts failed ──
END=$(date +%s)
# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
echo ""
echo "━━━━━ $ICON_SUMMARY WEBGUI WATCHDOG SUMMARY ━━━━━"
echo "$ICON_WEBGUI Status: $ICON_ERROR UNRECOVERED — manual intervention needed"
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_WEBGUI URL: $WEBGUI_URL"
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
echo ""
if [[ "$DRY_RUN" == false ]]; then
notify "WebGUI unrecovered on $(hostname) after nginx and emhttp restart — manual intervention needed" "WebGUI Watchdog" "warning"
exit 1
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — no services restarted"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
fi
if [[ "$RECOVERY_OK" == true ]]; then
warn "$ICON_SUCCESS WebGUI recovered via: $RECOVERY_ACTION"
notify "WebGUI recovered on $(hostname) ($MY_ID) via $RECOVERY_ACTION — monitor for recurrence" \
"WebGUI Watchdog" "warning"
else
echo "$ICON_ERROR Status: UNRECOVERED — all three restart steps failed"
echo "$ICON_ERROR Manual intervention needed:"
echo " 1. Check: pgrep -x nginx emhttp"
echo " 2. Check: journalctl -u nginx --since '10 minutes ago'"
echo " 3. Try: server_reboot.sh if nothing else works"
notify "WebGUI UNRECOVERED on $(hostname) ($MY_ID) — nginx + php-fpm + emhttp restart all failed — manual intervention needed" \
"WebGUI Watchdog" "warning"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
[[ "$RECOVERY_OK" == false && "$DRY_RUN" == false ]] && exit 1
exit 0
user_script_plug-in.sh
+1006 -349
View File
File diff suppressed because it is too large Load Diff