From a3f90ed175744579ee4a489e5a5e196a3ba616f2 Mon Sep 17 00:00:00 2001 From: FailedProxy Date: Thu, 23 Apr 2026 22:44:56 -0400 Subject: [PATCH] created loop for both watchdogs and consolidated orch lists --- Docker_Essentials/docker_watchdog.sh | 571 ++++++++- Master.conf | 1100 ++++++----------- Orchestrators/README-Orchestrators.md | 476 ++++--- Orchestrators/array_start.sh | 106 ++ Orchestrators/daily_sync_maintenance.sh | 284 +++++ Orchestrators/media_management.sh | 177 --- Orchestrators/media_shares_sync.sh | 141 --- ...tenance.sh => weekly _sync_maintenance.sh} | 79 +- Rsync/README-Rsync_Setup.md | 571 ++++++--- unRAID_Essentials/system_watchdog.sh | 427 +++++++ user_script_plug-in.sh | 65 +- 11 files changed, 2530 insertions(+), 1467 deletions(-) create mode 100644 Orchestrators/array_start.sh create mode 100644 Orchestrators/daily_sync_maintenance.sh delete mode 100644 Orchestrators/media_management.sh delete mode 100644 Orchestrators/media_shares_sync.sh rename Orchestrators/{critical_shares_maintenance.sh => weekly _sync_maintenance.sh} (77%) diff --git a/Docker_Essentials/docker_watchdog.sh b/Docker_Essentials/docker_watchdog.sh index 229ab9e..3663fa6 100644 --- a/Docker_Essentials/docker_watchdog.sh +++ b/Docker_Essentials/docker_watchdog.sh @@ -2,7 +2,576 @@ # ----------------------------------------------------------------------------------------------- # --------------------------------- Docker Watchdog -------------------------------------------- # ----------------------------------------------------------------------------------------------- -# Two-tier self-healing container monitoring system. +# Two-tier self-healing container monitoring system — runs continuously as a background process. +# Started by array_start.sh at array start — runs until array stops or SIGTERM received. +# +# Tier 1 — Strict monitoring (configured containers only) +# Memory hard limits — immediate restart if exceeded +# CPU thresholds — strike system, restart after CPU_FAIL_LIMIT strikes +# HTTP responsiveness — strike system, restart after RESP_FAIL_LIMIT strikes +# Required containers — must always be running, strike system with skip list +# +# Tier 2 — Global health scan (all running containers) +# Unhealthy status — Docker HEALTHCHECK unhealthy → restart +# OOM killed — kernel killed container → restart + notify +# Crash loop detection — RestartCount climbing → notify, critical above limit +# Dead containers — remove and restart +# Unexpected exits — non-zero exit code → restart +# +# Cross-cutting intelligence: +# Startup grace period — skip restarts while system is still booting +# Dependency ordering — restart database before app +# Restart loop protect — stop restarting after X restarts in X hours → skip list +# Skip list auto-clear — clears when container recovers +# Notification batching — one clean summary per cycle, not one ping per event +# Quiet when healthy — only logs when something needs attention +# Parity awareness — skips restarts during parity check +# +# Continuous loop: +# Checks run every DOCKER_WATCHDOG_INTERVAL seconds (default 900 = 15min) +# Clean shutdown on SIGTERM/SIGINT — sent by array stop +# Variables scoped per-cycle — no state accumulation between cycles +# +# State files: +# WATCHDOG_STATE_FILE — strike counts (/tmp — resets on reboot) +# SYS_WATCHDOG_FAILED_FILE — persistent skip list (/boot — survives reboots) +# WATCHDOG_CONTAINER_RESTART_LOG — restart history for loop detection +# +# All configuration in Master.conf under Docker Watchdog section. +# Supports --dry-run and --status. +# ----------------------------------------------------------------------------------------------- + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "$SCRIPT_DIR/../Master.conf" +source "$SCRIPT_DIR/../common.sh" + +parse_args "$@" + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_GEAR Setup — runs once at start ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_GEAR Setup ━━━" + +if [[ "$EUID" -ne 0 ]]; then + error "Must be run as root" + exit 1 +fi + +success "Running as root" + +acquire_lock + +# Select correct per-host watchdog lists — done once at startup +detect_hosts + +if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then + WATCHDOG_REQUIRED_CONTAINERS=("${HOST1_WATCHDOG_REQUIRED_CONTAINERS[@]}") + declare -A WATCHDOG_CONTAINER_URLS + for key in "${!HOST1_WATCHDOG_CONTAINER_URLS[@]}"; do + WATCHDOG_CONTAINER_URLS["$key"]="${HOST1_WATCHDOG_CONTAINER_URLS[$key]}" + done +else + WATCHDOG_REQUIRED_CONTAINERS=("${HOST2_WATCHDOG_REQUIRED_CONTAINERS[@]}") + declare -A WATCHDOG_CONTAINER_URLS + for key in "${!HOST2_WATCHDOG_CONTAINER_URLS[@]}"; do + WATCHDOG_CONTAINER_URLS["$key"]="${HOST2_WATCHDOG_CONTAINER_URLS[$key]}" + done +fi + +info "Watchdog running as: $LOCAL_SERVER_NAME" +info "Check interval: ${DOCKER_WATCHDOG_INTERVAL}s" + +if ! command -v docker >/dev/null 2>&1; then + error "Docker not found" + exit 1 +fi + +success "Docker found" +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be restarted" + +# Ensure state files exist +touch "$WATCHDOG_STATE_FILE" "$WATCHDOG_CONTAINER_RESTART_LOG" \ + "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_SUMMARY Status ━━━ +# ----------------------------------------------------------------------------------------------- +if [[ "$SHOW_STATUS" == true ]]; then + echo "" + echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" + echo "$ICON_CONTAINERS Watched: ${!WATCHDOG_CONTAINERS[@]}" + echo "$ICON_CONTAINERS Required: ${WATCHDOG_REQUIRED_CONTAINERS[*]}" + echo "$ICON_WATCHDOG Scan all: $WATCHDOG_SCAN_ALL" + echo "$ICON_WATCHDOG Interval: ${DOCKER_WATCHDOG_INTERVAL}s" + echo "$ICON_WATCHDOG Startup grace: ${WATCHDOG_STARTUP_GRACE}s" + echo "$ICON_WATCHDOG Restart limit: $WATCHDOG_CONTAINER_RESTART_LIMIT in ${WATCHDOG_CONTAINER_RESTART_WINDOW}h" + echo "$ICON_WATCHDOG Batch notify: $WATCHDOG_BATCH_NOTIFY" + echo "$ICON_WATCHDOG Ignore list: ${WATCHDOG_SCAN_IGNORE[*]:-none}" + echo "$ICON_GEAR Dry Run: $DRY_RUN" + UPTIME_S=$(awk '{print int($1)}' /proc/uptime) + echo "$ICON_TIME System uptime: $(format_duration $UPTIME_S)" + [[ "$UPTIME_S" -lt "$WATCHDOG_STARTUP_GRACE" ]] && \ + warn "Within startup grace period — restarts suppressed" + echo "━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +# ----------------------------------------------------------------------------------------------- +# HELPERS — defined once, used every cycle +# ----------------------------------------------------------------------------------------------- + +get_strikes() { + grep "^${1}:" "${2}" 2>/dev/null | cut -d: -f2 || echo "0" +} + +set_strikes() { + local container="$1" count="$2" file="$3" + if grep -q "^${container}:" "$file" 2>/dev/null; then + sed -i "s/^${container}:.*/${container}:${count}/" "$file" + else + echo "${container}:${count}" >> "$file" + fi +} + +is_skipped() { + grep -q "^${1}$" "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null +} + +add_to_skip_list() { + local container="$1" reason="$2" + if ! is_skipped "$container"; then + echo "$container" >> "$SYS_WATCHDOG_FAILED_FILE" + error "$container added to skip list — $reason" + queue_notify "$container added to skip list on $(hostname) — $reason — manual intervention needed" "critical" + fi +} + +remove_from_skip_list() { + sed -i "/^${1}$/d" "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null + success "$1 removed from skip list — recovered" +} + +log_restart() { + local container="$1" + local now + now=$(date '+%Y-%m-%d %H:%M:%S') + local cutoff + cutoff=$(date -d "${WATCHDOG_CONTAINER_RESTART_WINDOW} hours ago" '+%Y-%m-%d %H:%M:%S') + echo "${container}|${now}" >> "$WATCHDOG_CONTAINER_RESTART_LOG" + local tmp="${WATCHDOG_CONTAINER_RESTART_LOG}.tmp" + awk -F'|' -v cutoff="$cutoff" '$2 >= cutoff' \ + "$WATCHDOG_CONTAINER_RESTART_LOG" > "$tmp" && \ + mv "$tmp" "$WATCHDOG_CONTAINER_RESTART_LOG" +} + +get_restart_count() { + local container="$1" + local cutoff + cutoff=$(date -d "${WATCHDOG_CONTAINER_RESTART_WINDOW} hours ago" '+%Y-%m-%d %H:%M:%S') + awk -F'|' -v c="$container" -v cutoff="$cutoff" \ + '$1==c && $2>=cutoff' "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | wc -l +} + +dependencies_satisfied() { + local container="$1" + local deps="${WATCHDOG_DEPENDENCIES[$container]:-}" + [[ -z "$deps" ]] && return 0 + for dep in $deps; do + local status + status=$(docker inspect -f '{{.State.Running}}' "$dep" 2>/dev/null) + if [[ "$status" != "true" ]]; then + warn "$container — dependency $dep is not running — skipping restart this cycle" + return 1 + fi + done + return 0 +} + +safe_restart() { + local container="$1" reason="$2" + local restart_count + restart_count=$(get_restart_count "$container") + if [[ "$restart_count" -ge "$WATCHDOG_CONTAINER_RESTART_LIMIT" ]]; then + add_to_skip_list "$container" \ + "restarted $restart_count times in ${WATCHDOG_CONTAINER_RESTART_WINDOW}h" + return 2 + fi + dependencies_satisfied "$container" || return 1 + local uptime_s + uptime_s=$(awk '{print int($1)}' /proc/uptime) + if [[ "$uptime_s" -lt "$WATCHDOG_STARTUP_GRACE" ]]; then + warn "$container — within startup grace period — skipping restart" + return 1 + fi + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would restart $container ($reason)" + return 0 + fi + info "Restarting $container ($reason) [restart $((restart_count + 1))/$WATCHDOG_CONTAINER_RESTART_LIMIT in window]..." + if docker restart "$container" >/dev/null 2>&1; then + success "$ICON_STARTED $container restarted" + log_restart "$container" + return 0 + else + error "Failed to restart $container" + return 1 + fi +} + +queue_notify() { + local message="$1" severity="${2:-warning}" + NOTIFY_EVENTS+=("${severity}|${message}") + log "Queued: $message" +} + +flush_notify() { + [[ ${#NOTIFY_EVENTS[@]} -eq 0 ]] && return + if [[ "$WATCHDOG_BATCH_NOTIFY" == "true" ]]; then + local highest_severity="normal" + local messages=() + for event in "${NOTIFY_EVENTS[@]}"; do + local sev="${event%%|*}" msg="${event#*|}" + messages+=("$msg") + [[ "$sev" == "critical" ]] && highest_severity="warning" + [[ "$sev" == "warning" && "$highest_severity" == "normal" ]] && highest_severity="warning" + done + local summary + summary=$(printf '%s. ' "${messages[@]}") + notify "Docker Watchdog on $(hostname) — ${#NOTIFY_EVENTS[@]} event(s): $summary" \ + "Docker Watchdog" "$highest_severity" + else + for event in "${NOTIFY_EVENTS[@]}"; do + local sev="${event%%|*}" msg="${event#*|}" + [[ "$sev" == "critical" ]] && sev="warning" + notify "$msg" "Docker Watchdog" "$sev" + done + fi + NOTIFY_EVENTS=() +} + +is_parity_running() { + grep -q "progress" /var/local/emhttp/parity-date.txt 2>/dev/null +} + +# ----------------------------------------------------------------------------------------------- +# CLEAN SHUTDOWN — trap SIGTERM/SIGINT from array stop +# ----------------------------------------------------------------------------------------------- +WATCHDOG_RUNNING=true + +cleanup() { + echo "" + info "Docker watchdog received shutdown signal — stopping cleanly" + WATCHDOG_RUNNING=false + exit 0 +} + +trap cleanup SIGTERM SIGINT + +# ----------------------------------------------------------------------------------------------- +# ━━━ CONTINUOUS MONITORING LOOP ━━━ +# ----------------------------------------------------------------------------------------------- +info "Docker watchdog started — checking every ${DOCKER_WATCHDOG_INTERVAL}s" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + +CYCLE=0 + +while [[ "$WATCHDOG_RUNNING" == true ]]; do + ((CYCLE++)) + CYCLE_START=$(date +%s) + + # Re-source Master.conf each cycle — picks up any config changes without restart + source "$SCRIPT_DIR/../Master.conf" + + # Rebuild per-host lists after re-source in case they changed + if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then + WATCHDOG_REQUIRED_CONTAINERS=("${HOST1_WATCHDOG_REQUIRED_CONTAINERS[@]}") + for key in "${!HOST1_WATCHDOG_CONTAINER_URLS[@]}"; do + WATCHDOG_CONTAINER_URLS["$key"]="${HOST1_WATCHDOG_CONTAINER_URLS[$key]}" + done + else + WATCHDOG_REQUIRED_CONTAINERS=("${HOST2_WATCHDOG_REQUIRED_CONTAINERS[@]}") + for key in "${!HOST2_WATCHDOG_CONTAINER_URLS[@]}"; do + WATCHDOG_CONTAINER_URLS["$key"]="${HOST2_WATCHDOG_CONTAINER_URLS[$key]}" + done + fi + + # Per-cycle variables — cleared each iteration, no accumulation + NOTIFY_EVENTS=() + T1_RESTARTS=0 + T1_WARNINGS=0 + T2_RESTARTS=0 + T2_WARNINGS=0 + + # Rebuild ignore map each cycle in case config was updated + declare -A IGNORE_MAP + for c in "${WATCHDOG_SCAN_IGNORE[@]}"; do + [[ -n "$c" ]] && IGNORE_MAP["$c"]=1 + done + + # Startup grace check + UPTIME_SECONDS=$(awk '{print int($1)}' /proc/uptime) + IN_GRACE_PERIOD=false + if [[ "$UPTIME_SECONDS" -lt "$WATCHDOG_STARTUP_GRACE" ]]; then + IN_GRACE_PERIOD=true + fi + + # Parity check — skip restarts during parity to avoid I/O interference + if is_parity_running; then + log "Parity check in progress — skipping restart actions this cycle" + sleep "$DOCKER_WATCHDOG_INTERVAL" + continue + fi + + # ── TIER 1 — Strict Monitoring ────────────────────────────────────────────────────────── + + # Required containers + if [[ ${#WATCHDOG_REQUIRED_CONTAINERS[@]} -gt 0 ]]; then + for container in "${WATCHDOG_REQUIRED_CONTAINERS[@]}"; do + [[ -z "$container" ]] && continue + STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null) + + if is_skipped "$container"; then + if [[ "$STATUS" == "true" ]]; then + remove_from_skip_list "$container" + set_strikes "$container" 0 "$WATCHDOG_STATE_FILE" + sed -i "/^${container}|/d" "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null + queue_notify "$container recovered on $(hostname) — removed from skip list" "normal" + else + warn "$container — on skip list, manual intervention needed" + fi + continue + fi + + if [[ "$STATUS" == "true" ]]; then + set_strikes "$container" 0 "$WATCHDOG_STATE_FILE" + else + STRIKES=$(get_strikes "$container" "$WATCHDOG_STATE_FILE") + STRIKES=$(( STRIKES + 1 )) + set_strikes "$container" "$STRIKES" "$WATCHDOG_STATE_FILE" + warn "$container — not running (strike $STRIKES/$SYS_WATCHDOG_STRIKE_LIMIT)" + ((T1_WARNINGS++)) + + if [[ "$STRIKES" -ge "$SYS_WATCHDOG_STRIKE_LIMIT" ]]; then + result=0 + safe_restart "$container" "required container down" || result=$? + case $result in + 0) set_strikes "$container" 0 "$WATCHDOG_STATE_FILE" + ((T1_RESTARTS++)) + queue_notify "$container was down and restarted on $(hostname)" "warning" ;; + 2) : ;; + *) queue_notify "$container failed to restart on $(hostname)" "warning" ;; + esac + fi + fi + done + fi + + # Memory and CPU monitoring + if [[ ${#WATCHDOG_CONTAINERS[@]} -gt 0 ]]; then + STATS=$(docker stats --no-stream \ + --format "{{.Name}}|{{.MemUsage}}|{{.CPUPerc}}" 2>/dev/null) + TOTAL_CORES=$(nproc 2>/dev/null || echo 1) + + for container in "${!WATCHDOG_CONTAINERS[@]}"; do + MEM_LIMIT_MB="${WATCHDOG_CONTAINERS[$container]}" + CONTAINER_STATS=$(echo "$STATS" | grep "^${container}|" | head -1) + [[ -z "$CONTAINER_STATS" ]] && continue + + MEM_USAGE=$(echo "$CONTAINER_STATS" | cut -d'|' -f2 | awk '{print $1}') + MEM_UNIT=$(echo "$MEM_USAGE" | grep -oE '[A-Za-z]+') + MEM_VALUE=$(echo "$MEM_USAGE" | grep -oE '[0-9.]+') + + case "$MEM_UNIT" in + GiB|GB) MEM_MB=$(awk "BEGIN {printf \"%.0f\", $MEM_VALUE * 1024}") ;; + MiB|MB) MEM_MB=$(awk "BEGIN {printf \"%.0f\", $MEM_VALUE}") ;; + KiB|KB) MEM_MB=$(awk "BEGIN {printf \"%.0f\", $MEM_VALUE / 1024}") ;; + *) MEM_MB=0 ;; + esac + + CPU_RAW=$(echo "$CONTAINER_STATS" | cut -d'|' -f3 | tr -d '%') + CPU_NORM=$(awk "BEGIN {printf \"%.1f\", $CPU_RAW / $TOTAL_CORES}") + CPU_INT=$(printf "%.0f" "$CPU_NORM") + + if [[ "$MEM_MB" -ge "$MEM_LIMIT_MB" ]]; then + error "$container — memory exceeded hard limit ${MEM_LIMIT_MB}MB" + safe_restart "$container" "memory hard limit exceeded" + ((T1_RESTARTS++)) + queue_notify "$container exceeded memory limit on $(hostname) — restarted" "warning" + fi + + if [[ "$CPU_INT" -ge "$HARD_CPU_THRESHOLD" ]]; then + CPU_STRIKES=$(get_strikes "${container}_cpu" "$WATCHDOG_STATE_FILE") + CPU_STRIKES=$(( CPU_STRIKES + 1 )) + set_strikes "${container}_cpu" "$CPU_STRIKES" "$WATCHDOG_STATE_FILE" + if [[ "$CPU_STRIKES" -ge "$CPU_FAIL_LIMIT" ]]; then + safe_restart "$container" "CPU threshold exceeded" + set_strikes "${container}_cpu" 0 "$WATCHDOG_STATE_FILE" + ((T1_RESTARTS++)) + queue_notify "$container CPU ${CPU_NORM}% on $(hostname) — restarted" "warning" + fi + else + set_strikes "${container}_cpu" 0 "$WATCHDOG_STATE_FILE" + fi + done + fi + + # HTTP responsiveness + if [[ ${#WATCHDOG_CONTAINER_URLS[@]} -gt 0 ]]; then + for container in "${!WATCHDOG_CONTAINER_URLS[@]}"; do + URL="${WATCHDOG_CONTAINER_URLS[$container]}" + if curl -sf --max-time "$CURL_TIMEOUT" "$URL" >/dev/null 2>&1; then + set_strikes "${container}_http" 0 "$WATCHDOG_STATE_FILE" + else + HTTP_STRIKES=$(get_strikes "${container}_http" "$WATCHDOG_STATE_FILE") + HTTP_STRIKES=$(( HTTP_STRIKES + 1 )) + set_strikes "${container}_http" "$HTTP_STRIKES" "$WATCHDOG_STATE_FILE" + warn "$container — not responding at $URL (strike $HTTP_STRIKES/$RESP_FAIL_LIMIT)" + ((T1_WARNINGS++)) + if [[ "$HTTP_STRIKES" -ge "$RESP_FAIL_LIMIT" ]]; then + safe_restart "$container" "HTTP unresponsive" + set_strikes "${container}_http" 0 "$WATCHDOG_STATE_FILE" + ((T1_RESTARTS++)) + queue_notify "$container unresponsive at $URL on $(hostname) — restarted" "warning" + fi + fi + done + fi + + # ── TIER 2 — Global Health Scan ───────────────────────────────────────────────────────── + if [[ "$WATCHDOG_SCAN_ALL" == "true" ]]; then + ALL_CONTAINERS=$(docker ps --format "{{.Names}}" 2>/dev/null) + + # Unhealthy containers + if [[ "$WATCHDOG_RESTART_UNHEALTHY" == "true" ]]; then + UNHEALTHY=$(docker ps --filter health=unhealthy --format "{{.Names}}" 2>/dev/null) + while IFS= read -r container; do + [[ -z "$container" ]] && continue + [[ -n "${IGNORE_MAP[$container]:-}" ]] && continue + is_skipped "$container" && continue + error "$container — unhealthy" + ((T2_WARNINGS++)) + result=0 + safe_restart "$container" "unhealthy health status" || result=$? + [[ $result -eq 0 ]] && ((T2_RESTARTS++)) && \ + queue_notify "$container unhealthy on $(hostname) — restarted" "warning" + done <<< "$UNHEALTHY" + fi + + # OOM killed + if [[ "$WATCHDOG_NOTIFY_OOM" == "true" ]]; then + while IFS= read -r container; do + [[ -z "$container" ]] && continue + [[ -n "${IGNORE_MAP[$container]:-}" ]] && continue + is_skipped "$container" && continue + OOM=$(docker inspect -f '{{.State.OOMKilled}}' "$container" 2>/dev/null) + if [[ "$OOM" == "true" ]]; then + error "$container — OOM killed" + ((T2_WARNINGS++)) + result=0 + safe_restart "$container" "OOM killed" || result=$? + [[ $result -eq 0 ]] && ((T2_RESTARTS++)) && \ + queue_notify "$container OOM killed on $(hostname) — restarted" "warning" + fi + done <<< "$ALL_CONTAINERS" + fi + + # Crash loop detection + if [[ "$WATCHDOG_NOTIFY_CRASHLOOP" == "true" ]]; then + while IFS= read -r container; do + [[ -z "$container" ]] && continue + [[ -n "${IGNORE_MAP[$container]:-}" ]] && continue + is_skipped "$container" && continue + RESTART_COUNT=$(docker inspect -f '{{.RestartCount}}' "$container" 2>/dev/null || echo 0) + PREV_COUNT=$(grep "^${container}_docker:" "$WATCHDOG_STATE_FILE" 2>/dev/null | cut -d: -f2 || echo 0) + set_strikes "${container}_docker" "$RESTART_COUNT" "$WATCHDOG_STATE_FILE" + if [[ "$RESTART_COUNT" -gt "$PREV_COUNT" && "$RESTART_COUNT" -gt 0 ]]; then + ((T2_WARNINGS++)) + if [[ "$RESTART_COUNT" -ge "$WATCHDOG_CRASH_LIMIT" ]]; then + error "$container — crash loop CRITICAL: $RESTART_COUNT restarts" + queue_notify "$container crash loop CRITICAL on $(hostname) — manual intervention needed" "critical" + else + warn "$container — restarted since last check (total: $RESTART_COUNT)" + queue_notify "$container restarted on $(hostname) — count: $RESTART_COUNT" "warning" + fi + fi + done <<< "$ALL_CONTAINERS" + fi + + # Dead containers + if [[ "$WATCHDOG_RESTART_DEAD" == "true" ]]; then + DEAD=$(docker ps -a --filter status=dead --format "{{.Names}}" 2>/dev/null) + while IFS= read -r container; do + [[ -z "$container" ]] && continue + [[ -n "${IGNORE_MAP[$container]:-}" ]] && continue + is_skipped "$container" && continue + error "$container — dead" + ((T2_WARNINGS++)) + RESTART_COUNT=$(get_restart_count "$container") + if [[ "$RESTART_COUNT" -ge "$WATCHDOG_CONTAINER_RESTART_LIMIT" ]]; then + add_to_skip_list "$container" "dead — restarted $RESTART_COUNT times" + elif [[ "$DRY_RUN" == false ]]; then + docker rm "$container" >/dev/null 2>&1 + if docker start "$container" >/dev/null 2>&1; then + success "$container removed from dead state and restarted" + log_restart "$container" + ((T2_RESTARTS++)) + queue_notify "$container was dead on $(hostname) — restarted" "warning" + fi + fi + done <<< "$DEAD" + fi + + # Unexpected exits + if [[ "$WATCHDOG_RESTART_CRASHED" == "true" ]]; then + CRASHED=$(docker ps -a \ + --filter status=exited \ + --format "{{.Names}}|{{.Status}}" 2>/dev/null | \ + grep -v "Exited (0)") + while IFS='|' read -r container status; do + [[ -z "$container" ]] && continue + [[ -n "${IGNORE_MAP[$container]:-}" ]] && continue + is_skipped "$container" && continue + SKIP=false + for req in "${WATCHDOG_REQUIRED_CONTAINERS[@]}"; do + [[ "$container" == "$req" ]] && SKIP=true && break + done + [[ "$SKIP" == true ]] && continue + error "$container — $status (unexpected exit)" + ((T2_WARNINGS++)) + result=0 + safe_restart "$container" "unexpected exit" || result=$? + [[ $result -eq 0 ]] && ((T2_RESTARTS++)) && \ + queue_notify "$container crashed on $(hostname) ($status) — restarted" "warning" + done <<< "$CRASHED" + fi + fi + + # Send notifications if any events this cycle + flush_notify + + # Only log summary if something happened — quiet when all healthy + TOTAL_RESTARTS=$(( T1_RESTARTS + T2_RESTARTS )) + TOTAL_WARNINGS=$(( T1_WARNINGS + T2_WARNINGS )) + + if [[ "$TOTAL_RESTARTS" -gt 0 || "$TOTAL_WARNINGS" -gt 0 ]]; then + CYCLE_END=$(date +%s) + echo "" + echo "━━━ $ICON_WATCHDOG Cycle $CYCLE — $(date '+%Y-%m-%d %H:%M:%S') ━━━" + echo "$ICON_WATCHDOG T1: $T1_RESTARTS restarts / $T1_WARNINGS warnings" + echo "$ICON_WATCHDOG T2: $T2_RESTARTS restarts / $T2_WARNINGS warnings" + echo "$ICON_TIME Duration: $(format_duration $(( CYCLE_END - CYCLE_START )))" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + else + log "Cycle $CYCLE — all healthy ($(date '+%H:%M:%S'))" + fi + + # Sleep until next cycle — interruptible by SIGTERM + sleep "$DOCKER_WATCHDOG_INTERVAL" & + wait $! + +done # # Tier 1 — Strict monitoring (configured containers only) # Memory hard limits — immediate restart if exceeded diff --git a/Master.conf b/Master.conf index a49e45a..25603d7 100644 --- a/Master.conf +++ b/Master.conf @@ -10,23 +10,27 @@ # Change a value here and it affects all scripts that use it — no hunting through files. # To disable something: comment it out with # rather than deleting it. # To add a new rsync profile: add a key to each PROFILE_* array. -# To add a new media maintenance job: add a line to MEDIA_MAINTENANCE_JOBS. +# To add or remove orchestrator jobs: edit the arrays in the ORCHESTRATORS section. # # ── INDEX ───────────────────────────────────────────────────────────────────────────────────── # # Section Description # ─────────────────────────────────────────────────────────────────────────────────────────── -# HOST CONFIGURATION Server hostnames and SSH key paths +# HOST CONFIGURATION Server hostnames, SSH keys, and Emby connection details # LOGGING Enable or disable verbose logging # NOTIFICATIONS unRAID native and Discord webhook settings # GIT / REPO Gitea repository and SSH settings # +# ── ORCHESTRATORS ────────────────────────────────────────────────────────────────────────── +# ARRAY START Scripts launched at array start (array_start.sh) +# DAILY SYNC MAINTENANCE Job list + media shares (daily_sync_maintenance.sh) +# WEEKLY SYNC MAINTENANCE Job list + sync jobs + sync settings (weekly_sync_maintenance.sh) +# MEDIA MANAGEMENT Job list for media_management.sh +# # ── RSYNC ────────────────────────────────────────────────────────────────────────────────── -# RSYNC DEFAULTS Global fallback rsync settings +# RSYNC DEFAULTS Global fallback rsync options and limits # REMOTE HEALTH CHECKS Rootfs threshold for pre-flight abort -# DAILY SYNC SHARES Media shares synced by media_shares_sync.sh (per-host) -# PERSONAL ENCRYPTED SHARES Per-user private shares — ZFS encrypted, synced separately -# RSYNC PROFILE SYSTEM Per-profile overrides (appdata profiles) +# RSYNC PROFILE SYSTEM Per-profile overrides for appdata syncs # # ── FAILOVER ─────────────────────────────────────────────────────────────────────────────── # FAILOVER Mutual container failover between two servers @@ -40,8 +44,8 @@ # ── DOCKER ESSENTIALS ────────────────────────────────────────────────────────────────────── # DOCKER DAILY RESTART Containers restarted daily # DOCKER WEEKLY RESTART Containers restarted weekly -# DOCKER WATCHDOG Two-tier self-healing container monitoring -# DOCKER NETWORK CONNECT Connect containers to extra networks on boot +# DOCKER WATCHDOG Continuous two-tier self-healing container monitoring +# DOCKER NETWORK CONNECT Connect containers to extra networks on array start # # ── UNRAID ESSENTIALS ────────────────────────────────────────────────────────────────────── # REBOOT User warning delay before scheduled reboot @@ -54,7 +58,6 @@ # ── MEDIA ────────────────────────────────────────────────────────────────────────────────── # MEDIA PERMISSIONS Share list, mode and owner for permissions script # MEDIA CLEANER Anime and media folder lists and file patterns -# MEDIA MANAGEMENT Orchestrator job list for media_management.sh # ARR CLEANUP Lidarr, Sonarr, Radarr orphan file cleanup # ARR FAILED/STALLED RECOVERY Auto blocklist + re-search failed imports and stalled downloads # @@ -69,31 +72,32 @@ # ZFS MEMORY SNAPSHOT Weekly ZFS health and memory diagnostic report # BANDWIDTH MONITOR Daily rsync transfer logging and weekly summary # HEALTH DIGEST Aggregated system health digest — always/smart/weekly -# CRITICAL SHARES MAINTENANCE Weekly clean sync + container updates (Emby + auth stack) -# EMBY Emby URL and API key — used by multiple scripts # EMBY SESSION REPORT Weekly Emby usage statistics via API # # ── SYSTEM WATCHDOG ──────────────────────────────────────────────────────────────────────── -# SYSTEM WATCHDOG System health monitoring — last line of defense +# SYSTEM WATCHDOG Continuous system health monitoring — last line of defense # # ============================================================================================== -# ━━━ Host Configuration ━━━ -# Hostnames must match Tailscale machine names exactly — case sensitive. +# ============================================================================================== +# ── HOST CONFIGURATION ──────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +# ━━━ Hosts ━━━ +# Hostnames must match exact Docker/unRAID hostnames — case sensitive. # Used by detect_hosts() in common.sh to determine which server is local and which is remote. # Both servers run identical scripts — host detection makes them bidirectional. HOST1="unRAID-Gmer4Lfe" HOST2="unRAID-Jayred365" # SSH keys for server-to-server rsync and failover container operations. -# HOST1_SSH_KEY is used when HOST1 SSHes to HOST2 and vice versa. # Both keys must be in /root/.ssh/ and authorised in the remote server's authorized_keys. HOST1_SSH_KEY="/root/.ssh/Gmer4Lfe-rsync-key" HOST2_SSH_KEY="/root/.ssh/Jayred365-rsync-key" -# Emby — per-host container name, URL and API key. +# ━━━ Emby ━━━ # Defined once here — referenced by transcode_manager.sh, emby_session_report.sh, -# emby_database_repair.sh, critical_shares_maintenance.sh and TRANSCODE_SERVERS array. +# emby_database_repair.sh, weekly_sync_maintenance.sh, and TRANSCODE_SERVERS array. # API key: Emby Dashboard → API Keys → + New Key HOST1_EMBY_CONTAINER="Emby" HOST1_EMBY_URL="http://localhost:8096" @@ -103,91 +107,79 @@ HOST2_EMBY_URL="http://localhost:8096" # same port — different server, different key HOST2_EMBY_API_KEY="your-host2-emby-api-key" -# ━━━ Logging ━━━ +# ============================================================================================== +# ── LOGGING ─────────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== + # Controls verbose [LOG] output across all scripts. # true = show detailed [LOG] lines — useful for debugging or first-time setup # false = show only user-facing output — cleaner for scheduled runs ENABLE_LOGGING=true -# ━━━ Notifications ━━━ -# Two independent notification channels — either or both can be active simultaneously. +# ============================================================================================== +# ── NOTIFICATIONS ───────────────────────────────────────────────────────────────────────────── +# ============================================================================================== # unRAID native notification system — integrates with the bell icon in the WebGUI. -# Recommended: set Settings → Notification Settings to errors/warnings only so -# normal completions don't create noise. The ecosystem sends: -# normal — job completed successfully (informational) -# warning — something failed or needs attention +# normal = job completed successfully / warning = something failed or needs attention NOTIFY_UNRAID=true -# Discord webhook URL — paste the full webhook URL from your Discord server settings. -# Leave blank to disable Discord notifications entirely. +# Discord webhook URL — leave blank to disable DISCORD_WEBHOOK="" -# ━━━ Git / Repo ━━━ -# Gitea self-hosted repository settings used by git_pull_execute.sh. -# Running git_pull_execute.sh on either server pulls the latest scripts and sets -# executable permissions automatically — keeps both servers in sync with one command. -# -# git_pull_execute.sh detects where the Gitea container is running at runtime: -# Gitea running locally → connects via local IP -# Gitea running remotely → connects via remote server's Tailscale IP -# No hardcoded assumptions about which server hosts Gitea — works through failover. -# If Gitea fails over to HOST2, HOST1 automatically finds it there and vice versa. +# ============================================================================================== +# ── GIT / REPO ──────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== - GITEA_CONTAINER="Gitea" # exact Docker container name - GITEA_REPO_PATH="FailedProxy/Unraid_Scripts.git" # repo path on Gitea - GITEA_DOMAIN="" # public domain fallback — e.g. git.gmer4lfe.com - # used if local and Tailscale both fail - # requires NPM + DNS setup before enabling +# Gitea self-hosted repository — used by git_pull_execute.sh. +# Detects Gitea container location at runtime — works through failover automatically. +# Falls back to GITEA_DOMAIN if local and Tailscale both fail. + GITEA_CONTAINER="Gitea" + GITEA_REPO_PATH="FailedProxy/Unraid_Scripts.git" + GITEA_DOMAIN="" # e.g. git.gmer4lfe.com — requires NPM + DNS TARGET_DIR="/mnt/user/appdata/unraid_scripts" - GITEA_SSH_KEY="/root/.ssh/unraid_gitea" # SSH key for authenticating to Gitea - SSH_PORT=221 # Gitea SSH port + GITEA_SSH_KEY="/root/.ssh/unraid_gitea" + SSH_PORT=221 # ============================================================================================== -# ── RSYNC ───────────────────────────────────────────────────────────────────────────────────── +# ── ORCHESTRATORS ────────────────────────────────────────────────────────────────────────────── # ============================================================================================== +# All orchestrator job lists live here — edit arrays to add/remove scripts. +# No changes to orchestrator scripts needed when adding or removing jobs. -# ━━━ Rsync Defaults ━━━ -# Global fallback values used when no profile match is found for a directory. -# Shares in HOST1_DAILY_SYNC_SHARES and HOST2_DAILY_SYNC_SHARES always use these globals — no profile is defined for them. -# Appdata shares (Arrs_Stack, Critical-Data etc.) match profiles by directory basename. -# If a profile key exists in a PROFILE_* array that value overrides the global. -# If a profile key is missing the global below is used as the fallback. +# ━━━ Array Start ━━━ +# Scripts launched by array_start.sh when the array comes online. +# Launched in order — each as a background process. +# One-shot scripts (ramdisk, syslog, fpm, network) run and exit naturally. +# Continuous scripts (watchdogs, failover) run until array stops. - BW_LIMIT=12500 # network transfer speed cap in KB/s — 12500 ≈ 100Mbit - RETRY_COUNT=3 # number of retry attempts if rsync fails before giving up - SLEEP=300 # seconds to wait between retry attempts - CRITICAL_CONTAINER_NAMES=() # containers to stop on REMOTE before rsync — profiles override - DELAYED_CONTAINERS=() # containers needing delay before starting — profiles override - CONTAINER_DELAY=5 # seconds to wait before starting delayed containers - EXCLUDE_DIRS=() # directories to exclude from transfer — profiles override +ARRAY_START_SCRIPTS=( + "unRAID_Essentials/ramdisk_setup.sh" # creates ramdisk + symlink before Emby starts + "unRAID_Essentials/docker_syslog_filter.sh" # suppress veth noise before logs fill + "unRAID_Essentials/php_fpm_max_children.sh" # WebGUI performance tuning + "Docker_Essentials/docker_network_connect.sh" # connect containers to extra networks + "unRAID_Essentials/system_watchdog.sh" # system health monitor — continuous loop + "Docker_Essentials/docker_watchdog.sh" # container health monitor — continuous loop + "Failover/failover.sh" # mutual failover — continuous loop +) -# Default rsync options used when no profile match is found. -# --delete removes files on remote that no longer exist on source (mirror behaviour) -# --inplace writes directly to destination file — better for large files, avoids temp copies -# --no-whole-file forces delta transfer even on fast local-like connections - DEFAULT_RSYNC_OPTS=(-av --info=progress2 --human-readable --bwlimit="$BW_LIMIT" --delete --inplace --no-whole-file) +# ━━━ Daily Sync Maintenance ━━━ +# daily_sync_maintenance.sh runs the media share sync built into the script first, +# then iterates DAILY_MAINTENANCE_SCRIPTS for additional jobs. +# Schedule: 0 1 * * * (1am daily) -# ━━━ Remote Health Checks ━━━ -# Pre-flight check run before every rsync — aborts if remote rootfs (/) usage is at or -# above this percentage. When the remote array is down or drives are missing, rsync -# writes land on rootfs instead of /mnt/user — this fills the filesystem rapidly and -# can crash the remote server. 75% gives headroom to detect the problem early. - ROOTFS_WARN=75 +DAILY_MAINTENANCE_SCRIPTS=( + "git_pull_execute.sh" # pull latest scripts — always runs first + "Docker_Essentials/docker_daily_restart.sh" # daily container restarts +) -# ━━━ Daily Sync Shares ━━━ -# Media shares synced once daily by Orchestrators/media_shares_sync.sh. -# Each server only syncs the shares it is source of truth for — direction is automatic. -# detect_hosts() determines which server is running and picks the correct list. -# -# HOST1 pushes its truth shares TO HOST2. -# HOST2 pushes its truth shares TO HOST1. +# Media shares synced daily by daily_sync_maintenance.sh. +# Each server syncs only the shares it owns (source of truth) — direction is automatic. +# HOST1 pushes its truth shares to HOST2. HOST2 pushes its truth shares to HOST1. # Never both pushing the same share — one server is always the truth holder. -# # These shares use DEFAULT_RSYNC_OPTS — no profile entry needed. -# For shares needing custom bandwidth or container stops — create a profile below instead. +# For shares needing custom options or container stops — create a profile in the RSYNC section. -# HOST1 truth shares — pushed from HOST1 to HOST2 nightly HOST1_DAILY_SYNC_SHARES=( /mnt/user/Books /mnt/user/Intros @@ -204,126 +196,120 @@ HOST1_DAILY_SYNC_SHARES=( /mnt/user/Anime_Movies-Old ) -# HOST2 truth shares — pushed from HOST2 to HOST1 nightly -# HOST2 is source of truth for anime — his arrs manage these shares HOST2_DAILY_SYNC_SHARES=( /mnt/user/Anime_Movies /mnt/user/Anime_Shows ) -# ━━━ Personal Encrypted Shares ━━━ -# Personal shares synced to the remote server for offsite backup. -# These are independent of the failover container stack — data backup only. -# Each user syncs their own personal share to the other server. -# -# ── ZFS ENCRYPTION SETUP (unRAID 7) ───────────────────────────────────────────────────────── -# Encrypting your personal share means the remote admin can see the share exists -# and its file sizes but cannot read any content without your passphrase or keyfile. -# ZFS encrypts at the dataset level — rsync copies encrypted blocks as-is. -# The remote server never needs your key. -# -# Setup steps on HOST1: -# 1. In unRAID UI → go to your ZFS pool (Main tab → pool name) -# 2. Click the pool to expand it -# 3. Click "+ Dataset" to create a new dataset -# 4. Name it: e.g. Gmer4Lfe-Personal -# 5. Enable Encryption → set your passphrase (or keyfile path) -# ⚠️ Write your passphrase down — if lost, data is unrecoverable -# 6. Go to Settings → Shares → Add Share -# 7. Set the share path to your new encrypted dataset -# 8. Set Use cache: Only (keeps data on ZFS pool, not array) -# -# Auto-unlock on boot (optional — keyfile approach): -# 1. Create a keyfile: dd if=/dev/urandom bs=32 count=1 | base64 > /root/.zfs-keys/personal.key -# 2. Store keyfile on HOST1 only — never sync it to HOST2 -# 3. Set the dataset to use keyfile instead of passphrase -# 4. Add to /etc/rc.local or a startup script: -# zfs load-key -L file:///root/.zfs-keys/personal.key poolname/Gmer4Lfe-Personal -# zfs mount poolname/Gmer4Lfe-Personal -# Manual unlock alternative (most secure): -# zfs load-key poolname/Gmer4Lfe-Personal (prompts for passphrase) -# zfs mount poolname/Gmer4Lfe-Personal -# -# Verify encryption is active before syncing: -# zfs get encryption poolname/Gmer4Lfe-Personal -# Should show: encryption aes-256-gcm (or similar) -# -# Once set up — add the share to PERSONAL_SYNC_SHARES below. -# rsync copies encrypted blocks to remote — remote admin cannot decrypt without your key. -# ───────────────────────────────────────────────────────────────────────────────────────────── - -# HOST1 personal shares synced to HOST2 for offsite backup -# These sync via media_shares_sync.sh or on their own schedule -# Encrypted datasets sync as encrypted — remote cannot read content +# Personal encrypted shares — synced for offsite backup, independent of media shares. +# ZFS encrypted at dataset level — remote receives encrypted blocks, cannot read content. +# See README-Rsync_Setup.md for ZFS encryption setup before uncommenting. HOST1_PERSONAL_SHARES=( # /mnt/user/Gmer4Lfe-Personal # uncomment after creating encrypted dataset ) -# HOST2 personal shares synced to HOST1 for offsite backup HOST2_PERSONAL_SHARES=( # /mnt/user/Jayred365-Personal # uncomment after creating encrypted dataset ) +# ━━━ Weekly Sync Maintenance ━━━ +# weekly_sync_maintenance.sh handles the critical sync built into the script first: +# stop containers both sides → pull updates → sync Emby + Critical-Data → restart +# Then iterates WEEKLY_MAINTENANCE_SCRIPTS for additional jobs. +# Schedule: 30 2 * * 0 (Sunday 2:30am) + +WEEKLY_MAINTENANCE_SCRIPTS=( + "Docker_Essentials/docker_weekly_restart.sh" # weekly container restarts after sync +) + +# Shares synced by weekly_sync_maintenance.sh during the maintenance window. +# Containers are stopped both sides before these sync — full clean state guaranteed. +# Profiles drive container stops, excludes, and options — configure in RSYNC section. +# Order matters — Emby first, then auth stack. +WEEKLY_SYNC_JOBS=( + "/mnt/user/Media_Server/Emby" # emby profile — full clean mirror + "/mnt/user/appdata-Failover/Critical-Data" # critical-data profile — auth stack +) + +# Container update toggles for the weekly sync window. +# Containers are already stopped for the sync — updates pull at no extra downtime. +# Both false → sync only, no updates. +# Toggle false temporarily to skip updates without changing the schedule. + CRITICAL_SYNC_UPDATES=true # pull container updates locally + CRITICAL_SYNC_UPDATES_REMOTE=false # pull container updates on remote via SSH + +# ━━━ Media Management ━━━ +# Job list for Orchestrators/media_management.sh — called by daily_sync_maintenance.sh. +# Runs sequentially — permissions first, then cleaners, then arr cleanup. +# Comment out any job to disable without removing it. + +MEDIA_MANAGEMENT_JOBS=( + "Media/media_shares_permissions.sh" # apply permissions — runs first + "Media/media_cleaner.sh anime" # remove junk from anime shares + "Media/media_cleaner.sh media" # remove junk from media shares + #"Media/lidarr_cleanup.sh" # remove orphaned music files + #"Media/sonarr_cleanup.sh" # remove orphaned TV files + #"Media/radarr_cleanup.sh" # remove orphaned movie files +) + +# ============================================================================================== +# ── RSYNC ───────────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +# ━━━ Rsync Defaults ━━━ +# Global fallback values used when no profile match is found. +# Media shares in HOST*_DAILY_SYNC_SHARES always use these globals — no profile needed. +# Appdata shares match profiles by directory basename (lowercased). + BW_LIMIT=12500 + RETRY_COUNT=3 + SLEEP=300 + CRITICAL_CONTAINER_NAMES=() + DELAYED_CONTAINERS=() + CONTAINER_DELAY=5 + EXCLUDE_DIRS=() + DEFAULT_RSYNC_OPTS=(-av --info=progress2 --human-readable --bwlimit="$BW_LIMIT" --delete --inplace --no-whole-file) + +# ━━━ Remote Health Checks ━━━ +# Pre-flight check — aborts if remote rootfs (/) usage is at or above this percentage. +# When remote array is down, rsync writes land on rootfs — fills fast and crashes the server. + ROOTFS_WARN=75 + # ━━━ Rsync Profile System ━━━ # Profiles allow per-share rsync behaviour without touching script logic. -# The profile key is matched automatically by the basename of the directory -# passed to rsync.sh (lowercased). +# Profile key matched by basename of directory passed to rsync.sh (lowercased). +# Override with --profile=name flag. # -# Example: -# rsync.sh /mnt/user/appdata-Failover/Arrs_Stack -# basename = Arrs_Stack → lowercased = arrs_stack → matches [arrs_stack] profile -# -# To add a new profile: -# 1. Add a key to each PROFILE_* array below using your chosen name -# 2. Call rsync.sh with a directory whose basename matches that key -# 3. Any array you omit falls back to its global default automatically -# -# IMPORTANT: PROFILE_RSYNC_OPTS does NOT inherit from DEFAULT_RSYNC_OPTS. -# If you define a profile entry you must list ALL desired options explicitly. +# IMPORTANT: PROFILE_RSYNC_OPTS does NOT inherit DEFAULT_RSYNC_OPTS. +# List ALL desired options explicitly when defining a profile. # # Current profiles: -# arrs_stack — Sonarr, Radarr, Lidarr, Readarr, Prowlarr, Bazarr, Pinchflat -# Lower bandwidth — runs alongside media syncs -# Containers stopped during sync for data consistency -# critical-data — Auth stack: NPM, Authelia, Mariadb-Authelia, Redis-Authelia, LLDAP -# High bandwidth — small data, synced frequently -# Authelia needs delayed start — database containers must be ready first -# gmer4lfe — Server-specific appdata: Organizr, UptimeKuma, VaultWarden -# Medium bandwidth — personal services, no container stop needed -# important-data — NextCloud + Postgres database -# High bandwidth — NextCloud needs graceful stop before sync -# NextCloud needs delayed start — Postgres must be accepting connections -# emby — Emby media server appdata and metadata only -# Medium bandwidth — large appdata directory, no containers stopped +# arrs_stack — arr databases — lower bandwidth, containers stopped for consistency +# critical-data — auth stack — containers stopped both sides, Authelia delayed start +# gmer4lfe — server-specific appdata — no container stops needed +# important-data — NextCloud + Postgres — NextCloud delayed start after Postgres +# emby — weekly clean sync — both Emby stopped, full mirror, minimal excludes +# called by weekly_sync_maintenance.sh only — do NOT schedule separately +# emby-failover — frequent dirty sync — Emby stays running, WAL excluded, critical data only +# also used for failover writeback on handback -# Rsync options per profile — replaces DEFAULT_RSYNC_OPTS entirely for that profile run -# SPACE-SEPARATED STRINGS — converted to array at runtime by rsync.sh declare -A PROFILE_RSYNC_OPTS=( [arrs_stack]="-av --info=progress2 --human-readable --bwlimit=$BW_LIMIT --delete --inplace" [critical-data]="-av --human-readable --bwlimit=$BW_LIMIT --delete" [gmer4lfe]="-av --info=progress2 --bwlimit=$BW_LIMIT" [important-data]="-av --human-readable --bwlimit=$BW_LIMIT" - # emby — nightly clean sync, both Emby instances stopped - # WAL checkpointed on shutdown — full consistent mirror, minimal excludes [emby]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file" - # emby-failover — frequent dirty sync every 30-60min, Emby stays running - # Only critical failover data — what users need immediately on failover - # WAL excluded — safe dirty write while Emby is running - # This is also the list written back during failover handback [emby-failover]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file" ) -# Bandwidth limit in KB/s per profile — overrides global BW_LIMIT for this profile only declare -A PROFILE_BW_LIMIT=( - [arrs_stack]=5000 # lower — runs alongside other jobs, avoids saturating link - [critical-data]=9500 # high — small data, get it synced fast - [gmer4lfe]=8000 # medium - [important-data]=9500 # high — database sync needs to be fast and clean - [emby]=8000 # medium — full mirror, steady transfer - [emby-failover]=9500 # high — small critical dataset, get it synced fast + [arrs_stack]=5000 + [critical-data]=9500 + [gmer4lfe]=8000 + [important-data]=9500 + [emby]=8000 + [emby-failover]=9500 ) -# Retry attempts per profile before giving up — overrides global RETRY_COUNT declare -A PROFILE_RETRY_COUNT=( [arrs_stack]=3 [critical-data]=3 @@ -333,83 +319,58 @@ declare -A PROFILE_RETRY_COUNT=( [emby-failover]=3 ) -# Seconds between retry attempts — overrides global SLEEP declare -A PROFILE_SLEEP=( [arrs_stack]=300 [critical-data]=300 [gmer4lfe]=300 [important-data]=300 [emby]=300 - [emby-failover]=120 # shorter — frequent sync, retry faster + [emby-failover]=120 ) # Containers stopped on BOTH LOCAL and REMOTE servers before rsync. -# Local stops first — flushes databases cleanly before push. -# Remote stops next — prevents writes while receiving. -# Only containers that were running get restarted — stopped containers stay stopped. -# Same container names used on both HOST1 and HOST2 — naming scheme is consistent. -# If a container is not found on a server it is skipped gracefully, not errored. +# Local stops first — flushes databases cleanly. Remote stops next — prevents writes while receiving. +# Same container names on both servers — consistent naming is required by this ecosystem. +# If a container is not found it is skipped gracefully, not errored. # SPACE-SEPARATED STRINGS — converted to array at runtime declare -A PROFILE_CRITICAL_CONTAINER_NAMES=( [arrs_stack]="Sonarr Lidarr Readarr Radarr Prowlarr Bazarr Pinchflat" [critical-data]="Mariadb-Authelia Mariadb-Authelia-Secondary Redis-Authelia Redis-Authelia-Secondary Lldap-Gmer4Lfe NginxProxyManager Authelia Authelia-Secondary" [gmer4lfe]="Organizrv2-Gmer4Lfe UptimeKuma-Gmer4Lfe VaultWarden-Gmer4Lfe" [important-data]="Postgres-NextCloud NextCloud" - [emby]="Emby" # nightly clean sync — Emby stopped both sides, WAL checkpointed - [emby-failover]="" # dirty sync — Emby stays running both sides + [emby]="Emby" + [emby-failover]="" ) -# Containers that need a delay before starting after rsync completes. -# Used when a container depends on another that was also stopped. -# Startup order for critical-data: -# Immediate: Mariadb x2, Redis x2, Lldap, NginxProxyManager -# Delayed: Authelia, Authelia-Secondary (need databases ready) # SPACE-SEPARATED STRINGS — converted to array at runtime declare -A PROFILE_DELAYED_CONTAINERS=( [arrs_stack]="" [critical-data]="Authelia Authelia-Secondary" [gmer4lfe]="" - [important-data]="NextCloud" # NextCloud needs Postgres accepting connections first + [important-data]="NextCloud" [emby]="" [emby-failover]="" ) -# Seconds to wait before starting delayed containers declare -A PROFILE_CONTAINER_DELAY=( [arrs_stack]=5 - [critical-data]=15 # 15s gives Mariadb, Redis, and LLDAP time to accept connections + [critical-data]=15 [gmer4lfe]=5 [important-data]=10 [emby]=5 [emby-failover]=5 ) -# Directories excluded from rsync transfer per profile. # SPACE-SEPARATED STRINGS — converted to array at runtime declare -A PROFILE_EXCLUDE_DIRS=( [arrs_stack]="logs *.tmp" [gmer4lfe]="logs *.tmp" [important-data]="logs *.tmp" - # Critical-Data — auth stack - # Containers stopped during sync — databases flush cleanly - # Generated configs, logs, and temp files excluded [critical-data]="logs *.tmp *.log nginx/temp nginx/cache __pycache__ notification.txt" - # Emby nightly clean sync — both Emby instances stopped - # WAL checkpointed on shutdown — safe to push everything except true junk - # Full faithful mirror: metadata, plugins, config all included [emby]="logs transcodes cache crash*" - # Emby failover dirty sync — Emby stays running - # Only what users need immediately on failover: - # users.db ← watch states, continue watching, next up, played state - # library.db ← library structure - # authentication.db ← API keys, sessions - # config/ ← server settings - # WAL and SHM excluded — unsafe while Emby is running - # Also the list written back during failover handback [emby-failover]="logs transcodes cache metadata *.db-wal *.db-shm crash* plugins root" ) -# Per-disk check toggle declare -A PROFILE_SKIP_DISK_CHECK=( [arrs_stack]=true [critical-data]=true @@ -423,230 +384,133 @@ declare -A PROFILE_SKIP_DISK_CHECK=( # ── FAILOVER ────────────────────────────────────────────────────────────────────────────────── # ============================================================================================== # Mutual container failover between two unRAID servers. -# Each server runs Failover/failover.sh independently — no coordination between servers. -# All decisions based solely on two pings: remote reachable + internet reachable. +# Each server runs Failover/failover.sh independently via array_start.sh. +# All decisions based on two pings: remote reachable + internet reachable. # -# ── STATES ──────────────────────────────────────────────────────────────────────────────────── -# NORMAL — remote up, internet up — own containers only, DDNS ON, silent -# FAILOVER — remote down, internet up — start remote containers (tiered by time) -# NO_INTERNET — internet down — stop own DDNS immediately, wait for recovery -# DARK — remote down AND internet down — same as NO_INTERNET +# States: NORMAL | FAILOVER | NO_INTERNET | DARK # -# ── DDNS RULES — ABSOLUTE ───────────────────────────────────────────────────────────────────── -# Each server owns its own DDNS — ON when that server has internet -# Script controls DDNS exclusively — network state NEVER auto-starts DDNS +# DDNS rules — absolute: # Internet loss → stop own DDNS immediately -# Failover → start remote DDNS as first action (Tier 1) -# Handback → stop remote DDNS FIRST → rsync → start local containers -# → start local DDNS LAST — only after containers confirmed up -# One DDNS per domain active at all times — never two, never zero for long -# 1 minute TTL + 1 minute check interval = minimal user impact +# Failover → start remote DDNS first (Tier 1) +# Handback → stop remote DDNS → rsync → start containers → start local DDNS last # -# ── HANDBACK SEQUENCE ───────────────────────────────────────────────────────────────────────── -# Strike confirmation → pre-flight → stop remote DDNS → stop remote containers -# → rsync writeback → start local containers → start local DDNS → NORMAL -# Containers only down during rsync window — minimise this time -# -# ── TIERED FAILOVER ─────────────────────────────────────────────────────────────────────────── -# Tier 1 — Immediate — vital services + Live TV — people are watching, can't wait -# Tier 2 — configurable delay — shared productivity services +# Tiers: +# Tier 1 — Immediate — vital services + Live TV +# Tier 2 — configurable delay — productivity services # Tier 3 — configurable delay — secondary services -# Tier 4 — configurable delay — arrs + downloaders — workflow continuity -# Delays set independently per host below +# Tier 4 — configurable delay — arrs + downloaders - EXTERNAL_IP="8.8.8.8" # external IP to ping for internet connectivity check - FAILOVER_CHECK_INTERVAL=120 # seconds between state checks - # 1 minute TTL + 2 minute interval = minimal gap - FAILOVER_HANDBACK_STRIKES=2 # consecutive remote-up checks before handback - # 2 strikes x 120s = 4 min confirmation window + EXTERNAL_IP="8.8.8.8" + FAILOVER_CHECK_INTERVAL=120 + FAILOVER_HANDBACK_STRIKES=2 FAILOVER_STATE_FILE="/boot/config/failover_state.db" - # persists on /boot/ — survives reboots - # tracks: state, failover_start, strikes, tier flags # ━━━ Failover Test ━━━ -# Used by Failover/failover_test.sh — controlled simulation via iptables block. -# All failover logic stays in failover.sh — test script is the harness only. -# ⚠️ Run during maintenance window — real containers start and stop during the test. -# Use --dry-run first to walk through phases without touching anything. - FAILOVER_TEST_BLOCK_WAIT=150 # seconds to hold iptables block - # must be > FAILOVER_CHECK_INTERVAL + buffer - FAILOVER_TEST_HANDBACK_WAIT=360 # seconds to wait for handback completion - # covers FAILOVER_HANDBACK_STRIKES x INTERVAL + rsync - -# ━━━ DDNS — Script Controlled Exclusively ━━━ -# Each server owns its own DDNS containers — one domain per server. -# DDNS is started and stopped ONLY by this script — never by network state returning. -# HOST1 DDNS starts last in handback (after containers confirmed up). -# HOST1 DDNS stops first on internet loss. -# HOST2 DDNS starts when HOST2 detects HOST1 is down (Tier 1). -# HOST2 DDNS stops before handback rsync begins. + FAILOVER_TEST_BLOCK_WAIT=150 + FAILOVER_TEST_HANDBACK_WAIT=360 +# ━━━ DDNS ━━━ HOST1_DDNS_CONTAINERS=( - "Gmer4Lfe.com" # HOST1's own DDNS — ON when HOST1 has internet - # covers gmer4lfe.com pointing to HOST1 IP + "Gmer4Lfe.com" ) HOST2_DDNS_CONTAINERS=( - "Gmer4Lfe.us" # HOST2's own DDNS — ON when HOST2 has internet - # covers gmer4lfe.us pointing to HOST2 IP + "Gmer4Lfe.us" ) -# ━━━ Containers to stop on internet loss ━━━ -# Own DDNS handled separately above — list additional containers here if needed -# These stop when this server loses internet — regardless of remote state +# ━━━ Internet Loss ━━━ FAILOVER_HOST1_STOP_ON_NO_NET=( - "Gmer4Lfe.com" # stop when HOST1 loses internet + "Gmer4Lfe.com" ) FAILOVER_HOST2_STOP_ON_NO_NET=( - "Gmer4Lfe.us" # stop when HOST2 loses internet + "Gmer4Lfe.us" ) -# ━━━ HOST1 runs these for HOST2 when HOST2 goes down ━━━ -# HOST2's DDNS listed in Tier 1 — starts immediately as first action -# List HOST2's specific services here — HOST2's own containers only -# Do NOT list shared services that HOST1 already runs +# ━━━ Tiered Container Lists ━━━ -# Tier 1 — Immediate — starts as soon as HOST2 is detected down +# HOST1 runs for HOST2 FAILOVER_HOST1_RUNS_FOR_HOST2_IMMEDIATE=( - "Gmer4Lfe.us" # HOST2's DDNS — start first, covers HOST2's domain - "VaultWarden-Jayred365" # HOST2's password manager — immediate access needed - # "container-placeholder" # add HOST2 specific services here + "Gmer4Lfe.us" + "VaultWarden-Jayred365" + # "container-placeholder" ) -# Tier 2 — starts after HOST2_TIER2_DELAY minutes FAILOVER_HOST1_RUNS_FOR_HOST2_2HR=( # "container-placeholder" ) -# Tier 3 — starts after HOST2_TIER3_DELAY minutes FAILOVER_HOST1_RUNS_FOR_HOST2_6HR=( # "container-placeholder" ) -# Tier 4 — starts after HOST2_TIER4_DELAY minutes FAILOVER_HOST1_RUNS_FOR_HOST2_18HR=( # "container-placeholder" ) -# ━━━ HOST2 runs these for HOST1 when HOST1 goes down ━━━ -# HOST1's DDNS listed in Tier 1 — starts immediately to cover HOST1's domain -# Live TV in Tier 1 — people are watching, cannot wait for tiered startup -# Auth stack in Tier 1 — everything proxied through NPM needs auth - -# Tier 1 — Immediate — vital services and Live TV cannot wait +# HOST2 runs for HOST1 FAILOVER_HOST2_RUNS_FOR_HOST1_IMMEDIATE=( - "Gmer4Lfe.com" # HOST1's DDNS — start first, covers HOST1's domain - "Emby" # media server — users are actively watching - "VaultWarden-Gmer4Lfe" # password manager — critical, immediate access needed - "Dispatcharr" # Live TV — people are watching, cannot wait - "Dispatcharr-Basic" # Live TV basic profile - "Dispatcharr-Iptv-Users" # Live TV IPTV users - "ErsatzTV-Emby" # Live TV scheduling and channel management + "Gmer4Lfe.com" + "Emby" + "VaultWarden-Gmer4Lfe" + "Dispatcharr" + "Dispatcharr-Basic" + "Dispatcharr-Iptv-Users" + "ErsatzTV-Emby" ) -# Tier 2 — starts after HOST1_TIER2_DELAY minutes -# Productivity services — important but can wait a couple of hours FAILOVER_HOST2_RUNS_FOR_HOST1_2HR=( - "Postgres-NextCloud" # NextCloud database — must start before NextCloud - "NextCloud" # file access and collaboration - "PostgreSQL_Immich" # Immich database - "Immich-Gmer4Lfe" # photo management + "Postgres-NextCloud" + "NextCloud" + "PostgreSQL_Immich" + "Immich-Gmer4Lfe" # "container-placeholder" ) -# Tier 3 — starts after HOST1_TIER3_DELAY minutes -# Secondary services — useful but not immediately critical FAILOVER_HOST2_RUNS_FOR_HOST1_6HR=( - "Gitea" # git server + "Gitea" # "container-placeholder" ) -# Tier 4 — starts after HOST1_TIER4_DELAY minutes -# Full workflow mode — arrs and downloaders -# Minimal writeback on handback — start fresh is cleaner than syncing download state FAILOVER_HOST2_RUNS_FOR_HOST1_18HR=( - "Sonarr" # TV show management - "Radarr" # movie management - "Lidarr" # music management - "Readarr" # book management - "Prowlarr" # indexer management - "Bazarr" # subtitle management - "SABnzbd-Gmer4Lfe" # usenet downloader - "Qbittorrent-Gmer4Lfe" # torrent downloader - "LidaTube" # YouTube music downloader - "Pinchflat" # YouTube channel downloader - "ChannelTube" # YouTube channel management + "Sonarr" + "Radarr" + "Lidarr" + "Readarr" + "Prowlarr" + "Bazarr" + "SABnzbd-Gmer4Lfe" + "Qbittorrent-Gmer4Lfe" + "LidaTube" + "Pinchflat" + "ChannelTube" # "container-placeholder" ) # ━━━ Tier Delay Settings ━━━ -# How long primary must be down before each tier activates — set in minutes -# Tier 1 is always immediate — no delay -# Set independently per host — a large server may want longer delays than a small one -# Adjust based on your tolerance for resource usage on the covering server +# Minutes before each tier activates. Tier 1 is always immediate. +HOST1_TIER2_DELAY=240 +HOST1_TIER3_DELAY=720 +HOST1_TIER4_DELAY=1440 -# Delays for HOST1's containers running on HOST2 (HOST1 is down) -HOST1_TIER2_DELAY=240 # 4 hours — NextCloud, Immich can wait -HOST1_TIER3_DELAY=720 # 12 hours — secondary services -HOST1_TIER4_DELAY=1440 # 24 hours — full workflow, arrs and downloaders - -# Delays for HOST2's containers running on HOST1 (HOST2 is down) HOST2_TIER2_DELAY=240 HOST2_TIER3_DELAY=720 HOST2_TIER4_DELAY=1440 # ━━━ Rsync Writeback Jobs ━━━ -# Run during handback — syncs critical appdata back to primary before containers restart. -# Containers are stopped before this runs — clean source, no competing writes. -# Full bandwidth available — DDNS stopped, containers stopped, nothing competing. -# -# ── WRITEBACK SKIP WINDOW ───────────────────────────────────────────────────────────────────── -# Short outages do not benefit from writeback — the covering server accumulated dirty -# or minimal data not worth writing over the primary's cleaner state. -# -# Emby syncs every 30min from a live running container (dirty sync). -# A clean full sync runs nightly at 2:30am with Emby stopped. -# After a short outage HOST1's nightly clean state is more reliable -# than HOST2's dirty 30min sync data — skip writeback entirely. -# -# Real world outage profile: -# 2-10 minutes — power blip, most common → skip writeback -# 10-60 minutes — ISP issue, router restart → skip writeback -# 1hr+ — actual problem → writeback worthwhile -# 18hr+ — Tier 4 activated → always writeback -# -# Tier 1 writeback delay — separate from Tier 1 start delay (Tier 1 always starts immediately) -# Tier 2+ reuse their existing TIER_DELAY vars — if containers started, time passed = writeback warranted -# Tier 4 writeback — skips if under HOST*_TIER4_DELAY (same threshold as container start) -# Only runs if outage was long enough to activate Tier 4 containers -# Automatically uses the opposing host's daily sync share list (HOST*_DAILY_SYNC_SHARES) -# FAILOVER_HOST*_WRITEBACK_TIER4 is for edge cases only — empty by default +# Syncs critical appdata back to primary on handback — containers stopped before this runs. +# Short outages skip writeback — primary state is more reliable than dirty sync data. +# Tier 4 automatically syncs HOST*_DAILY_SYNC_SHARES — add edge cases here only. -HOST1_TIER1_WRITEBACK_DELAY=60 # minutes — skip Tier 1 writeback if outage under this -HOST2_TIER1_WRITEBACK_DELAY=60 # minutes — skip Tier 1 writeback if outage under this -# Tier 2 writeback threshold = HOST1_TIER2_DELAY (reused) -# Tier 3 writeback threshold = HOST1_TIER3_DELAY (reused) -# Tier 4 always writebacks — no threshold +HOST1_TIER1_WRITEBACK_DELAY=60 +HOST2_TIER1_WRITEBACK_DELAY=60 -# ── WRITEBACK JOB LISTS ─────────────────────────────────────────────────────────────────────── -# Organised by tier — writeback runs per tier based on outage duration -# Priority: -# Tier 1 — Emby userdata, auth stack — small, fast, most important -# Tier 2 — NextCloud, Immich — user files that may have changed -# Tier 3 — secondary services -# Tier 4 — always runs if Tier 4 activated — arrs accumulated meaningful state -# Skip — media files (already on primary, never moved) -# Skip — downloads (start fresh — cleaner than syncing partial state) - -# HOST1 writeback tiers — run by HOST2 during HOST1 handback FAILOVER_HOST1_WRITEBACK_TIER1=( - "/mnt/user/Media_Server/Emby" # Emby userdata, playstates, metadata - # "location-placeholder" + "/mnt/user/Media_Server/Emby" ) FAILOVER_HOST1_WRITEBACK_TIER2=( - "/mnt/user/appdata-Failover/Important-Data" # NextCloud + Postgres + "/mnt/user/appdata-Failover/Important-Data" ) FAILOVER_HOST1_WRITEBACK_TIER3=( @@ -654,15 +518,11 @@ FAILOVER_HOST1_WRITEBACK_TIER3=( ) FAILOVER_HOST1_WRITEBACK_TIER4=( - # Edge case paths outside of normal HOST1_DAILY_SYNC_SHARES - # Tier 4 automatically syncs HOST1_DAILY_SYNC_SHARES back — add extras here only "/mnt/user/appdata-Failover/Arrs_Stack" ) -# HOST2 writeback tiers — run by HOST1 during HOST2 handback FAILOVER_HOST2_WRITEBACK_TIER1=( # "/mnt/user/appdata-Failover/Jayred365-Emby" - # "/mnt/user/appdata-Failover/Jayred365-Critical" ) FAILOVER_HOST2_WRITEBACK_TIER2=( @@ -674,8 +534,6 @@ FAILOVER_HOST2_WRITEBACK_TIER3=( ) FAILOVER_HOST2_WRITEBACK_TIER4=( - # Edge case paths outside of normal HOST2_DAILY_SYNC_SHARES - # Tier 4 automatically syncs HOST2_DAILY_SYNC_SHARES back — add extras here only "/mnt/user/appdata-Failover/Arrs_Stack" ) @@ -684,8 +542,7 @@ FAILOVER_HOST2_WRITEBACK_TIER4=( # ============================================================================================== # ━━━ Docker Daily Restart ━━━ -# Containers restarted every day — keeps services fresh, clears memory leaks. -# Case-sensitive — must match exact Docker container names in the unRAID Docker tab. +# Restarted by docker_daily_restart.sh via daily_sync_maintenance.sh — 1am daily. DAILY_RESTART_CONTAINERS=( "NginxProxyManager" "Authelia" @@ -696,8 +553,7 @@ DAILY_RESTART_CONTAINERS=( ) # ━━━ Docker Weekly Restart ━━━ -# Less critical services that benefit from periodic restart but don't need daily cycling. -# Case-sensitive — must match exact Docker container names in the unRAID Docker tab. +# Restarted by docker_weekly_restart.sh via weekly_sync_maintenance.sh — Sunday 2:30am. WEEKLY_RESTART_CONTAINERS=( "NextCloud" "Organizrv2-Gmer4Lfe" @@ -706,20 +562,18 @@ WEEKLY_RESTART_CONTAINERS=( ) # ━━━ Docker Watchdog ━━━ -# Two-tier self-healing container monitoring. -# Tier 1 — strict monitoring of explicitly configured containers -# Tier 2 — global health scan of ALL running containers +# Continuous two-tier self-healing container monitoring. +# Started by array_start.sh — runs until array stops. +# Re-sources Master.conf each cycle — add/remove containers without restarting watchdog. +# Silent when all healthy — only logs when something needs attention. # -# Cross-cutting intelligence: -# Startup grace — skip restarts while system is still booting -# Dependency order — restart database before app -# Restart loop — stop restarting after limit hit → skip list → notify critical -# Skip list — persistent across reboots, auto-clears when container recovers -# Batch notify — one clean summary per run +# Tier 1 — strict monitoring of explicitly configured containers +# Memory hard limits, CPU thresholds, HTTP responsiveness, required container checks +# Tier 2 — global health scan of ALL running containers +# Unhealthy status, OOM kills, crash loops, dead containers, unexpected exits # Memory hard limits in MB — immediate restart if exceeded -# 20GB=20480 16GB=16384 14GB=14336 12GB=12288 10GB=10240 -# 8GB=8192 6GB=6144 4GB=4096 2GB=2048 1GB=1024 +# 20GB=20480 16GB=16384 12GB=12288 8GB=8192 4GB=4096 2GB=2048 1GB=1024 declare -A WATCHDOG_CONTAINERS=( ["Emby"]=16384 ["LidaTube"]=6144 @@ -727,18 +581,14 @@ declare -A WATCHDOG_CONTAINERS=( ["Code-Server"]=1024 ) -# Per-host HTTP health check URLs — docker_watchdog.sh picks correct list via detect_hosts() declare -A HOST1_WATCHDOG_CONTAINER_URLS=( ["Emby"]="http://localhost:8096" ) declare -A HOST2_WATCHDOG_CONTAINER_URLS=( - ["Emby"]="http://localhost:8096" # his Emby — same port, different server + ["Emby"]="http://localhost:8096" ) -# Per-host required containers — core stack that must always be running -# docker_watchdog.sh picks correct list via detect_hosts() -# Strike system — persistent skip list on /boot/, auto-clears on recovery HOST1_WATCHDOG_REQUIRED_CONTAINERS=( "NginxProxyManager" "Lldap-Gmer4Lfe" @@ -751,84 +601,51 @@ HOST1_WATCHDOG_REQUIRED_CONTAINERS=( HOST2_WATCHDOG_REQUIRED_CONTAINERS=( "NginxProxyManager" - # add HOST2's required containers here + # add HOST2 required containers here ) -# Strike state file — /tmp resets on reboot which is correct for strike tracking WATCHDOG_STATE_FILE="/tmp/container_watchdog_state.db" - -# CPU thresholds — normalised against total core count automatically at runtime - SOFT_CPU_THRESHOLD=80 # warn at this % of total system CPU - HARD_CPU_THRESHOLD=85 # strike at this % of total system CPU - CPU_FAIL_LIMIT=2 # consecutive CPU strikes before container restart - -# Memory soft threshold — warn when container reaches this % of its hard limit -# Hard limit exceeded triggers immediate restart regardless of strikes + SOFT_CPU_THRESHOLD=80 + HARD_CPU_THRESHOLD=85 + CPU_FAIL_LIMIT=2 SOFT_MEM_THRESHOLD=80 + RESP_FAIL_LIMIT=2 + CURL_TIMEOUT=5 + DOCKER_WATCHDOG_INTERVAL=900 -# HTTP responsiveness check settings - RESP_FAIL_LIMIT=2 # consecutive failed curl checks before restart - CURL_TIMEOUT=5 # seconds before curl gives up per check - -# Tier 2 master toggle — false disables global scan entirely WATCHDOG_SCAN_ALL=true -# Containers to skip in Tier 2 scan entirely -# Add intentionally stopped containers or containers managed by other systems WATCHDOG_SCAN_IGNORE=( # "container-name" ) -# Individual Tier 2 check toggles — disable checks that cause false positives - WATCHDOG_RESTART_UNHEALTHY=true # restart containers with unhealthy Docker health status - WATCHDOG_RESTART_DEAD=true # remove and restart containers in dead state - WATCHDOG_RESTART_CRASHED=true # restart containers that exited with non-zero exit code - WATCHDOG_NOTIFY_OOM=true # restart and notify when OOM killed by kernel - WATCHDOG_NOTIFY_CRASHLOOP=true # notify when Docker restart count is climbing - -# Crash loop threshold — notify critical if Docker has restarted container this many times + WATCHDOG_RESTART_UNHEALTHY=true + WATCHDOG_RESTART_DEAD=true + WATCHDOG_RESTART_CRASHED=true + WATCHDOG_NOTIFY_OOM=true + WATCHDOG_NOTIFY_CRASHLOOP=true WATCHDOG_CRASH_LIMIT=5 - -# Startup grace — skip restarts while system is still booting -# Prevents false positives while containers are coming up after array start - WATCHDOG_STARTUP_GRACE=600 # seconds after boot before watchdog acts on failures - -# Restart loop protection — stops hammering broken containers -# After limit hit → skip list → notify critical → manual intervention needed -# Skip list auto-clears when container is found running again - WATCHDOG_CONTAINER_RESTART_LIMIT=3 # max watchdog restarts allowed in window - WATCHDOG_CONTAINER_RESTART_WINDOW=1 # hours — rolling window for restart count + WATCHDOG_STARTUP_GRACE=600 + WATCHDOG_CONTAINER_RESTART_LIMIT=3 + WATCHDOG_CONTAINER_RESTART_WINDOW=1 WATCHDOG_CONTAINER_RESTART_LOG="/boot/config/container_restart_history.db" - # /boot/ survives reboots — bounded, auto-purges + WATCHDOG_BATCH_NOTIFY=true -# Dependency ordering — skip restarting a container if its dependency is also down -# Dependency gets restarted first, dependent picked up on the next watchdog cycle -# Prevents Authelia restarting before its database is ready — it would just fail again -# Format: ["dependent"]="dependency1 dependency2" declare -A WATCHDOG_DEPENDENCIES=( ["Authelia"]="Mariadb-Authelia Redis-Authelia" ["Authelia-Secondary"]="Mariadb-Authelia Redis-Authelia-Secondary" ["NextCloud"]="Postgres-NextCloud" ) -# Notification batching — one clean summary per run instead of one ping per event -# true = batch all events into a single notification at end of run -# false = send individual notification per event as it happens - WATCHDOG_BATCH_NOTIFY=true - # ━━━ Docker Network Connect ━━━ -# Connects containers to extra Docker networks on array start — many-to-many. -# Every container in the list connects to every network in the list. -# Useful when containers need to communicate across networks they were not originally -# configured with — e.g. memcached needing access to the nextcloud-aio network. +# Connects containers to extra networks on array start via array_start.sh. NETWORK_CONNECT_CONTAINERS=( - "memcached" - "Npm-CrowdSec" +# "memcached" +# "Npm-CrowdSec" ) NETWORK_CONNECT_NETWORKS=( - "high-availability" # Docker network name — must exist before array start - # containers needing their own network but accessible from main custom network +# "high-availability" ) # ============================================================================================== @@ -836,51 +653,36 @@ NETWORK_CONNECT_NETWORKS=( # ============================================================================================== # ━━━ Reboot ━━━ -# Seconds of warning broadcast to logged-in users before server_reboot.sh reboots. -# Gives users time to save work before the system goes down. REBOOT_SLEEP=300 # ━━━ Mover ━━━ -# Seconds to wait before mover_stop.sh sends SIGTERM to the mover process. -# Gives the mover time to finish its current file operation cleanly before being killed. MOVER_STOP_TIMEOUT=300 # ━━━ Syslog Filter ━━━ -# Path for the rsyslog filter file that suppresses Docker veth noise from syslog. -# Without this filter every Docker network interface change floods the syslog on boot. FILTER_FILE="/etc/rsyslog.d/ignore-docker-veth.conf" # ━━━ PHP-FPM ━━━ -# Higher max_children allows more concurrent PHP requests to the unRAID WebGUI. -# Set based on available RAM — too high can cause memory pressure on low-RAM systems. PHP_CONF="/etc/php-fpm.d/www.conf" PHP_MAX_CHILDREN=250 # ━━━ Clear Logs ━━━ -# System log files cleared weekly to prevent rootfs fill over time. LOG_FILES=(/var/log/syslog /var/log/messages /var/log/dmesg) # ━━━ WebGUI Watchdog ━━━ -# Escalation: nginx restart → recheck → emhttp restart → recheck → notify warning. -# emhttp is the core unRAID daemon — restarting is more disruptive but recovers cleanly. - WEBGUI_URL="http://localhost" # adjust if running non-standard port - WEBGUI_TIMEOUT=5 # seconds before curl gives up on the WebGUI check - WEBGUI_NGINX_WAIT=15 # seconds to wait after nginx restart before rechecking - WEBGUI_EMHTTP_WAIT=30 # seconds to wait after emhttp restart — takes longer + WEBGUI_URL="http://localhost" + WEBGUI_TIMEOUT=5 + WEBGUI_NGINX_WAIT=15 + WEBGUI_EMHTTP_WAIT=30 # ============================================================================================== # ── MEDIA ───────────────────────────────────────────────────────────────────────────────────── # ============================================================================================== # ━━━ Media Permissions ━━━ -# Mode and owner applied recursively to all shares in MEDIA_PERMISSION_SHARES. -# Run by Media/media_shares_permissions.sh via the media_management.sh orchestrator. -# 777 and nobody:users is standard for unRAID media shares accessible by Docker containers. -# Applied recursively so large shares take time — run overnight via orchestrator. +# Applied recursively by media_shares_permissions.sh via MEDIA_MANAGEMENT_JOBS. PERMISSIONS_MODE="777" PERMISSIONS_OWNER="nobody:users" -# Shares to apply permissions to — add or remove paths as your library grows. MEDIA_PERMISSION_SHARES=( /mnt/user/Anime_Movies /mnt/user/Anime_Movies-Old @@ -907,12 +709,9 @@ MEDIA_PERMISSION_SHARES=( ) # ━━━ Media Cleaner ━━━ -# Removes junk files from media shares using configurable file pattern lists. -# Two profiles: anime and media — each with their own folder list and patterns. -# Run via Media/media_cleaner.sh anime or Media/media_cleaner.sh media -# Called automatically by media_management.sh via MEDIA_MAINTENANCE_JOBS below. +# Removes junk files from media shares — two profiles: anime and media. +# Called via MEDIA_MANAGEMENT_JOBS. Run manually: Media/media_cleaner.sh anime|media -# Folders scanned by the anime profile — anime downloads commonly include these junk files ANIME_CLEAN_FOLDERS=( /mnt/user/Anime_Movies /mnt/user/Anime_Movies-Old @@ -920,7 +719,6 @@ ANIME_CLEAN_FOLDERS=( /mnt/user/Anime_Shows-Old ) -# Folders scanned by the media profile MEDIA_CLEAN_FOLDERS=( /mnt/user/Kids_Movies /mnt/user/Kids_Tv_Shows @@ -931,7 +729,6 @@ MEDIA_CLEAN_FOLDERS=( /mnt/user/Tv_Shows ) -# File patterns deleted by the anime profile — common junk from anime download groups ANIME_FILE_PATTERNS=( '*.sfv' '*.md5' '*.sha1' '*.txt' '*.url' '*.lnk' '*.rar' '*.zip' '*.info' '*.torrent' '*.sample*' '*.proof*' @@ -939,8 +736,6 @@ ANIME_FILE_PATTERNS=( '*.log' '*.json' ) -# File patterns deleted by the media profile -# Includes *.iso and *.lrc not needed in anime profile MEDIA_FILE_PATTERNS=( '*.sfv' '*.md5' '*.sha1' '*.txt' '*.url' '*.lnk' '*.rar' '*.zip' '*.info' '*.torrent' '*.sample*' '*.proof*' @@ -948,269 +743,160 @@ MEDIA_FILE_PATTERNS=( '*.log' '*.json' '*.iso' '*.lrc' ) -# ━━━ Media Management Orchestrator ━━━ -# Job list for Orchestrators/media_management.sh — runs scripts sequentially in order. -# Format: "folder/script.sh optional_argument" -# Order matters — permissions runs first so cleaners and arr scripts see correct ownership. -# Arr cleanup scripts run last — they depend on clean folders from the cleaner steps. -# Comment out any job to disable without removing it — easy to re-enable later. -MEDIA_MAINTENANCE_JOBS=( - "Media/media_shares_permissions.sh" # apply permissions first - "Media/media_cleaner.sh anime" # remove junk from anime shares - "Media/media_cleaner.sh media" # remove junk from media shares - #"Media/lidarr_cleanup.sh" # remove orphaned music files - #"Media/sonarr_cleanup.sh" # remove orphaned TV files - #"Media/radarr_cleanup.sh" # remove orphaned movie files -) - # ━━━ Arr Cleanup ━━━ -# Lidarr, Sonarr and Radarr orphan file cleanup via their respective APIs. -# Each arr script queries its API to get all tracked file paths, then compares against -# what exists on disk. Files not tracked by the arr and older than ORPHAN_AGE days are deleted. +# Orphan file cleanup via Lidarr, Sonarr, and Radarr APIs. +# Compares tracked file paths from API against disk — deletes untracked files older than ORPHAN_AGE. +# detect_hosts() selects correct URL, API key, and root path at runtime. # -# Each server runs different arrs managing different shares: -# HOST1: Sonarr (Tv_Shows), Radarr (Movies), Lidarr (Music) -# HOST2: Sonarr (Anime_Shows), Radarr (Anime_Movies) +# Protected patterns are NEVER deleted — cover art, metadata, subtitles generated by the arr +# are not included in the tracked file API response but must not be deleted. # -# detect_hosts() selects the correct URL, API key, and root path at runtime. -# Scripts run identically on both servers — configuration drives behavior. -# -# Why the age threshold matters: -# The arr downloads a file then processes it — there is a window where the file exists -# on disk but the arr hasn't imported it yet. ORPHAN_AGE prevents deleting files that -# are mid-import. 7 days is conservative and safe for any normal workflow. -# -# Protected patterns are NEVER deleted regardless of tracking status or age. -# These protect arr-generated metadata (cover art, .nfo files, subtitles) that the arr -# depends on but does not include in its tracked file API response. +# API versions: Sonarr v4 → /api/v3/ Radarr v6 → /api/v3/ Lidarr v3 → /api/v1/ +# Lidarr runs on HOST1 only. # ── Lidarr ──────────────────────────────────────────────────────────────────────────────────── -# Lidarr runs on HOST1 only — music library management -# HOST2 does not run Lidarr — no HOST2 Lidarr config needed - HOST1_LIDARR_URL="http://192.168.50.2:8686" HOST1_LIDARR_API_KEY="b2977e71ef074bc0a0529d9fcce3b2dc" -HOST1_LIDARR_MUSIC_ROOT="/mnt/user/Music-New" # must match root path set in Lidarr exactly +HOST1_LIDARR_MUSIC_ROOT="/mnt/user/Music-New" -LIDARR_ORPHAN_AGE=7 # days before untracked file eligible for deletion +LIDARR_ORPHAN_AGE=7 LIDARR_EXTENSIONS=("flac" "mp3" "m4a" "wav" "aac" "ogg" "opus" "wma") LIDARR_PROTECTED_PATTERNS=("*.jpg" "*.jpeg" "*.png" "*.nfo" "*.lrc") - # never deleted — cover art, metadata, lyrics +LIDARR_MAX_DELETE_GB=1 # require --i-know-what-im-doing if deletion exceeds this +LIDARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run +LIDARR_TRACKED_COUNT_FILE="/boot/config/lidarr_tracked.count" # ── Sonarr ──────────────────────────────────────────────────────────────────────────────────── -# HOST1 Sonarr manages Tv_Shows — HOST2 Sonarr manages Anime_Shows -# Scripts run on each server and hit their own local Sonarr instance - HOST1_SONARR_URL="http://192.168.50.2:8989" HOST1_SONARR_API_KEY="130decd3db5b4c25afad64864cd03f9f" -HOST1_SONARR_TV_ROOT="/mnt/user/Tv_Shows" # must match root path set in HOST1 Sonarr exactly +HOST1_SONARR_TV_ROOT="/mnt/user/Tv_Shows" -HOST2_SONARR_URL="http://localhost:8989" # HOST2 local Sonarr — update with actual port +HOST2_SONARR_URL="http://localhost:8989" HOST2_SONARR_API_KEY="your-host2-sonarr-api-key" -HOST2_SONARR_TV_ROOT="/mnt/user/Anime_Shows" # must match root path set in HOST2 Sonarr exactly +HOST2_SONARR_TV_ROOT="/mnt/user/Anime_Shows" SONARR_ORPHAN_AGE=7 SONARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "ts" "wmv" "mov") SONARR_PROTECTED_PATTERNS=("*.jpg" "*.jpeg" "*.png" "*.nfo" "*.srt" "*.sub" "*.ass" "*.ssa") - # never deleted — artwork, metadata, subtitles # ── Radarr ──────────────────────────────────────────────────────────────────────────────────── -# HOST1 Radarr manages Movies — HOST2 Radarr manages Anime_Movies -# Scripts run on each server and hit their own local Radarr instance - HOST1_RADARR_URL="http://192.168.50.2:7878" HOST1_RADARR_API_KEY="d43a3ec6cf1549edb4af0cc63f98b2a9" -HOST1_RADARR_MOVIES_ROOT="/mnt/user/Movies" # must match root path set in HOST1 Radarr exactly +HOST1_RADARR_MOVIES_ROOT="/mnt/user/Movies" -HOST2_RADARR_URL="http://localhost:7878" # HOST2 local Radarr — update with actual port +HOST2_RADARR_URL="http://localhost:7878" HOST2_RADARR_API_KEY="your-host2-radarr-api-key" -HOST2_RADARR_MOVIES_ROOT="/mnt/user/Anime_Movies" # must match root path set in HOST2 Radarr exactly +HOST2_RADARR_MOVIES_ROOT="/mnt/user/Anime_Movies" RADARR_ORPHAN_AGE=7 RADARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "wmv" "mov") RADARR_PROTECTED_PATTERNS=("*.jpg" "*.jpeg" "*.png" "*.nfo" "*.srt" "*.sub" "*.ass" "*.ssa") -# ━━━ Arr Import Recovery ━━━ -# Automatically blocklists and re-searches failed imports and stalled downloads. -# Runs daily at 5am — by this time overnight downloads are complete and any -# failures have had time to surface. Items under ARR_IMPORT_RECOVERY_AGE are -# skipped — gives the arr time to retry on its own before we intervene. +# ━━━ Arr Failed/Stalled Recovery ━━━ +# Auto blocklist + re-search failed imports and stalled downloads. +# Runs every 6 hours — schedule: 0 */6 * * * # -# Targets two problem types from the queue API: -# importFailed — downloaded successfully but arr couldn't import it -# stalled — download stuck with no connections or progress +# Targets four problem types: +# importFailed — downloaded but arr couldn't import +# importPending — downloaded, stuck waiting to import (won't self-resolve) +# error status — serious failure not covered above +# stalled — download stuck with no connections or progress # -# Action: blocklist the release + remove from queue + trigger new search -# Blocklist prevents the same bad release being grabbed again -# New search finds a different release automatically -# If all releases are bad → arr will exhaust options, manual check needed -# Notification tells you what was actioned so you can monitor -# -# Per-arr toggles — disable temporarily if an arr is having issues -# detect_hosts() selects correct URL and API key per server at runtime -# Lidarr runs on HOST1 only — exits cleanly on HOST2 +# Items newer than ARR_IMPORT_RECOVERY_AGE are skipped — gives arr time to retry first. +# API versions: Sonarr /api/v3/ — Radarr /api/v3/ — Lidarr /api/v1/ +# Lidarr runs on HOST1 only — exits cleanly on HOST2. -ARR_IMPORT_RECOVERY_AGE=12 # hours — skip items newer than this, give arr time to retry +ARR_IMPORT_RECOVERY_AGE=6 # hours — skip items newer than this -# Per-arr enable/disable -HOST1_SONARR_RECOVERY=true # Tv_Shows import recovery -HOST1_RADARR_RECOVERY=true # Movies import recovery -HOST1_LIDARR_RECOVERY=true # Music import recovery — HOST1 only -HOST2_SONARR_RECOVERY=true # Anime_Shows import recovery -HOST2_RADARR_RECOVERY=true # Anime_Movies import recovery +HOST1_SONARR_RECOVERY=true +HOST1_RADARR_RECOVERY=true +HOST1_LIDARR_RECOVERY=true # HOST1 only +HOST2_SONARR_RECOVERY=true +HOST2_RADARR_RECOVERY=true # ============================================================================================== # ── TRANSCODES ──────────────────────────────────────────────────────────────────────────────── # ============================================================================================== # Session-based storage allocator using filesystem symlink indirection. # ffmpeg resolves the symlink ONCE at session start — existing sessions are never affected. -# Only new sessions care about where the symlink currently points. # # How it works: -# ramdisk_setup.sh — run once at array start, creates tmpfs and sets symlink -# transcode_management.sh — every 3 min, runs cleanup then manager in correct order -# transcode_cleanup.sh — called by transcode_management.sh — removes old inactive files -# transcode_manager.sh — called by transcode_management.sh — manages symlink direction +# ramdisk_setup.sh — creates tmpfs and symlink at array start via array_start.sh +# transcode_management.sh — every 3min, runs cleanup then manager in correct order +# transcode_cleanup.sh — removes old inactive files +# transcode_manager.sh — manages symlink direction based on usage thresholds # -# ⚠️ Docker mount warning: -# Mount must use shared propagation so symlink flips are visible inside the container. -# In unRAID Extra Parameters — do NOT use standard path mapping for this mount: -# --mount type=bind,source=/mnt/ram-transcode,target=/ext-ram-transcode,bind-propagation=shared -# Standard bind mounts use rprivate — Docker locks the inode on first symlink flip -# and new sessions land on SSD permanently for that container run. +# ⚠️ Docker mount — must use shared propagation: +# --mount type=bind,source=/mnt/ram-transcode,target=/ext-ram-transcode,bind-propagation=shared +# Standard rprivate bind mounts lock the inode — sessions drift to SSD permanently. - RAMDISK_PATH="/mnt/ramdisk_transcodes" # tmpfs mount point created at array start - RAMDISK_SIZE="8G" # ceiling — tmpfs only uses RAM actually needed - TRANSCODE_LINK="/mnt/ram-transcode" # symlink Emby points at — location never changes - TRANSCODE_SSD="/mnt/cache/Temp_Storage/Emby/Transcodes/" # SSD fallback location - -# Usage thresholds in GB — hysteresis gap between WARN and LOW prevents flip-flop - RAMDISK_WARN_GB=6.8 # flip symlink to SSD at or above this usage - RAMDISK_LOW_GB=5.5 # flip symlink back to ramdisk when usage drops here - RAMDISK_SSD_MIN_GB=20 # minimum free GB on SSD required before allowing flip to SSD - -# Cleanup age thresholds — files must be older than these AND not open by any process - TRANSCODE_MAX_AGE=20 # minutes before a transcode file is eligible for cleanup - TRANSCODE_ORPHAN_AGE=30 # minutes before an orphaned file is eligible — extra caution buffer - -# Flip frequency alert — too many flips per hour indicates ramdisk needs to be larger - TRANSCODE_FLIP_WARN=3 # notify if symlink flips this many times in one hour - -# Permissions — must match your Emby container user +# ━━━ Transcode Manager ━━━ + RAMDISK_PATH="/mnt/ramdisk_transcodes" + RAMDISK_SIZE="8G" + TRANSCODE_LINK="/mnt/ram-transcode" + TRANSCODE_SSD="/mnt/cache/Temp_Storage/Emby/Transcodes/" + RAMDISK_WARN_GB=6.8 + RAMDISK_LOW_GB=5.5 + RAMDISK_SSD_MIN_GB=20 + TRANSCODE_MAX_AGE=20 + TRANSCODE_ORPHAN_AGE=30 + TRANSCODE_FLIP_WARN=3 TRANSCODE_OWNER="nobody:users" - TRANSCODE_CHMOD="755" # renamed from TRANSCODE_MODE to avoid ambiguity with manager mode - -# Operating mode — controls symlink routing behavior -# smart — auto-flips between ramdisk and SSD based on usage thresholds (default) -# ramdisk — always uses ramdisk, never flips to SSD regardless of usage -# useful when load is light and you want guaranteed ramdisk performance -# warns if usage exceeds threshold but does not flip -# ssd — always uses SSD, never uses ramdisk -# useful during ramdisk maintenance, testing, or after a flip issue -# switch to this mode to drain ramdisk sessions gracefully + TRANSCODE_CHMOD="755" TRANSCODE_MANAGER_MODE="smart" # smart | ramdisk | ssd + TRANSCODE_DAILY_LOG="/boot/config/transcode_daily.db" + TRANSCODE_LOG_RETENTION=90 # ━━━ Transcode Server Array ━━━ -# All media servers that share the ramdisk transcode space. -# transcode_manager.sh reads this array and queries each server's API for active sessions. -# Session display, storage detection, and threshold decisions cover ALL servers combined. -# -# One entry → works exactly as before — single server behavior -# Many entries → aggregates sessions from all servers, one threshold on total ramdisk usage -# -# Each server writes to its own subfolder inside transcoding-temp: -# /mnt/ram-transcode/transcoding-temp/ABC123/ ← Emby session -# /mnt/ram-transcode/transcoding-temp/XYZ789/ ← Jellyfin session -# They never touch each other's files — the ramdisk is shared scratch space. -# -# Docker Extra Parameters — each media server container that uses the ramdisk needs: -# --mount type=bind,source=/mnt/ram-transcode,target=/ext-ram-transcode,bind-propagation=shared -# The target path (/ext-ram-transcode) must match what's configured in each server's -# transcoding settings. Use a different target path per container if needed. -# -# Format: "ContainerName|URL|APIKey|Type" -# ContainerName — exact Docker container name (used for running check) -# URL — API base URL including port -# APIKey — server API key (Emby/Jellyfin token, Plex token etc.) -# Type — emby | jellyfin | plex (controls API endpoint format) -# -# The first entry is the primary server — used for container running check -# (TRANSCODE_CHECK_EMBY still applies to first entry). -# Additional entries are queried if their container is running. -# Entries with placeholder APIKey values are skipped automatically. -# -# ⚠️ Tdarr does NOT belong here — Tdarr encodes full files, not HLS segments. -# Large working files would fill the ramdisk rapidly and cause constant flips. -# Keep Tdarr on SSD. Use tdarr_cleanup.sh for Tdarr orphan management. +# All media servers sharing the ramdisk transcode space. +# Format: "ContainerName|URL|APIKey|Type" — Type: emby | jellyfin | plex +# Entries with placeholder API keys are skipped automatically. +# ⚠️ Tdarr does NOT belong here — keep Tdarr on SSD. TRANSCODE_SERVERS=( "${HOST1_EMBY_CONTAINER}|${HOST1_EMBY_URL}|${HOST1_EMBY_API_KEY}|emby" - # "${HOST2_EMBY_CONTAINER}|${HOST2_EMBY_URL}|${HOST2_EMBY_API_KEY}|emby" # his Emby temporarily - # "Jellyfin|http://localhost:8097|jellyfin-api-key|jellyfin" # test Jellyfin instance - # "Plex|http://localhost:32400|plex-token|plex" # Plex if needed + # "${HOST2_EMBY_CONTAINER}|${HOST2_EMBY_URL}|${HOST2_EMBY_API_KEY}|emby" + # "Jellyfin|http://localhost:8097|jellyfin-api-key|jellyfin" + # "Plex|http://localhost:32400|plex-token|plex" ) -# Container running check — still applies — skips threshold checks when no servers active -# Checked against the first entry in TRANSCODE_SERVERS automatically TRANSCODE_CHECK_EMBY=true - -# Daily transcode statistics log — read by weekly_health_digest.sh -# Tracks peak usage, flip count, session ratio, files cleaned per day -# Bounded to TRANSCODE_LOG_RETENTION days — auto-purges old entries - TRANSCODE_DAILY_LOG="/boot/config/transcode_daily.db" - TRANSCODE_LOG_RETENTION=90 - # ============================================================================================== # ── MONITORS ────────────────────────────────────────────────────────────────────────────────── # ============================================================================================== # ━━━ Certificate Monitor ━━━ -# Checks SSL cert expiry via direct openssl connection — no NPM dependency. -# Reads the actual cert the server is presenting — catches real-world issues API checks miss. -# Each domain and subdomain is a separate entry — they have independent certs. CERT_MONITOR_DOMAINS=( "Gmer4Lfe.com" "Gmer4Lfe.us" ) - CERT_WARN_DAYS=30 # notify warning when cert expires within this many days - CERT_CRIT_DAYS=7 # notify critical when cert expires within this many days - CERT_TIMEOUT=10 # seconds before openssl connection attempt gives up per domain + CERT_WARN_DAYS=30 + CERT_CRIT_DAYS=7 + CERT_TIMEOUT=10 # ━━━ Backup Verify ━━━ -# Verifies the rsync mirror is healthy by comparing random file checksums between servers. -# Uses existing SSH keys — no additional configuration needed beyond the share list. -# Leave BACKUP_VERIFY_SHARES empty to automatically use the local host's daily sync shares -# (HOST1_DAILY_SYNC_SHARES or HOST2_DAILY_SYNC_SHARES based on detect_hosts()). +# Leave empty to use HOST*_DAILY_SYNC_SHARES automatically. BACKUP_VERIFY_SHARES=( - # leave empty to use host-specific daily sync shares automatically + # leave empty to use daily sync shares automatically ) - BACKUP_VERIFY_SAMPLE=10 # number of files to randomly sample per share per run - BACKUP_VERIFY_MIN_SIZE=1M # skip files smaller than this — avoids tiny junk files + BACKUP_VERIFY_SAMPLE=10 + BACKUP_VERIFY_MIN_SIZE=1M # ━━━ SMART Health ━━━ -# Monitors drive SMART attributes — reads live from each drive, no persistent writes. -# Discovers all drives automatically via /dev/sd* and /dev/nvme* — no drive list needed. -# Add drives to SMART_IGNORE_DRIVES to skip specific drives (e.g. your unRAID boot USB). - SMART_TEMP_WARN=45 # degrees C — warn if drive temperature exceeds this - SMART_TEMP_CRIT=55 # degrees C — critical if drive temperature exceeds this + SMART_TEMP_WARN=45 + SMART_TEMP_CRIT=55 SMART_IGNORE_DRIVES=( - "sda" # boot USB — SMART not meaningful on flash drives + "sda" ) # ━━━ ZFS Memory Snapshot ━━━ -# Weekly ZFS pool health and memory diagnostic report — informational only, no action taken. -# system_watchdog.sh handles threshold-based intervention. -# Output written to ZFS_REPORT_LOG for historical review in addition to console output. -# Pools in ZFS_REPORT_IGNORE_POOLS are excluded from reporting — still monitored by unRAID. ZFS_REPORT_LOG="/var/log/zfs-weekly-health.log" - ZFS_REPORT_ARC_WARN_PCT=90 # warn in report if ARC utilization above this % - ZFS_REPORT_FREE_WARN_GB=10 # warn in report if free RAM drops below this GB - ZFS_REPORT_AVAIL_WARN_GB=20 # warn in report if available RAM drops below this GB - ZFS_REPORT_DOCKER_TOP=10 # number of top Docker memory users to show in report + ZFS_REPORT_ARC_WARN_PCT=90 + ZFS_REPORT_FREE_WARN_GB=10 + ZFS_REPORT_AVAIL_WARN_GB=20 + ZFS_REPORT_DOCKER_TOP=10 ZFS_REPORT_IGNORE_POOLS=( - # Pools excluded from health reporting — expected to run at high usage - # All pools still monitored by unRAID regardless of this list "disk10" "disk9" "disk8" @@ -1219,92 +905,59 @@ ZFS_REPORT_IGNORE_POOLS=( ) # ━━━ Bandwidth Monitor ━━━ -# Called by rsync.sh after each sync — one bounded write per run, minimal flash wear. -# Log format: YYYY-MM-DD|HH:MM|profile|duration_seconds|status — version-proof -# File stays bounded to BANDWIDTH_LOG_RETENTION days — old entries auto-purged on write. +# Called by rsync.sh after each sync — bounded write, minimal flash wear. BANDWIDTH_LOG="/boot/config/bandwidth_history.db" - BANDWIDTH_LOG_RETENTION=90 # days to keep — file never grows beyond ~90 lines - BANDWIDTH_WARN_GB=50 # flag in reports if a single sync transfer exceeds this GB + BANDWIDTH_LOG_RETENTION=90 + BANDWIDTH_WARN_GB=50 # ━━━ Health Digest ━━━ -# Aggregated system health summary — reads existing state files, no new writes to flash. -# Three profiles — switch by changing DIGEST_PROFILE, no cron changes needed: -# always — sends every run -# smart — sends only if findings worth reporting -# weekly — sends once per week on DIGEST_DAY only - DIGEST_PROFILE="weekly" # always | smart | weekly - DIGEST_DAY="Sunday" # must match date +%A output +# Reads existing state files — no new flash writes. +# Profiles: always | smart | weekly + DIGEST_PROFILE="weekly" + DIGEST_DAY="Sunday" + DIGEST_SMART_ON_WATCHDOG=true + DIGEST_SMART_ON_FAILOVER=true + DIGEST_SMART_ON_CERT_WARN=true + DIGEST_SMART_ON_BANDWIDTH=true -# Smart profile triggers — set true to send digest when this condition is found - DIGEST_SMART_ON_WATCHDOG=true # send if any watchdog strikes are active - DIGEST_SMART_ON_FAILOVER=true # send if failover state is not NORMAL - DIGEST_SMART_ON_CERT_WARN=true # send if any cert is under CERT_WARN_DAYS - DIGEST_SMART_ON_BANDWIDTH=true # send if any transfer exceeded BANDWIDTH_WARN_GB - -# ━━━ Critical Shares Maintenance ━━━ -# Controls container update behaviour in critical_shares_maintenance.sh -# Both local and remote containers are already stopped for the sync window -# Updates pull new images while containers are down — starts fresh on new version -# -# CRITICAL_SYNC_UPDATES — pull updates on LOCAL server during maintenance window -# CRITICAL_SYNC_UPDATES_REMOTE — pull updates on REMOTE server during maintenance window -# -# Both false → sync only, no updates (sync only, no updates) -# Both true → full maintenance both servers — recommended for Sunday window -# Toggle false temporarily to skip updates without changing the schedule - - CRITICAL_SYNC_UPDATES=true # pull container updates locally during maintenance - CRITICAL_SYNC_UPDATES_REMOTE=true # pull container updates on remote via SSH - -# ━━━ Emby ━━━ # ━━━ Emby Session Report ━━━ -# No persistent writes — queries fresh each run. -# Emby URL and API key pulled from HOST1/HOST2_EMBY_URL and HOST1/HOST2_EMBY_API_KEY -# defined in Host Configuration at the top of this file. - EMBY_REPORT_DAYS=7 # number of days to include in the report period - EMBY_REPORT_TOP_N=10 # number of top content items to show in report +# Weekly Emby usage statistics via API — no persistent writes. +# URL and API key from HOST1/HOST2_EMBY_URL and HOST1/HOST2_EMBY_API_KEY in Host Configuration. + EMBY_REPORT_DAYS=7 + EMBY_REPORT_TOP_N=10 # ============================================================================================== # ── SYSTEM WATCHDOG ─────────────────────────────────────────────────────────────────────────── # ============================================================================================== -# Last line of defense — reboots cleanly when system is about to become unstable. +# Continuous system health monitoring — last line of defense before a crash. +# Started by array_start.sh — runs until array stops. +# Re-sources Master.conf each cycle — config changes take effect on next cycle. # Strike system: sustained threshold hits trigger reboot — single spikes ignored. -# Reboot loop protection: shuts down instead if reboot limit hit in window. +# Reboot loop protection: shuts down instead if reboot limit hit in rolling window. +# Silent when healthy — logs only when a threshold is triggered. # ━━━ State Files ━━━ -# Strike counts reset on reboot — /tmp is correct (fresh start after each reboot) - SYS_WATCHDOG_STATE_FILE="/tmp/system_watchdog_state.db" -# Persistent container skip list — on /boot/ so it survives reboots -# Containers added here when docker_watchdog.sh exhausts all restart attempts -# Auto-clears when container is found running again after reboot or manual fix - SYS_WATCHDOG_FAILED_FILE="/boot/config/system_watchdog_failed.db" -# Reboot timestamp log — on /boot/ for reboot loop detection across reboots - SYS_WATCHDOG_REBOOT_LOG="/boot/config/system_watchdog_reboots.db" + SYS_WATCHDOG_STATE_FILE="/tmp/system_watchdog_state.db" # /tmp resets on reboot ✅ + SYS_WATCHDOG_FAILED_FILE="/boot/config/system_watchdog_failed.db" # survives reboots + SYS_WATCHDOG_REBOOT_LOG="/boot/config/system_watchdog_reboots.db" # reboot loop detection # ━━━ Strike and Reboot Loop Settings ━━━ -# Consecutive threshold hits required before triggering reboot SYS_WATCHDOG_STRIKE_LIMIT=2 -# Maximum reboots allowed within the window before shutting down instead -# A reboot loop means something fundamental is broken that rebooting is not fixing + SYSTEM_WATCHDOG_INTERVAL=300 # seconds between cycles (5min default) SYS_WATCHDOG_REBOOT_LIMIT=3 -# Window in hours — controls BOTH the reboot count window AND the rolling log purge -# Entries older than this many hours are automatically removed from the reboot log SYS_WATCHDOG_REBOOT_WINDOW_HRS=12 # ━━━ Thresholds ━━━ -# Set these at "I am about to become unstable" levels — not just "things are a bit high" - SYS_WATCHDOG_ROOTFS_PCT=95 # rootfs % — at 95% something is seriously wrong - SYS_WATCHDOG_LOG_PCT=95 # /var/log % — log spam filling the filesystem - SYS_WATCHDOG_MEM_GB=4 # free RAM GB — 4GB free on 128GB system is critical - SYS_WATCHDOG_ARC_PINNED_PCT=98 # ZFS ARC % of max before attempting reclaim - SYS_WATCHDOG_ARC_RELEASE_PCT=95 # ZFS ARC % after reclaim that still triggers reboot - SYS_WATCHDOG_LOAD_MULTIPLIER=3 # strike if load avg > cores x this multiplier - SYS_WATCHDOG_ZOMBIE_LIMIT=50 # zombie process count before strike - SYS_WATCHDOG_CPU_TEMP_MAX=95 # degrees C — adjust for your specific CPU tjmax + SYS_WATCHDOG_ROOTFS_PCT=95 + SYS_WATCHDOG_LOG_PCT=95 + SYS_WATCHDOG_MEM_GB=4 + SYS_WATCHDOG_ARC_PINNED_PCT=98 + SYS_WATCHDOG_ARC_RELEASE_PCT=95 + SYS_WATCHDOG_LOAD_MULTIPLIER=3 + SYS_WATCHDOG_ZOMBIE_LIMIT=50 + SYS_WATCHDOG_CPU_TEMP_MAX=95 # ━━━ Check Toggles ━━━ -# true = run this check on every watchdog cycle / false = skip entirely -# Disable checks not relevant to your hardware or that cause false positives SYS_WATCHDOG_CHECK_ROOTFS=true SYS_WATCHDOG_CHECK_LOG=true SYS_WATCHDOG_CHECK_RAM=true @@ -1312,17 +965,14 @@ ZFS_REPORT_IGNORE_POOLS=( SYS_WATCHDOG_CHECK_CPU_TEMP=true SYS_WATCHDOG_CHECK_LOAD=false # disabled — load spikes during transcoding are normal SYS_WATCHDOG_CHECK_ZOMBIES=true - SYS_WATCHDOG_CHECK_CONTAINERS=true # checks persistent skip list from docker_watchdog.sh + SYS_WATCHDOG_CHECK_CONTAINERS=true SYS_WATCHDOG_CHECK_DOCKER_DAEMON=true # ━━━ Abort Toggles ━━━ -# Controls whether certain conditions prevent a reboot from happening. -# true = abort reboot if this condition is active (conservative — default) -# false = reboot anyway regardless of this condition (aggressive) -# Philosophy: a graceful reboot before crash is always better than a hard crash mid-operation - SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY=true # unhealthy pool + reboot risks data loss - SYS_WATCHDOG_ABORT_ON_PARITY=false # aborting parity check beats crashing mid-check - SYS_WATCHDOG_ABORT_ON_MOVER=false # aborting mover beats crashing mid-move +# true = abort reboot if condition active / false = reboot anyway + SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY=true + SYS_WATCHDOG_ABORT_ON_PARITY=false + SYS_WATCHDOG_ABORT_ON_MOVER=false # ============================================================================================== # ──────────────────────── End Of User Variables ─────────────────────────────────────────────── diff --git a/Orchestrators/README-Orchestrators.md b/Orchestrators/README-Orchestrators.md index f88821a..f0e4f22 100644 --- a/Orchestrators/README-Orchestrators.md +++ b/Orchestrators/README-Orchestrators.md @@ -13,14 +13,68 @@ Without orchestrators, each script runs independently on its own schedule. This - **Race conditions** — two scripts running simultaneously on the same data - **Order dependency failures** — media cleaner runs before permissions, finds wrong ownership - **No combined summary** — 6 separate notifications instead of one clean report -- **Scheduling complexity** — 6+ cron entries instead of one +- **Scheduling complexity** — many cron entries instead of a few clean ones Orchestrators solve this by making a set of related scripts into a single scheduled unit with a defined execution order and a unified summary. --- +## The Orchestrator Model + +The ecosystem is designed so the User Scripts plugin contains only a small number of entries — each one an orchestrator that owns a domain: + +``` +At Startup of Array: + array_start.sh ← single entry, launches everything + +Cron: + transcode_management.sh ← */3 * * * * + arrs_failed_stalled_recovery.sh ← 0 */6 * * * + rsync.sh ... emby-failover ← */30 * * * * + daily_sync_maintenance.sh ← 0 1 * * * + weekly_sync_maintenance.sh ← 30 2 * * 0 + weekly_health_digest.sh ← Saturday morning + +Manual only: + failover_test.sh, emby_database_repair.sh, repair tools +``` + +All job lists are configured in the `ORCHESTRATORS` section of `Master.conf`. No changes to orchestrator scripts needed when adding or removing jobs. + +--- + ## Scripts +### `array_start.sh` + +Single entry point for the User Scripts "At Startup of Array" schedule. Launches all array-start scripts in order — each as a background process. + +```bash +# Scheduled as: At Startup of Array +/mnt/user/appdata/unraid_scripts/Orchestrators/array_start.sh +``` + +One-shot scripts (ramdisk, syslog filter, php-fpm, network connect) run and exit naturally. Continuous scripts (system watchdog, docker watchdog, failover) run until the array stops. + +**Configuration:** + +```bash +# Master.conf — ORCHESTRATORS section +ARRAY_START_SCRIPTS=( + "unRAID_Essentials/ramdisk_setup.sh" # creates ramdisk before Emby starts + "unRAID_Essentials/docker_syslog_filter.sh" # suppress veth log noise + "unRAID_Essentials/php_fpm_max_children.sh" # WebGUI tuning + "Docker_Essentials/docker_network_connect.sh" # connect containers to extra networks + "unRAID_Essentials/system_watchdog.sh" # continuous system health monitor + "Docker_Essentials/docker_watchdog.sh" # continuous container health monitor + "Failover/failover.sh" # continuous mutual failover +) +``` + +Add or remove scripts from `ARRAY_START_SCRIPTS` — no changes to `array_start.sh` needed. Order matters — ramdisk before Emby, network before watchdogs, watchdogs before failover. + +--- + ### `transcode_management.sh` Runs `transcode_cleanup.sh` then `transcode_manager.sh` in the correct order every 3 minutes. Replaces two separate cron entries with one. @@ -32,13 +86,13 @@ Runs `transcode_cleanup.sh` then `transcode_manager.sh` in the correct order eve **Why cleanup must run before manager:** -If the manager runs first it may see inflated ramdisk usage from stale segment files left by ended sessions — and trigger an unnecessary flip to SSD. Cleanup runs first to clear those files, then the manager makes its threshold decision based on real active session usage. +If the manager runs first it sees inflated ramdisk usage from stale segment files left by ended sessions — and triggers an unnecessary flip to SSD. Cleanup runs first to clear those files, then the manager decides based on real active session usage. ``` Without correct order: Manager checks usage → 6.8GB (includes stale files) → flips to SSD Cleanup runs → removes stale files → actual usage 2.1GB - Manager was wrong — unnecessary flip + Unnecessary flip — sessions now on SSD With correct order: Cleanup runs → removes stale files → actual usage 2.1GB @@ -47,7 +101,7 @@ With correct order: **Daily statistics tracking:** -Every cycle `transcode_management.sh` records stats to `/boot/config/transcode_daily.db`: +Every cycle `transcode_management.sh` records stats to `TRANSCODE_DAILY_LOG`: - Peak ramdisk usage for the day - Total flip count for the day - Ramdisk vs SSD session counts @@ -57,193 +111,114 @@ Every cycle `transcode_management.sh` records stats to `/boot/config/transcode_d --- -### `media_shares_sync.sh` +### `arrs_failed_stalled_recovery.sh` -Syncs each server's source-of-truth media shares to the remote server sequentially. Each server only pushes the shares it owns — direction and share list are automatic based on which server is running the script. +Automatically detects and recovers from failed imports and stalled downloads across Sonarr, Radarr, and Lidarr. Blocklists the bad release and triggers a new search — hands-free recovery while you sleep. + +```bash +# Scheduled as: 0 */6 * * * (every 6 hours) +/mnt/user/appdata/unraid_scripts/Media/arrs_failed_stalled_recovery.sh +``` + +Targets four problem types: `importFailed`, `importPending`, `error` status, and `stalled` downloads. Items newer than `ARR_IMPORT_RECOVERY_AGE` (6 hours) are skipped — gives the arr time to retry on its own first. + +**API versions:** Sonarr v4 → `/api/v3/` — Radarr v6 → `/api/v3/` — Lidarr v3 → `/api/v1/` + +Lidarr runs on HOST1 only — exits cleanly on HOST2. + +**Configuration:** + +```bash +# Master.conf — MEDIA section +ARR_IMPORT_RECOVERY_AGE=6 # hours — skip items newer than this + +HOST1_SONARR_RECOVERY=true +HOST1_RADARR_RECOVERY=true +HOST1_LIDARR_RECOVERY=true +HOST2_SONARR_RECOVERY=true +HOST2_RADARR_RECOVERY=true +``` + +--- + +### `daily_sync_maintenance.sh` + +Full daily maintenance window orchestrator — git pull, media share sync, media management, and docker daily restarts. All driven by `Master.conf` arrays. ```bash # Scheduled as: 0 1 * * * (1am daily — on both servers) -/mnt/user/appdata/unraid_scripts/Orchestrators/media_shares_sync.sh +/mnt/user/appdata/unraid_scripts/Orchestrators/daily_sync_maintenance.sh +``` + +**Execution order:** + +``` +1. Pre-sync jobs (DAILY_MAINTENANCE_SCRIPTS — git pull first): + git_pull_execute.sh ← always runs first — pulls latest scripts + +2. Media share sync: + HOST*_DAILY_SYNC_SHARES ← each server pushes its own truth shares + HOST*_PERSONAL_SHARES ← personal encrypted shares appended after + +3. Post-sync jobs (DAILY_MAINTENANCE_SCRIPTS — remaining): + media_management.sh ← permissions + cleaners + arr cleanup + docker_daily_restart.sh ← daily container restarts ``` **Bidirectional — same script, correct direction automatically:** ``` -HOST1 runs media_shares_sync.sh → pushes HOST1_DAILY_SYNC_SHARES → TO HOST2 - Movies, Tv_Shows, Music, Books etc. — HOST1 is source of truth +HOST1 runs daily_sync_maintenance.sh: + git pull → sync HOST1_DAILY_SYNC_SHARES → TO HOST2 → media_management → docker restart -HOST2 runs media_shares_sync.sh → pushes HOST2_DAILY_SYNC_SHARES → TO HOST1 - Anime_Shows, Anime_Movies — HOST2 is source of truth +HOST2 runs daily_sync_maintenance.sh: + git pull → sync HOST2_DAILY_SYNC_SHARES → TO HOST1 → media_management → docker restart ``` -`detect_hosts()` determines which server is local at runtime and selects the correct share list. No script changes needed to reconfigure who syncs what — only `Master.conf` changes required. - -**What it does:** -1. Detects local server via `detect_hosts()` — determines HOST1 or HOST2 -2. Resolves remote Tailscale IP -3. Runs a single pre-flight check — connectivity + remote rootfs -4. Builds share list from `HOST1_DAILY_SYNC_SHARES` or `HOST2_DAILY_SYNC_SHARES` -5. Appends personal shares (`HOST1_PERSONAL_SHARES` or `HOST2_PERSONAL_SHARES`) -6. Calls `Rsync/rsync.sh` for each share -7. Tracks pass/fail and duration per share -8. Reports a combined summary +`detect_hosts()` determines which server is local at runtime and selects the correct share list. No script changes needed — only `Master.conf` changes required. **Why one pre-flight check upfront:** -Connectivity and rootfs are checked once before the loop starts — not once per share. If the remote is unreachable or the rootfs is nearly full, the whole run fails fast. Individual share existence and disk checks still run per-share inside `rsync.sh`. + +Connectivity and rootfs are checked once before the loop starts — not once per share. If the remote is unreachable or rootfs is nearly full, the whole run fails fast. Individual share checks still run per-share inside `rsync.sh`. **Configuration:** + ```bash -# Master.conf — per-host share lists -# HOST1 truth shares — pushed from HOST1 to HOST2 nightly +# Master.conf — ORCHESTRATORS section + +DAILY_MAINTENANCE_SCRIPTS=( + "git_pull_execute.sh" # always first + "Docker_Essentials/docker_daily_restart.sh" # after sync and media jobs +) + +# Media jobs run between sync and docker restart +# Permissions first, cleaners second, arr cleanup last +MEDIA_MANAGEMENT_JOBS=( + "Media/media_shares_permissions.sh" # permissions — everything depends on this + "Media/media_cleaner.sh anime" # clean junk before arr scripts scan + "Media/media_cleaner.sh media" + "Media/lidarr_cleanup.sh" # arr cleanup last — depends on clean folders + "Media/sonarr_cleanup.sh" + "Media/radarr_cleanup.sh" +) + HOST1_DAILY_SYNC_SHARES=( /mnt/user/Movies /mnt/user/Tv_Shows /mnt/user/Music - # ... + # all HOST1-owned shares ) -# HOST2 truth shares — pushed from HOST2 to HOST1 nightly HOST2_DAILY_SYNC_SHARES=( /mnt/user/Anime_Shows /mnt/user/Anime_Movies - # ... ) ``` -These shares use global rsync defaults — no profile needed. For shares requiring custom bandwidth limits, container stops, or different rsync options, create a named profile in the Rsync profile system and call `rsync.sh` directly on a separate schedule instead. - -**Relationship to failover writeback:** - -The same share lists are used by `failover.sh` for Tier 4 writeback — but in the opposite direction. If HOST1 was down for 18hr+ and HOST2's arrs downloaded new content, Tier 4 writeback pushes `HOST1_DAILY_SYNC_SHARES` FROM HOST2 back TO HOST1. No duplicate configuration needed. - -**Example output:** -``` -━━━ 🔄 Daily Sync Starting — 2026-04-14 01:00:00 ━━━ -📋 Shares: 11 - -━━━ [1/11] Movies ━━━ -...rsync output... -✅ Movies — 4m32s - -━━━ [2/11] Tv_Shows ━━━ -... -━━━━━ 📋 DAILY SYNC SUMMARY ━━━━━ -✅ Pass: 10 ❌ Fail: 1 -⏱️ Duration: 47m12s -❌ Failed: Anime_Shows-Old -``` - ---- - -### `critical_shares_full_sync.sh` - -Runs a clean nightly sync for Emby and the auth stack (Critical-Data) with containers stopped. This is the companion to the hourly dirty sync — it provides a fully consistent state on HOST2 once per night. +**Adding a media job:** ```bash -# Scheduled as: 30 2 * * 0 (2:30am Sunday — weekly clean sync) -/mnt/user/appdata/unraid_scripts/Orchestrators/critical_shares_full_sync.sh -``` - -**Why two Emby syncs:** - -The hourly dirty sync runs with Emby up — WAL files excluded, watch states pushed continuously. This means HOST2 is never more than an hour behind on watch state. But it's not a clean database snapshot. - -The nightly clean sync stops Emby, syncs the full clean database state, then restarts. HOST2 gets a fully consistent Emby state every night. The two syncs work together: - -``` -Hourly dirty sync (Emby running): - users.db, library.db, authentication.db, config/ - WAL excluded — safe mid-write - HOST2 always within 1hr of HOST1 on watch state - -Nightly clean sync (Emby stopped): - Full clean snapshot — all databases flushed - No WAL files in flight - HOST2 gets gold-standard state once per night -``` - -**Why clean auth sync matters:** - -The auth stack runs warm on both servers continuously. During normal operation HOST2's auth stack serves its own domain — it doesn't receive dirty updates from HOST1. The nightly clean sync is the only time auth state propagates. - -This means: -- New user added on HOST1 → propagates to HOST2 overnight automatically -- Proxy rule changes → propagated overnight -- No manual intervention needed for most auth changes - -For users who just want failover to work — this script handles it. No thinking required about dirty writes, WAL files, or when to sync. - -**What it syncs:** - -``` -Emby appdata: - users.db, library.db, authentication.db, config/ - Containers stopped → clean flush → safe copy - -Critical-Data (auth stack): - NPM proxy rules + SSL certs - Authelia config + database - Mariadb-Authelia data - Redis-Authelia session store - LLDAP users and groups database - All auth containers stopped → clean databases → safe copy - Authelia delayed start on restart — Mariadb + Redis must be ready first -``` - -**What it excludes (per rsync profile):** - -``` -Emby: logs, transcodes, cache, metadata, *.db-wal, *.db-shm -Auth: logs, *.tmp, nginx/temp, nginx/cache, notification.txt -``` - -### `media_management.sh` - -Runs all media maintenance scripts sequentially in the order defined in `Master.conf`. Scheduled once daily, typically after the nightly sync. - -```bash -# Scheduled as: 0 2 * * * (2am daily — after media_shares_sync.sh) -/mnt/user/appdata/unraid_scripts/Orchestrators/media_management.sh -``` - -**What it does:** -1. Reads `MEDIA_MAINTENANCE_JOBS` from `Master.conf` -2. Runs each job in order — script path + optional argument -3. Tracks pass/fail per job -4. Reports a combined summary -5. A failure in one job does not stop the others - -**Why order matters:** - -``` -1. media_shares_permissions.sh ← permissions first — everything else depends on correct ownership -2. media_cleaner.sh anime ← clean junk before arr scripts scan -3. media_cleaner.sh media ← same -4. lidarr_cleanup.sh ← arr cleanup last — depends on clean folders -5. sonarr_cleanup.sh -6. radarr_cleanup.sh -``` - -If arr cleanup runs before permissions, it may fail to delete files it doesn't have access to. If it runs before the cleaner, it finds junk files mixed in with real content. The order is intentional. - -**Configuration:** -```bash -# Master.conf — add, remove, or reorder jobs here -# Format: "folder/script.sh optional_argument" -MEDIA_MAINTENANCE_JOBS=( - "Media/media_shares_permissions.sh" - "Media/media_cleaner.sh anime" - "Media/media_cleaner.sh media" - "Media/lidarr_cleanup.sh" - "Media/sonarr_cleanup.sh" - "Media/radarr_cleanup.sh" -) -``` - -**Adding a new job:** -```bash -# Add a line to MEDIA_MAINTENANCE_JOBS — no script changes needed -MEDIA_MAINTENANCE_JOBS=( +MEDIA_MANAGEMENT_JOBS=( "Media/media_shares_permissions.sh" "Media/media_cleaner.sh anime" "Media/media_cleaner.sh media" @@ -254,10 +229,10 @@ MEDIA_MAINTENANCE_JOBS=( ) ``` -**Disabling a job temporarily:** +**Disabling a media job temporarily:** + ```bash -# Comment it out — easy to re-enable -MEDIA_MAINTENANCE_JOBS=( +MEDIA_MANAGEMENT_JOBS=( "Media/media_shares_permissions.sh" # "Media/media_cleaner.sh anime" # ← disabled, not deleted "Media/media_cleaner.sh media" @@ -267,31 +242,152 @@ MEDIA_MAINTENANCE_JOBS=( ) ``` -**--dry-run support:** -`media_management.sh --dry-run` passes `--dry-run` through to every child script. All scripts report what they would do without making changes. Useful for testing a new job before adding it to the live schedule. +**Relationship to failover writeback:** + +The same share lists are used by `failover.sh` for Tier 4 writeback — but in the opposite direction. If HOST1 was down for 24hr+ and HOST2's arrs accumulated content, writeback pushes `HOST1_DAILY_SYNC_SHARES` FROM HOST2 BACK TO HOST1. No duplicate configuration needed. + +--- + +### `weekly_sync_maintenance.sh` + +Weekly maintenance window orchestrator — critical appdata clean sync, container updates, and weekly docker restarts. Runs Sunday 2:30am, fits before the 3am network reboot. ```bash -/mnt/user/appdata/unraid_scripts/Orchestrators/media_management.sh --dry-run +# Scheduled as: 30 2 * * 0 (Sunday 2:30am) +/mnt/user/appdata/unraid_scripts/Orchestrators/weekly_sync_maintenance.sh +``` + +**Execution order:** + +``` +1. Stop local containers — auth stack + Emby stopped locally +2. Stop remote containers — auth stack + Emby stopped remotely via SSH +3. Pull updates locally — if CRITICAL_SYNC_UPDATES=true +4. Pull updates remotely — if CRITICAL_SYNC_UPDATES_REMOTE=true +5. rsync WEEKLY_SYNC_JOBS — Emby + Critical-Data clean sync +6. Start remote containers — starts on new images, correct order +7. Start local containers — starts on new images, correct order + +Post-sync jobs (WEEKLY_MAINTENANCE_SCRIPTS): +8. docker_weekly_restart.sh +``` + +**Why two Emby syncs:** + +The emby-failover dirty sync runs every 30-60 minutes with Emby running — WAL files excluded, watch states and library pushed continuously. HOST2 stays current on what users are watching. But it is not a clean database snapshot. + +The weekly clean sync stops Emby on both sides, checkpoints the WAL, and pushes a full consistent mirror. HOST2 gets a gold-standard Emby state once per week. + +``` +emby-failover every 30-60min (Emby running): + users.db, library.db, authentication.db, config/ + WAL excluded — safe mid-write + HOST2 always within 30-60min of HOST1 on watch state + +weekly clean sync Sunday 2:30am (Emby stopped): + Full clean mirror — all databases flushed + metadata, plugins, config all included + ~30s downtime — both Emby instances down during sync only + Cache stays warm on HOST2 all week — only reset Sunday +``` + +**Why weekly instead of nightly:** + +Emby builds a warm image cache on HOST2 naturally throughout the week. Syncing nightly resets this cache — users experience slow image loads every morning. Weekly sync lets the cache stay warm for 6 days and only resets on Sunday night when most users are asleep. + +**What it syncs:** + +``` +WEEKLY_SYNC_JOBS (configurable in Master.conf): + /mnt/user/Media_Server/Emby ← emby profile — full clean mirror + /mnt/user/appdata-Failover/Critical-Data ← critical-data profile — auth stack + +Emby excludes: logs, transcodes, cache, crash* +Auth excludes: logs, *.tmp, nginx/temp, nginx/cache, notification.txt +``` + +**Container update window:** + +Containers are already stopped for the sync — container image updates pull at zero extra downtime. Both servers start on the same new image version after the sync. + +```bash +# Master.conf toggles +CRITICAL_SYNC_UPDATES=true # pull updates locally +CRITICAL_SYNC_UPDATES_REMOTE=true # pull updates on remote via SSH + +# Toggle false to skip updates without changing the schedule +CRITICAL_SYNC_UPDATES=false +``` + +**Why auth stack matters:** + +The auth stack (Authelia, NPM, Mariadb, Redis, LLDAP) runs warm on both servers. During normal operation HOST2 serves its own domain independently. The weekly clean sync is the only time auth state propagates from HOST1 to HOST2. + +- New user added on HOST1 → propagates to HOST2 on Sunday automatically +- Proxy rule changes → propagated Sunday +- No manual intervention needed for routine auth changes + +**Sunday maintenance window:** + +``` +2:30am weekly_sync_maintenance.sh ← clean sync + updates (~3-5min) +2:50am CA Auto Update plugin ← plugin updates +2:55am CA container updates ← docker container updates +3:00am Network reboot ← router/switch restart + +Everything comes back clean: + Network fresh, Emby updated, auth stack updated + All in one maintenance window while users sleep +``` + +**Configuration:** + +```bash +# Master.conf — ORCHESTRATORS section + +WEEKLY_SYNC_JOBS=( + "/mnt/user/Media_Server/Emby" + "/mnt/user/appdata-Failover/Critical-Data" +) + +WEEKLY_MAINTENANCE_SCRIPTS=( + "Docker_Essentials/docker_weekly_restart.sh" +) + +CRITICAL_SYNC_UPDATES=true +CRITICAL_SYNC_UPDATES_REMOTE=true +``` + +--- + +### `media_management.sh` + +Runs all media maintenance scripts sequentially in the order defined in `Master.conf`. Absorbed into `daily_sync_maintenance.sh` via `MEDIA_MANAGEMENT_JOBS` — not scheduled separately. Available for manual runs. + +```bash +# Manual use only — called automatically by daily_sync_maintenance.sh +bash /mnt/user/appdata/unraid_scripts/Orchestrators/media_management.sh --dry-run +bash /mnt/user/appdata/unraid_scripts/Orchestrators/media_management.sh ``` --- ## The Orchestrator Pattern -Both orchestrators follow the same pattern. This is by design — any script that needs to coordinate multiple operations should follow it: +All orchestrators follow the same pattern: ``` -1. Setup — validate config, detect hosts if needed +1. Setup — validate config, detect hosts if needed, acquire lock 2. Pre-flight — fail fast checks before doing any work 3. Job loop — run each job, track pass/fail, continue on failure 4. Summary — one clean report of all results 5. Notification — one notification per run, not one per job ``` -This pattern means: -- **Consistent output** — every orchestrator looks the same in the logs +This means: +- **Consistent output** — every orchestrator looks the same in logs - **No silent failures** — pass/fail tracked per job, reported in summary -- **Single notification** — one bell ring, not six +- **Single notification** — one bell ring per run - **Resilient** — one job failing doesn't stop the rest --- @@ -299,23 +395,39 @@ This pattern means: ## Scheduling ```bash -# Recommended schedule -*/3 * * * * transcode_management.sh # cleanup then manager — every 3 minutes -0 1 * * * media_shares_sync.sh # 1am — media shares to remote -0 2 * * * media_management.sh # 2am — permissions, cleaners, arr cleanup -30 2 * * 0 critical_shares_full_sync.sh # 2:30am Sunday — clean Emby + auth stack +# At Startup of Array +array_start.sh # single entry — launches all startup scripts + +# Every 3 minutes +*/3 * * * * transcode_management.sh + +# Every 6 hours +0 */6 * * * arrs_failed_stalled_recovery.sh + +# Every 30-60 minutes +*/30 * * * * rsync.sh /mnt/user/Media_Server/Emby --profile=emby-failover + +# Daily 1am — full daily maintenance window: +# git pull → media sync → permissions → cleaners → arr cleanup → docker restart +0 1 * * * daily_sync_maintenance.sh + +# Weekly — Sunday morning +30 2 * * 0 weekly_sync_maintenance.sh # clean sync + updates + docker weekly restart +50 2 * * 0 CA plugin update +55 2 * * 0 CA container updates ``` -`media_shares_sync.sh` and `media_management.sh` run nightly — media shares and maintenance. `critical_shares_full_sync.sh` runs weekly on Sunday — it stops Emby and the auth stack for a clean consistent sync. Running it weekly instead of nightly lets Emby's image cache stay warm on HOST2 throughout the week. The emby-failover dirty sync handles watch states, library structure, and auth every 30-60 minutes — the weekly clean sync covers metadata, plugins, and a full database flush. +`daily_sync_maintenance.sh` owns the entire daily window — git pull, media sync, permissions, cleaners, arr cleanup, and docker restarts in one scheduled run. Everything configured in `Master.conf` via `DAILY_MAINTENANCE_SCRIPTS` and `MEDIA_MANAGEMENT_JOBS`. --- ## Adding a New Orchestrator -If you find yourself running 3 or more related scripts on the same schedule, consider wrapping them in a new orchestrator. The pattern is simple: +If you find yourself running 3 or more related scripts on the same schedule, consider wrapping them in a new orchestrator. Model it directly on `media_management.sh` which handles dry-run passthrough, status display, pass/fail tracking and summary reporting. + +Minimal skeleton: ```bash -# Minimal orchestrator skeleton JOBS=( "Folder/script1.sh" "Folder/script2.sh arg" @@ -327,13 +439,11 @@ FAIL=() for JOB in "${JOBS[@]}"; do SCRIPT=$(echo "$JOB" | cut -d' ' -f1) ARG=$(echo "$JOB" | cut -d' ' -f2-) - + if bash "$ECOSYSTEM_ROOT/$SCRIPT" $ARG; then PASS+=("$SCRIPT") else FAIL+=("$SCRIPT") fi done -``` - -Better yet — model it directly on `media_management.sh` which already handles dry-run passthrough, status display, pass/fail tracking and summary reporting. \ No newline at end of file +``` \ No newline at end of file diff --git a/Orchestrators/array_start.sh b/Orchestrators/array_start.sh new file mode 100644 index 0000000..26cd311 --- /dev/null +++ b/Orchestrators/array_start.sh @@ -0,0 +1,106 @@ +#!/bin/bash +# ----------------------------------------------------------------------------------------------- +# --------------------------------- Array Start Orchestrator ----------------------------------- +# ----------------------------------------------------------------------------------------------- +# Launches all scripts configured in ARRAY_START_SCRIPTS when the unRAID array comes online. +# Set this script to run at "Startup of Array" in the User Scripts plugin. +# +# Each script is launched as a background process: +# One-shot scripts (ramdisk_setup, syslog_filter etc.) run and exit naturally +# Continuous scripts (system_watchdog, docker_watchdog, failover) run until array stops +# +# Scripts are launched in the order defined in ARRAY_START_SCRIPTS in Master.conf. +# Order matters — ramdisk before Emby, network before watchdogs, watchdogs before failover. +# +# To add or remove a script: edit ARRAY_START_SCRIPTS in Master.conf. +# No changes to this script needed. +# +# Logs: each script logs its own output independently. +# This orchestrator exits after launching all scripts — unRAID sees it complete normally. +# ----------------------------------------------------------------------------------------------- + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "$SCRIPT_DIR/Master.conf" +source "$SCRIPT_DIR/common.sh" + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_GEAR Setup ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_GEAR Array Start — $(date '+%Y-%m-%d %H:%M:%S') ━━━" + +if [[ "$EUID" -ne 0 ]]; then + error "Must be run as root" + exit 1 +fi + +success "Running as root" + +SCRIPT_COUNT=${#ARRAY_START_SCRIPTS[@]} +info "Launching $SCRIPT_COUNT script(s)..." +echo "" + +LAUNCHED=0 +FAILED=0 + +for relative_path in "${ARRAY_START_SCRIPTS[@]}"; do + [[ -z "$relative_path" ]] && continue + + SCRIPT_PATH="$SCRIPT_DIR/$relative_path" + SCRIPT_NAME=$(basename "$SCRIPT_PATH") + + if [[ ! -f "$SCRIPT_PATH" ]]; then + error "$SCRIPT_NAME — not found at $SCRIPT_PATH" + ((FAILED++)) + continue + fi + + if [[ ! -x "$SCRIPT_PATH" ]]; then + error "$SCRIPT_NAME — not executable" + ((FAILED++)) + continue + fi + + info "$ICON_START Launching $SCRIPT_NAME..." + bash "$SCRIPT_PATH" & + PID=$! + + # Brief pause to let script initialize and catch immediate failures + sleep 1 + + if kill -0 "$PID" 2>/dev/null; then + success "$SCRIPT_NAME — running (PID $PID)" + ((LAUNCHED++)) + else + # Script exited — check if it was a one-shot (exit 0) or a failure + wait "$PID" + EXIT_CODE=$? + if [[ "$EXIT_CODE" -eq 0 ]]; then + success "$SCRIPT_NAME — completed (one-shot)" + ((LAUNCHED++)) + else + error "$SCRIPT_NAME — exited with code $EXIT_CODE" + ((FAILED++)) + fi + fi + +done + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_SUMMARY Summary ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━━━ $ICON_SUMMARY ARRAY START SUMMARY ━━━━━" +echo "$ICON_SUCCESS Launched: $LAUNCHED" +echo "$ICON_ERROR Failed: $FAILED" +echo "$ICON_TIME Time: $(date '+%H:%M:%S')" +echo "" + +if [[ "$FAILED" -gt 0 ]]; then + echo "$ICON_WARN Status: $FAILED script(s) failed to launch — check logs" + notify "Array start on $(hostname) — $FAILED script(s) failed to launch" "Array Start" "warning" +else + echo "$ICON_DONE Status: $ICON_SUCCESS All scripts launched" +fi +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" \ No newline at end of file diff --git a/Orchestrators/daily_sync_maintenance.sh b/Orchestrators/daily_sync_maintenance.sh new file mode 100644 index 0000000..82b91ab --- /dev/null +++ b/Orchestrators/daily_sync_maintenance.sh @@ -0,0 +1,284 @@ +#!/bin/bash +# ----------------------------------------------------------------------------------------------- +# --------------------------------- Daily Sync Maintenance ------------------------------------ +# ----------------------------------------------------------------------------------------------- +# Daily orchestrator — runs the full daily maintenance window in the correct order. +# +# What it does: +# 1. Iterates DAILY_MAINTENANCE_SCRIPTS — runs git pull first, then additional jobs +# 2. Syncs all media shares in the correct direction for the local server +# 3. Additional scripts in DAILY_MAINTENANCE_SCRIPTS run after the sync completes +# +# Media share sync: +# Each server pushes only the shares it owns (source of truth) — direction is automatic. +# HOST1 pushes: Movies, Tv_Shows, Music, Books etc. → HOST2 +# HOST2 pushes: Anime_Shows, Anime_Movies → HOST1 +# Personal encrypted shares synced after media shares. +# detect_hosts() determines which server is running — no script changes needed. +# Share lists configured in Master.conf ORCHESTRATORS section. +# +# Per-share rsync handled by rsync.sh — this script tracks pass/fail and total time. +# All job lists configured in Master.conf — no script changes needed to add or remove jobs. +# Schedule: 0 1 * * * (1am daily — configured in User Scripts plugin) +# ----------------------------------------------------------------------------------------------- + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "$SCRIPT_DIR/../Master.conf" +source "$SCRIPT_DIR/../common.sh" + +RSYNC_SCRIPT="$SCRIPT_DIR/../Rsync/rsync.sh" +SCRIPTS_ROOT="$SCRIPT_DIR/.." + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_GEAR Setup ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_GEAR Setup ━━━" + +detect_hosts +resolve_remote_ip + +acquire_lock + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_SHIELD Pre-flight Checks ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_SHIELD Pre-flight Checks ━━━" + +check_connectivity +check_remote_rootfs + +WINDOW_START=$(date +%s) +JOB_PASS=() +JOB_FAIL=() + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_GIT Pre-sync Jobs ━━━ +# git_pull_execute.sh runs first — pulls latest scripts before anything else runs +# Identified by script name — all other DAILY_MAINTENANCE_SCRIPTS run after sync +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_SYNC Pre-sync Jobs ━━━" + +PRE_SYNC_SCRIPTS=() +POST_SYNC_SCRIPTS=() + +for script_entry in "${DAILY_MAINTENANCE_SCRIPTS[@]}"; do + [[ -z "$script_entry" ]] && continue + script_name=$(basename "${script_entry%% *}") + if [[ "$script_name" == "git_pull_execute.sh" ]]; then + PRE_SYNC_SCRIPTS+=("$script_entry") + else + POST_SYNC_SCRIPTS+=("$script_entry") + fi +done + +for script_entry in "${PRE_SYNC_SCRIPTS[@]}"; do + [[ -z "$script_entry" ]] && continue + script_args=($script_entry) + script_path="$SCRIPTS_ROOT/${script_args[0]}" + script_name=$(basename "${script_args[0]}") + extra_args=("${script_args[@]:1}") + + echo "" + info "$ICON_START Running: $script_name" + + if [[ ! -f "$script_path" ]]; then + error "$script_name — not found at $script_path" + JOB_FAIL+=("$script_name") + continue + fi + + if bash "$script_path" "${extra_args[@]}"; then + success "$script_name — done" + JOB_PASS+=("$script_name") + else + error "$script_name — failed" + JOB_FAIL+=("$script_name") + fi +done + +# ----------------------------------------------------------------------------------------------- +# Build share list — host-specific truth shares + personal shares +# ----------------------------------------------------------------------------------------------- +PASS=() +FAIL=() +SHARE_TIMES=() +TOTAL_START=$(date +%s) + +ALL_SHARES=() + +if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then + for share in "${HOST1_DAILY_SYNC_SHARES[@]}"; do + [[ -n "$share" ]] && ALL_SHARES+=("$share") + done + for share in "${HOST1_PERSONAL_SHARES[@]}"; do + [[ -n "$share" ]] && ALL_SHARES+=("$share") + done +elif [[ "$LOCAL_SERVER_NAME" == "$HOST2" ]]; then + for share in "${HOST2_DAILY_SYNC_SHARES[@]}"; do + [[ -n "$share" ]] && ALL_SHARES+=("$share") + done + for share in "${HOST2_PERSONAL_SHARES[@]}"; do + [[ -n "$share" ]] && ALL_SHARES+=("$share") + done +fi + +SHARE_COUNT=${#ALL_SHARES[@]} + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_SYNC Media Share Sync ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_SYNC Media Share Sync — $(date '+%Y-%m-%d %H:%M:%S') ━━━" +echo "$ICON_SUMMARY Shares: $SHARE_COUNT" +echo "" + +SHARE_INDEX=0 + +for SHARE in "${ALL_SHARES[@]}"; do + SHARE_INDEX=$((SHARE_INDEX + 1)) + SHARE_NAME=$(basename "$SHARE") + SHARE_START=$(date +%s) + + echo "━━━ $ICON_SYNC Share $SHARE_INDEX of $SHARE_COUNT: $SHARE_NAME ━━━" + + if bash "$RSYNC_SCRIPT" "$SHARE"; then + SHARE_END=$(date +%s) + SHARE_TIMES+=("$SHARE_NAME:$((SHARE_END - SHARE_START))") + PASS+=("$SHARE_NAME") + echo "$ICON_DONE $SHARE_NAME complete" + else + SHARE_END=$(date +%s) + SHARE_TIMES+=("$SHARE_NAME:$((SHARE_END - SHARE_START))") + FAIL+=("$SHARE_NAME") + error "$SHARE_NAME failed — continuing to next share" + fi + + echo "" +done + +TOTAL_END=$(date +%s) +TOTAL_DURATION=$((TOTAL_END - TOTAL_START)) + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_CLEAN Post-sync Media Jobs ━━━ +# Reads MEDIA_MANAGEMENT_JOBS from Master.conf — permissions, cleaners, arr cleanup +# Runs after sync completes — correct ownership available, clean folders guaranteed +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_CLEAN Post-sync Media Jobs ━━━" + +if [[ ${#MEDIA_MANAGEMENT_JOBS[@]} -gt 0 ]]; then + for script_entry in "${MEDIA_MANAGEMENT_JOBS[@]}"; do + [[ -z "$script_entry" ]] && continue + script_args=($script_entry) + script_path="$SCRIPTS_ROOT/${script_args[0]}" + script_name=$(basename "${script_args[0]}") + extra_args=("${script_args[@]:1}") + + echo "" + info "$ICON_START Running: $script_name ${extra_args[*]}" + + if [[ ! -f "$script_path" ]]; then + error "$script_name — not found at $script_path" + JOB_FAIL+=("$script_name ${extra_args[*]}") + continue + fi + + if [[ "$DRY_RUN" == true ]]; then + if bash "$script_path" "${extra_args[@]}" --dry-run; then + success "$script_name — done (dry run)" + JOB_PASS+=("$script_name ${extra_args[*]}") + else + error "$script_name — failed" + JOB_FAIL+=("$script_name ${extra_args[*]}") + fi + else + if bash "$script_path" "${extra_args[@]}"; then + success "$script_name — done" + JOB_PASS+=("$script_name ${extra_args[*]}") + else + error "$script_name — failed" + JOB_FAIL+=("$script_name ${extra_args[*]}") + fi + fi + done +fi + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_GEAR Post-sync System Jobs ━━━ +# Reads remaining DAILY_MAINTENANCE_SCRIPTS — docker restart etc. +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_GEAR Post-sync Jobs ━━━" + +for script_entry in "${POST_SYNC_SCRIPTS[@]}"; do + [[ -z "$script_entry" ]] && continue + script_args=($script_entry) + script_path="$SCRIPTS_ROOT/${script_args[0]}" + script_name=$(basename "${script_args[0]}") + extra_args=("${script_args[@]:1}") + + echo "" + info "$ICON_START Running: $script_name" + + if [[ ! -f "$script_path" ]]; then + error "$script_name — not found at $script_path" + JOB_FAIL+=("$script_name") + continue + fi + + if bash "$script_path" "${extra_args[@]}"; then + success "$script_name — done" + JOB_PASS+=("$script_name") + else + error "$script_name — failed" + JOB_FAIL+=("$script_name") + fi +done + +WINDOW_END=$(date +%s) + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_SUMMARY Summary ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━━━ $ICON_SUMMARY DAILY SYNC MAINTENANCE SUMMARY ━━━━━" +echo "$ICON_TIME Window: $(date -d @$WINDOW_START '+%Y-%m-%d %H:%M:%S') → $(date -d @$WINDOW_END '+%H:%M:%S')" +echo "$ICON_TIME Duration: $(format_duration $(( WINDOW_END - WINDOW_START )))" +echo "" + +echo "$ICON_SYNC Media shares:" +for entry in "${SHARE_TIMES[@]}"; do + SHARE_NAME="${entry%%:*}" + DURATION="${entry##*:}" + if printf '%s\n' "${FAIL[@]}" | grep -qx "$SHARE_NAME"; then + echo " $ICON_ERROR $SHARE_NAME — $(format_duration $DURATION)" + else + echo " $ICON_DONE $SHARE_NAME — $(format_duration $DURATION)" + fi +done +echo " Passed: ${#PASS[@]}/$SHARE_COUNT Failed: ${#FAIL[@]}/$SHARE_COUNT" +echo "" + +if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then + echo "$ICON_GEAR Jobs (media + system):" + for job in "${JOB_PASS[@]}"; do echo " $ICON_DONE $job"; done + for job in "${JOB_FAIL[@]}"; do echo " $ICON_ERROR $job"; done + echo "" +fi + +TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} )) + +if [[ "$TOTAL_FAIL" -gt 0 ]]; then + echo "$ICON_WARN Status: $TOTAL_FAIL failure(s) — check logs" + notify "Daily sync maintenance completed with failures on $(hostname) — shares: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" "Daily Maintenance" "warning" + exit 1 +else + echo "$ICON_DONE Status: $ICON_SUCCESS ALL COMPLETE" + notify "Daily sync maintenance complete on $(hostname) — ${#PASS[@]} shares synced, ${#JOB_PASS[@]} jobs run in $(format_duration $(( WINDOW_END - WINDOW_START )))" "Daily Maintenance" "normal" + exit 0 +fi \ No newline at end of file diff --git a/Orchestrators/media_management.sh b/Orchestrators/media_management.sh deleted file mode 100644 index eefd606..0000000 --- a/Orchestrators/media_management.sh +++ /dev/null @@ -1,177 +0,0 @@ -#!/bin/bash -# ----------------------------------------------------------------------------------------------- -# --------------------------------- Media Management Orchestrator ------------------------------ -# ----------------------------------------------------------------------------------------------- -# Runs all media maintenance scripts sequentially in the order defined in Master.conf. -# Each job in MEDIA_MAINTENANCE_JOBS is a script path with an optional argument. -# Scripts are resolved relative to the ecosystem root directory. -# -# To add a new job — edit MEDIA_MAINTENANCE_JOBS in Master.conf: -# "Media/my_new_script.sh" — script with no argument -# "Media/media_cleaner.sh profile" — script with argument -# -# Order matters — permissions runs before cleaners so files are correctly owned first. -# Each job runs independently — a failure in one does not stop the others. -# Supports --dry-run — passes through to all child scripts. -# ----------------------------------------------------------------------------------------------- - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" - -source "$ECOSYSTEM_ROOT/Master.conf" -source "$ECOSYSTEM_ROOT/common.sh" - -parse_args "$@" - -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_GEAR Setup ━━━ -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_GEAR Setup ━━━" - -if [[ "$EUID" -ne 0 ]]; then - error "Must be run as root" - exit 1 -fi - -success "Running as root" - -acquire_lock -[[ "$DRY_RUN" == true ]] && warn "DRY RUN — passing --dry-run to all child scripts" - -# Validate job list -if [[ ${#MEDIA_MAINTENANCE_JOBS[@]} -eq 0 ]]; then - warn "MEDIA_MAINTENANCE_JOBS is empty in Master.conf — nothing to run" - exit 0 -fi - -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Status ━━━ -# ----------------------------------------------------------------------------------------------- -if [[ "$SHOW_STATUS" == true ]]; then - echo "" - echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" - echo "$ICON_CLEAN Jobs to run: ${#MEDIA_MAINTENANCE_JOBS[@]}" - local_idx=1 - for job in "${MEDIA_MAINTENANCE_JOBS[@]}"; do - echo " $local_idx. $job" - ((local_idx++)) - done - echo "$ICON_GEAR Dry Run: $DRY_RUN" - echo "━━━━━━━━━━━━━━━━━━━━━━━" - exit 0 -fi - -# ----------------------------------------------------------------------------------------------- -# Tracking -# ----------------------------------------------------------------------------------------------- -PASS=() -FAIL=() -JOB_TIMES=() -TOTAL_START=$(date +%s) - -# ----------------------------------------------------------------------------------------------- -# JOB RUNNER -# Splits each MEDIA_MAINTENANCE_JOBS entry into script path + optional argument. -# Resolves script relative to ecosystem root. Passes --dry-run if active. -# Records pass/fail and duration for summary. -# ----------------------------------------------------------------------------------------------- -run_job() { - local entry="$1" - - # Split entry into script path and optional argument - local script_rel arg="" - read -r script_rel arg <<< "$entry" - - local script="$ECOSYSTEM_ROOT/$script_rel" - local label - label="$(basename "$script_rel" .sh)${arg:+ $arg}" - - local job_start - job_start=$(date +%s) - - echo "" - echo "━━━ $ICON_CLEAN $label ━━━" - - if [[ ! -f "$script" ]]; then - error "$script not found — skipping" - FAIL+=("$label") - JOB_TIMES+=("$label:0") - return - fi - - if [[ ! -x "$script" ]]; then - warn "$script is not executable — attempting to fix" - chmod +x "$script" - fi - - local dry_flag="" - [[ "$DRY_RUN" == true ]] && dry_flag="--dry-run" - - # Run script with optional argument and optional dry-run flag - if bash "$script" $arg $dry_flag; then - local job_end - job_end=$(date +%s) - PASS+=("$label") - JOB_TIMES+=("$label:$((job_end - job_start))") - success "$label complete" - else - local job_end - job_end=$(date +%s) - FAIL+=("$label") - JOB_TIMES+=("$label:$((job_end - job_start))") - error "$label failed — continuing to next job" - fi -} - -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_CLEAN Media Management ━━━ -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_CLEAN Media Management — $(date '+%Y-%m-%d %H:%M:%S') ━━━" -echo "$ICON_SUMMARY Jobs: ${#MEDIA_MAINTENANCE_JOBS[@]}" -echo "" - -JOB_INDEX=1 -for job in "${MEDIA_MAINTENANCE_JOBS[@]}"; do - info "Job $JOB_INDEX of ${#MEDIA_MAINTENANCE_JOBS[@]}: $job" - run_job "$job" - ((JOB_INDEX++)) -done - -TOTAL_END=$(date +%s) -TOTAL_DURATION=$((TOTAL_END - TOTAL_START)) -JOB_COUNT=$(( ${#PASS[@]} + ${#FAIL[@]} )) - -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Summary ━━━ -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━━━ $ICON_SUMMARY MEDIA MANAGEMENT SUMMARY ━━━━━" -echo "$ICON_TIME Started: $(date -d @$TOTAL_START '+%Y-%m-%d %H:%M:%S')" -echo "$ICON_TIME Finished: $(date -d @$TOTAL_END '+%Y-%m-%d %H:%M:%S')" -echo "" - -for entry in "${JOB_TIMES[@]}"; do - label="${entry%%:*}" - duration="${entry##*:}" - if printf '%s\n' "${FAIL[@]}" | grep -qx "$label"; then - echo " $ICON_ERROR $label — $(format_duration $duration)" - else - echo " $ICON_DONE $label — $(format_duration $duration)" - fi -done - -echo "" -echo " $ICON_SUCCESS Passed: ${#PASS[@]}/$JOB_COUNT" -echo " $ICON_ERROR Failed: ${#FAIL[@]}/$JOB_COUNT" -echo " $ICON_TIME Duration: $(format_duration $TOTAL_DURATION)" -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - -if [[ ${#FAIL[@]} -gt 0 ]]; then - notify "Media management completed with failures on $(hostname) — failed: ${FAIL[*]}" "Media Management" "warning" - exit 1 -else - notify "Media management complete on $(hostname) — ${#PASS[@]}/$JOB_COUNT jobs in $(format_duration $TOTAL_DURATION)" "Media Management" "normal" - exit 0 -fi \ No newline at end of file diff --git a/Orchestrators/media_shares_sync.sh b/Orchestrators/media_shares_sync.sh deleted file mode 100644 index 3ed4f3d..0000000 --- a/Orchestrators/media_shares_sync.sh +++ /dev/null @@ -1,141 +0,0 @@ -#!/bin/bash -# ----------------------------------------------------------------------------------------------- -# --------------------------------- Media Shares Sync ------------------------------------ -# ----------------------------------------------------------------------------------------------- -# Runs all media shares sequentially in the correct direction for the local server. -# Each server pushes its own source-of-truth shares to the remote — direction is automatic. -# -# HOST1 pushes: its truth shares (Movies, Tv_Shows, Music etc.) + personal → HOST2 -# HOST2 pushes: its truth shares (Anime_Shows, Anime_Movies etc.) + personal → HOST1 -# -# detect_hosts() determines which server is running the script at runtime. -# Share lists are configured per host in Master.conf — no script changes needed -# to add, remove, or reconfigure shares. -# -# Per-share rsync handled by rsync.sh — this script tracks pass/fail and total time only. -# Scheduled via unRAID User Scripts plugin at 1am on both servers. -# ----------------------------------------------------------------------------------------------- - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - -source "$SCRIPT_DIR/../Master.conf" -source "$SCRIPT_DIR/../common.sh" - -RSYNC_SCRIPT="$SCRIPT_DIR/../Rsync/rsync.sh" - -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_GEAR Setup ━━━ -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_GEAR Setup ━━━" - -detect_hosts -resolve_remote_ip - -acquire_lock - -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SHIELD Pre-flight Checks ━━━ -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_SHIELD Pre-flight Checks ━━━" - -# Single connectivity and rootfs check upfront — fail fast before attempting all shares -# Individual share and disk checks run per-share inside rsync.sh -check_connectivity -check_remote_rootfs - -# ----------------------------------------------------------------------------------------------- -# Build share list — host-specific truth shares + personal shares -# Each server only syncs the shares it is source of truth for -# Personal shares appended after media shares -# ----------------------------------------------------------------------------------------------- -PASS=() -FAIL=() -SHARE_TIMES=() -TOTAL_START=$(date +%s) - -ALL_SHARES=() - -if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then - for share in "${HOST1_DAILY_SYNC_SHARES[@]}"; do - [[ -n "$share" ]] && ALL_SHARES+=("$share") - done - for share in "${HOST1_PERSONAL_SHARES[@]}"; do - [[ -n "$share" ]] && ALL_SHARES+=("$share") - done -elif [[ "$LOCAL_SERVER_NAME" == "$HOST2" ]]; then - for share in "${HOST2_DAILY_SYNC_SHARES[@]}"; do - [[ -n "$share" ]] && ALL_SHARES+=("$share") - done - for share in "${HOST2_PERSONAL_SHARES[@]}"; do - [[ -n "$share" ]] && ALL_SHARES+=("$share") - done -fi - -SHARE_COUNT=${#ALL_SHARES[@]} - -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SYNC Transfer ━━━ -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_SYNC Daily Sync Starting — $(date '+%Y-%m-%d %H:%M:%S') ━━━" -echo "$ICON_SUMMARY Shares: $SHARE_COUNT" -echo "" - -SHARE_INDEX=0 - -for SHARE in "${ALL_SHARES[@]}"; do - SHARE_INDEX=$((SHARE_INDEX + 1)) - SHARE_NAME=$(basename "$SHARE") - SHARE_START=$(date +%s) - - echo "━━━ $ICON_SYNC Share $SHARE_INDEX of $SHARE_COUNT: $SHARE_NAME ━━━" - - if bash "$RSYNC_SCRIPT" "$SHARE"; then - SHARE_END=$(date +%s) - SHARE_TIMES+=("$SHARE_NAME:$((SHARE_END - SHARE_START))") - PASS+=("$SHARE_NAME") - echo "$ICON_DONE $SHARE_NAME complete" - else - SHARE_END=$(date +%s) - SHARE_TIMES+=("$SHARE_NAME:$((SHARE_END - SHARE_START))") - FAIL+=("$SHARE_NAME") - error "$SHARE_NAME failed — continuing to next share" - fi - - echo "" -done - -TOTAL_END=$(date +%s) -TOTAL_DURATION=$((TOTAL_END - TOTAL_START)) - -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Summary ━━━ -# ----------------------------------------------------------------------------------------------- -echo "━━━━━ $ICON_SUMMARY DAILY SYNC SUMMARY ━━━━━" -echo "$ICON_TIME Started: $(date -d @$TOTAL_START '+%Y-%m-%d %H:%M:%S')" -echo "$ICON_TIME Finished: $(date -d @$TOTAL_END '+%Y-%m-%d %H:%M:%S')" -echo "" -for entry in "${SHARE_TIMES[@]}"; do - SHARE_NAME="${entry%%:*}" - DURATION="${entry##*:}" - if printf '%s\n' "${FAIL[@]}" | grep -qx "$SHARE_NAME"; then - echo " $ICON_ERROR $SHARE_NAME — $(format_duration $DURATION)" - else - echo " $ICON_DONE $SHARE_NAME — $(format_duration $DURATION)" - fi -done -echo "" -echo " $ICON_SUCCESS Passed: ${#PASS[@]}/$SHARE_COUNT" -echo " $ICON_ERROR Failed: ${#FAIL[@]}/$SHARE_COUNT" -echo " $ICON_TIME Duration: $(format_duration $TOTAL_DURATION)" -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - -if [[ ${#FAIL[@]} -gt 0 ]]; then - notify "Daily sync completed with failures — ${#FAIL[@]}/$SHARE_COUNT failed: ${FAIL[*]}" "Daily Sync" "warning" - exit 1 -else - notify "Daily sync complete — ${#PASS[@]}/$SHARE_COUNT shares in $(format_duration $TOTAL_DURATION)" "Daily Sync" "normal" - exit 0 -fi \ No newline at end of file diff --git a/Orchestrators/critical_shares_maintenance.sh b/Orchestrators/weekly _sync_maintenance.sh similarity index 77% rename from Orchestrators/critical_shares_maintenance.sh rename to Orchestrators/weekly _sync_maintenance.sh index 6b658e3..8695a41 100644 --- a/Orchestrators/critical_shares_maintenance.sh +++ b/Orchestrators/weekly _sync_maintenance.sh @@ -1,6 +1,6 @@ #!/bin/bash # ----------------------------------------------------------------------------------------------- -# ----------------------------- Critical Shares Maintenance ------------------------------------ +# ----------------------------- Weekly Sync Maintenance ------------------------------------ # ----------------------------------------------------------------------------------------------- # Maintenance window orchestrator for Emby and the auth stack (Critical-Data). # Both local and remote containers are stopped for the entire window — clean state @@ -161,11 +161,7 @@ PASS=() FAIL=() TOTAL_START=$(date +%s) -SYNC_JOBS=( - "/mnt/user/Media_Server/Emby" - "/mnt/user/appdata-Failover/Critical-Data" -) - +SYNC_JOBS=("${WEEKLY_SYNC_JOBS[@]}") SHARE_COUNT=${#SYNC_JOBS[@]} echo "" @@ -221,32 +217,85 @@ fi TOTAL_END=$(date +%s) TOTAL_DURATION=$(format_duration $(( TOTAL_END - TOTAL_START ))) +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_GEAR Post-sync Jobs ━━━ +# docker_weekly_restart.sh and any other WEEKLY_MAINTENANCE_SCRIPTS run after sync +# ----------------------------------------------------------------------------------------------- +JOB_PASS=() +JOB_FAIL=() +SCRIPTS_ROOT="$SCRIPT_DIR/.." + +if [[ ${#WEEKLY_MAINTENANCE_SCRIPTS[@]} -gt 0 ]]; then + echo "" + echo "━━━ $ICON_GEAR Post-sync Jobs ━━━" + + for script_entry in "${WEEKLY_MAINTENANCE_SCRIPTS[@]}"; do + [[ -z "$script_entry" ]] && continue + script_args=($script_entry) + script_path="$SCRIPTS_ROOT/${script_args[0]}" + script_name=$(basename "${script_args[0]}") + extra_args=("${script_args[@]:1}") + + echo "" + info "$ICON_START Running: $script_name" + + if [[ ! -f "$script_path" ]]; then + error "$script_name — not found at $script_path" + JOB_FAIL+=("$script_name") + continue + fi + + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would run: $script_name" + JOB_PASS+=("$script_name (dry run)") + elif bash "$script_path" "${extra_args[@]}"; then + success "$script_name — done" + JOB_PASS+=("$script_name") + else + error "$script_name — failed" + JOB_FAIL+=("$script_name") + fi + done +fi + +WINDOW_END=$(date +%s) + # ----------------------------------------------------------------------------------------------- # ━━━ $ICON_SUMMARY Summary ━━━ # ----------------------------------------------------------------------------------------------- echo "" -echo "━━━━━ $ICON_SUMMARY CRITICAL SHARES MAINTENANCE SUMMARY ━━━━━" +echo "━━━━━ $ICON_SUMMARY WEEKLY SYNC MAINTENANCE SUMMARY ━━━━━" echo "$ICON_TIME Duration: $TOTAL_DURATION" echo "$ICON_GEAR Updates: local=$CRITICAL_SYNC_UPDATES remote=$CRITICAL_SYNC_UPDATES_REMOTE" -echo "$ICON_SUCCESS Passed: ${#PASS[@]} $ICON_ERROR Failed: ${#FAIL[@]}" echo "" +echo "$ICON_SYNC Sync jobs:" if [[ ${#PASS[@]} -gt 0 ]]; then - for job in "${PASS[@]}"; do echo " $ICON_SUCCESS $job"; done + for job in "${PASS[@]}"; do echo " $ICON_SUCCESS $job"; done fi if [[ ${#FAIL[@]} -gt 0 ]]; then - for job in "${FAIL[@]}"; do echo " $ICON_ERROR $job"; done + for job in "${FAIL[@]}"; do echo " $ICON_ERROR $job"; done +fi +echo " Passed: ${#PASS[@]} Failed: ${#FAIL[@]}" + +if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then + echo "" + echo "$ICON_GEAR Post-sync jobs:" + for job in "${JOB_PASS[@]}"; do echo " $ICON_SUCCESS $job"; done + for job in "${JOB_FAIL[@]}"; do echo " $ICON_ERROR $job"; done fi echo "" +TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} )) + if [[ "$DRY_RUN" == true ]]; then echo "$ICON_WARN Status: DRY RUN — no changes made" -elif [[ ${#FAIL[@]} -eq 0 ]]; then - echo "$ICON_DONE Status: $ICON_SUCCESS ALL JOBS COMPLETE" - notify "Critical maintenance complete on $(hostname) — synced + updated (local=$CRITICAL_SYNC_UPDATES remote=$CRITICAL_SYNC_UPDATES_REMOTE)" "Critical Maintenance" "normal" +elif [[ "$TOTAL_FAIL" -eq 0 ]]; then + echo "$ICON_DONE Status: $ICON_SUCCESS ALL COMPLETE" + notify "Weekly sync maintenance complete on $(hostname) — synced + updated (local=$CRITICAL_SYNC_UPDATES remote=$CRITICAL_SYNC_UPDATES_REMOTE)" "Weekly Maintenance" "normal" else - echo "$ICON_ERROR Status: $ICON_ERROR ${#FAIL[@]} JOB(S) FAILED" - notify "Critical maintenance failed on $(hostname) — failed: ${FAIL[*]}" "Critical Maintenance" "warning" + echo "$ICON_ERROR Status: $TOTAL_FAIL failure(s) — check logs" + notify "Weekly sync maintenance failed on $(hostname) — sync: ${#FAIL[@]} failed, jobs: ${#JOB_FAIL[@]} failed" "Weekly Maintenance" "warning" fi echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" \ No newline at end of file diff --git a/Rsync/README-Rsync_Setup.md b/Rsync/README-Rsync_Setup.md index 64e7d23..a49ce72 100644 --- a/Rsync/README-Rsync_Setup.md +++ b/Rsync/README-Rsync_Setup.md @@ -1,6 +1,6 @@ # Rsync Setup Guide -> **Status:** Work in Progress -> For the unRAID Rsync Ecosystem — `common.sh` · `Master.conf` · `rsync.sh` · `daily_sync.sh` + +> For the unRAID Script Ecosystem — `Master.conf` · `common.sh` · `rsync.sh` · `daily_sync_maintenance.sh` · `weekly_sync_maintenance.sh` --- @@ -8,11 +8,13 @@ This guide walks through setting up the rsync ecosystem on both your primary and secondary unRAID 7.x servers. By the end you will have: -- A Gitea repository cloned to both servers - SSH keys configured for server-to-server communication - Tailscale running on both servers for secure networking -- Scripts scheduled and running via the User Scripts plugin -- Automated daily sync of media shares and appdata profiles +- A Gitea repository cloned to both servers +- All scripts scheduled via the User Scripts plugin +- Automated daily sync of media shares driven by orchestrators +- Automated weekly clean sync of critical appdata (Emby + auth stack) +- Optional personal encrypted shares synced for offsite backup --- @@ -21,141 +23,130 @@ This guide walks through setting up the rsync ecosystem on both your primary and Both servers need the following before starting: - unRAID 7.x +- [Community Applications plugin](https://forums.unraid.net/topic/38582-plug-in-community-applications/) installed - [User Scripts plugin](https://forums.unraid.net/topic/48286-plugin-user-scripts/) installed via Community Applications - [Tailscale plugin](https://forums.unraid.net/topic/136889-tailscale-plugin/) installed via Community Applications -- Access to a Gitea instance (self-hosted or remote) -- Terminal access to both servers (via unRAID UI → Tools → Terminal, or SSH) +- Access to a Gitea instance (self-hosted recommended — Gitea runs as a Docker container on HOST1) +- Terminal access to both servers (unRAID UI → Tools → Terminal, or SSH) --- ## Step 1 — Tailscale Setup -Tailscale provides the secure network tunnel between your two servers. The scripts resolve the remote server's IP via Tailscale at runtime. +Tailscale provides the secure network tunnel between your two servers. Scripts resolve the remote server's IP via Tailscale at runtime — no hardcoded IPs needed. ### On Both Servers 1. Open **Apps** in the unRAID UI 2. Search for **Tailscale** and install the plugin -3. Once installed go to **Settings → Tailscale** +3. Go to **Settings → Tailscale** 4. Click **Connect** and authenticate with your Tailscale account 5. Verify both servers appear in your [Tailscale admin console](https://login.tailscale.com/admin/machines) ### Verify Connectivity -Run this on the primary to confirm it can see the secondary: +Run this on HOST1 to confirm it can reach HOST2: ```bash tailscale ip -4 unRAID-Jayred365 ``` -You should get back a `100.x.x.x` IP. If not, check that both machines are authenticated and connected in the Tailscale admin console. +You should get back a `100.x.x.x` IP. If not, check both machines are authenticated in the Tailscale admin console. -> **Note:** The hostnames used in `Master.conf` (`HOST1` and `HOST2`) must match the Tailscale machine names exactly — these are case sensitive. +> **Important:** The hostnames in `Master.conf` (`HOST1` and `HOST2`) must match the Tailscale machine names exactly — case sensitive. --- -## Step 2 — Generate SSH Keys +## Step 2 — Enable SSH on unRAID + +unRAID 7.x has SSH disabled by default. Enable it on both servers so scripts can connect between them. + +1. Go to **Settings → Management Access** +2. Under **Secure Shell** set **SSH** to `Enabled` +3. Set **SSH port** to `22` +4. Click **Apply** + +> SSH is only exposed on your local network and Tailscale interface. Scripts connect via Tailscale IP — traffic is encrypted end-to-end. + +--- + +## Step 3 — Generate SSH Keys The scripts use SSH keys for two purposes: -- **Server-to-server rsync** — primary authenticates to secondary (and vice versa) +- **Server-to-server rsync and failover** — each server authenticates to the other - **Gitea access** — both servers pull from the git repository -### 2a — Server-to-Server Keys +### 3a — Server-to-Server Keys -Run the following on **each server** to generate its rsync key. Replace the filename with the appropriate server name. - -**On Primary (unRAID-Gmer4Lfe):** +**On HOST1 (unRAID-Gmer4Lfe):** ```bash ssh-keygen -t ed25519 -f /root/.ssh/Gmer4Lfe-rsync-key -C "gmer4lfe-rsync" -N "" ``` -**On Secondary (unRAID-Jayred365):** +**On HOST2 (unRAID-Jayred365):** ```bash ssh-keygen -t ed25519 -f /root/.ssh/Jayred365-rsync-key -C "jayred365-rsync" -N "" ``` -### 2b — Copy Public Keys to Each Server +### 3b — Authorise Keys on Each Server -The primary's public key must be authorised on the secondary, and vice versa. +HOST1's public key must be authorised on HOST2, and vice versa. -**Copy primary key → secondary:** +**Copy HOST1 key → HOST2:** ```bash -# Run on primary +# On HOST1 — print the public key cat /root/.ssh/Gmer4Lfe-rsync-key.pub -``` -Copy the output. Then on the secondary: - -```bash -# Run on secondary +# On HOST2 — paste and authorise mkdir -p /root/.ssh echo "PASTE_PUBLIC_KEY_HERE" >> /root/.ssh/authorized_keys chmod 600 /root/.ssh/authorized_keys ``` -**Copy secondary key → primary:** +**Copy HOST2 key → HOST1:** ```bash -# Run on secondary +# On HOST2 — print the public key cat /root/.ssh/Jayred365-rsync-key.pub -``` -Copy the output. Then on the primary: - -```bash -# Run on primary -mkdir -p /root/.ssh +# On HOST1 — paste and authorise echo "PASTE_PUBLIC_KEY_HERE" >> /root/.ssh/authorized_keys -chmod 600 /root/.ssh/authorized_keys ``` -### 2c — Test the Connection +### 3c — Test the Connection -From the primary, test that it can SSH to the secondary without a password prompt: +From HOST1, verify it can SSH to HOST2 without a password prompt: ```bash ssh -i /root/.ssh/Gmer4Lfe-rsync-key root@$(tailscale ip -4 unRAID-Jayred365) "echo connected" ``` -You should see `connected`. If prompted for a password the key was not authorised correctly — recheck Step 2b. +You should see `connected`. If prompted for a password the key was not authorised correctly — recheck Step 3b. -### 2d — Gitea SSH Key +### 3d — Gitea SSH Key Generate a separate key for Gitea access on each server: ```bash -ssh-keygen -t ed25519 -f /root/.ssh/id_gitea_rsync -C "unraid-gitea" -N "" +ssh-keygen -t ed25519 -f /root/.ssh/unraid_gitea -C "unraid-gitea" -N "" ``` Add the public key to your Gitea account: ```bash -cat /root/.ssh/id_gitea_rsync.pub +cat /root/.ssh/unraid_gitea.pub ``` Copy the output and add it in Gitea under **Settings → SSH / GPG Keys → Add Key**. --- -## Step 3 — Enable SSH on unRAID - -unRAID 7.x has SSH disabled by default. Enable it on both servers so the scripts can connect between them. - -1. Go to **Settings → Management Access** -2. Under **Secure Shell** set **SSH** to `Enabled` -3. Set **SSH port** to `22` (default) -4. Click **Apply** - -> **Security note:** SSH is only exposed on your local network and Tailscale interface. The rsync scripts connect via the Tailscale IP so traffic is encrypted end-to-end. - ---- - ## Step 4 — Clone the Git Repository -The scripts live in a Gitea repository. Both servers clone from the same repo so updates propagate everywhere via a single git pull. +Both servers clone from the same Gitea repository. Updates pushed to the repo propagate to both servers on the next daily git pull. ### On Both Servers @@ -164,12 +155,12 @@ The scripts live in a Gitea repository. Both servers clone from the same repo so mkdir -p /mnt/user/appdata/unraid_scripts # Clone the repository -GIT_SSH_COMMAND="ssh -i /root/.ssh/id_gitea_rsync" \ - git clone git@YOUR_GITEA_HOST:YOUR_USER/Unraid_Scripts.git \ +GIT_SSH_COMMAND="ssh -i /root/.ssh/unraid_gitea" \ + git clone git@YOUR_GITEA_HOST:FailedProxy/Unraid_Scripts.git \ /mnt/user/appdata/unraid_scripts ``` -Replace `YOUR_GITEA_HOST` and `YOUR_USER` with your Gitea server address and username. +Replace `YOUR_GITEA_HOST` with your Gitea server address and port. ### Verify the Structure @@ -182,252 +173,385 @@ You should see: ``` Master.conf common.sh -Rsync/ - rsync.sh Orchestrators/ - daily_sync.sh +Rsync/ +Failover/ +Docker_Essentials/ +unRAID_Essentials/ +Media/ +Transcodes/ +Monitors/ Tools/ - recreate_shares.sh ``` ### Make Scripts Executable ```bash -chmod +x /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh -chmod +x /mnt/user/appdata/unraid_scripts/Orchestrators/daily_sync.sh -chmod +x /mnt/user/appdata/unraid_scripts/Tools/recreate_shares.sh +find /mnt/user/appdata/unraid_scripts -name "*.sh" -exec chmod +x {} \; ``` --- ## Step 5 — Configure Master.conf -All user configuration lives in `Master.conf`. Open it and adjust the following to match your setup: +All user configuration lives in `Master.conf`. Open it and fill in your values: ```bash nano /mnt/user/appdata/unraid_scripts/Master.conf ``` -### Required Changes - -| Variable | Description | Example | -|---|---|---| -| `HOST1` | Hostname of your primary server | `unRAID-Gmer4Lfe` | -| `HOST2` | Hostname of your secondary server | `unRAID-Jayred365` | -| `HOST1_SSH_KEY` | Path to primary's rsync private key | `/root/.ssh/Gmer4Lfe-rsync-key` | -| `HOST2_SSH_KEY` | Path to secondary's rsync private key | `/root/.ssh/Jayred365-rsync-key` | -| `REPO_SSH` | SSH URL of your Gitea repository | `git@192.168.50.2:User/Unraid_Scripts.git` | -| `GITEA_SSH_KEY` | Path to Gitea private key | `/root/.ssh/id_gitea_rsync` | -| `BW_LIMIT` | Global bandwidth limit in KB/s | `12500` | -| `ROOTFS_WARN` | Remote rootfs % threshold before aborting | `75` | - -### Daily Sync Shares - -Add the full paths of all media shares you want synced nightly: +### Host Configuration ```bash -DAILY_SYNC_SHARES=( +HOST1="unRAID-Gmer4Lfe" # must match Tailscale machine name exactly +HOST2="unRAID-Jayred365" + +HOST1_SSH_KEY="/root/.ssh/Gmer4Lfe-rsync-key" +HOST2_SSH_KEY="/root/.ssh/Jayred365-rsync-key" + +HOST1_EMBY_CONTAINER="Emby" +HOST1_EMBY_URL="http://localhost:8096" +HOST1_EMBY_API_KEY="your-host1-emby-api-key" # Emby Dashboard → API Keys → + New Key + +HOST2_EMBY_CONTAINER="Emby-Jayred365" +HOST2_EMBY_URL="http://localhost:8096" +HOST2_EMBY_API_KEY="your-host2-emby-api-key" +``` + +### Git / Repo + +```bash +GITEA_CONTAINER="Gitea" # exact Docker container name +GITEA_REPO_PATH="FailedProxy/Unraid_Scripts.git" +GITEA_DOMAIN="" # optional public domain fallback +TARGET_DIR="/mnt/user/appdata/unraid_scripts" +GITEA_SSH_KEY="/root/.ssh/unraid_gitea" +SSH_PORT=221 +``` + +### Orchestrators — Daily Sync Shares + +Define which shares each server owns. Each server only pushes the shares it is source of truth for — the other server mirrors and treats them as read-only. + +```bash +HOST1_DAILY_SYNC_SHARES=( /mnt/user/Movies /mnt/user/Tv_Shows /mnt/user/Music - # add more here + # add all HOST1-managed shares here +) + +HOST2_DAILY_SYNC_SHARES=( + /mnt/user/Anime_Shows + /mnt/user/Anime_Movies + # add all HOST2-managed shares here ) ``` -### Profiles +> Never put the same share in both lists. One server is always the truth holder for each share. -Profiles control per-share rsync behaviour for your frequently synced appdata shares. Each profile is matched by the directory basename (lowercased) — or overridden with `--profile=name`. +### Orchestrators — Weekly Sync Jobs + +Shares synced during the Sunday maintenance window with containers stopped both sides: ```bash -# One array drives both local and remote container stops -# Local stops first (flush databases) then remote stops (clean receive) -# Same container names on both HOST1 and HOST2 — no duplication needed -# Containers not found on a server are skipped gracefully -declare -A PROFILE_CRITICAL_CONTAINER_NAMES=( - [critical-data]="Mariadb-Authelia Mariadb-Authelia-Secondary Redis-Authelia Redis-Authelia-Secondary Lldap-Gmer4Lfe NginxProxyManager Authelia Authelia-Secondary" - [important-data]="Postgres-NextCloud NextCloud" - [emby]="Emby" # nightly clean sync — Emby stopped both sides - [emby-failover]="" # dirty sync — Emby stays running +WEEKLY_SYNC_JOBS=( + "/mnt/user/Media_Server/Emby" # full clean Emby mirror + "/mnt/user/appdata-Failover/Critical-Data" # auth stack ) ``` -Any share with no matching profile falls through to the global `DEFAULT_RSYNC_OPTS`. Containers not found on a server are skipped gracefully — only containers that were actually running get restarted. +--- -**Two Emby profiles:** +## Step 6 — Rsync Profiles + +Profiles control per-share rsync behaviour for appdata syncs. The profile key is matched automatically by the basename of the directory passed to `rsync.sh` (lowercased). Override with `--profile=name`. + +One array drives both local and remote container stops. Same container names on both servers — consistent naming is a requirement of this ecosystem. + +### Current Profiles + +| Profile | Purpose | Containers Stopped | +|---|---|---| +| `arrs_stack` | Arr databases | Sonarr, Radarr, Lidarr, Prowlarr, Bazarr, Pinchflat | +| `critical-data` | Auth stack | Mariadb, Redis, LLDAP, NPM, Authelia (delayed start) | +| `important-data` | NextCloud + Postgres | Postgres, NextCloud (delayed start) | +| `gmer4lfe` | Server-specific appdata | Organizr, UptimeKuma, VaultWarden | +| `emby` | Weekly clean sync | Emby both sides — WAL checkpointed | +| `emby-failover` | Frequent dirty sync | None — Emby stays running | + +### Two Emby Profiles ``` -emby-failover — frequent dirty sync (every 30-60min): - Emby stays running on both sides +emby-failover — every 30-60min, Emby stays running: WAL and SHM excluded — safe while Emby is active - Only critical failover data: users.db, library.db, authentication.db, config/ - Fast, small dataset, high bandwidth - This is also what gets written back during failover handback + Critical failover data only: users.db, library.db, authentication.db, config/ + Fast, small dataset — users continue watching without interruption on failover + Also used for failover writeback on handback -emby — weekly clean sync (via nightly_critical_full_sync.sh Sunday 2:30am): - Emby stopped on both sides — WAL checkpointed on shutdown +emby — weekly Sunday 2:30am, both Emby instances stopped: + WAL checkpointed on shutdown — full consistent mirror Full mirror: metadata, plugins, config all included Minimal excludes: logs, transcodes, cache, crash files only - Complete faithful state pushed once per week Cache stays warm on HOST2 all week — only reset on Sunday - emby-failover handles the critical state between weekly syncs -``` - -Usage with profile override: -```bash -# Dirty sync — Emby stays running -bash rsync.sh /mnt/user/Media_Server/Emby --profile=emby-failover - -# Clean sync — called by nightly_critical_full_sync.sh, Emby stopped via profile -bash rsync.sh /mnt/user/Media_Server/Emby + emby-failover covers the critical state between weekly syncs ``` --- -## Step 6 — Set Up User Scripts +## Step 7 — Personal Encrypted Shares -The User Scripts plugin is how unRAID schedules and runs the scripts. Each sync job is its own script entry in the plugin. +Personal shares can be synced to the remote server for offsite backup. ZFS encrypts at the dataset level — the remote server receives encrypted blocks and cannot read the content without your passphrase or keyfile. -### Frequent Sync Jobs (Scheduled Individually) +### ZFS Encryption Setup (unRAID 7) -Create one script entry per appdata profile. Go to **Plugins → User Scripts → Add New Script**. +**Step 1 — Create an encrypted dataset:** -Name it descriptively — e.g. `rsync appdata arrs_stack`. +1. In the unRAID UI go to **Main** → click your ZFS pool name +2. Click **+ Dataset** to create a new dataset +3. Name it — e.g. `Gmer4Lfe-Personal` +4. Enable **Encryption** → set your passphrase + > ⚠️ Write your passphrase down — if lost, data is unrecoverable -In the script body paste: +**Step 2 — Create the share:** + +1. Go to **Settings → Shares → Add Share** +2. Set the share path to your new encrypted dataset +3. Set **Use cache:** `Only` — keeps data on ZFS pool, not array + +**Step 3 — Verify encryption is active before syncing:** + +```bash +zfs get encryption poolname/Gmer4Lfe-Personal +# Should show: encryption aes-256-gcm +``` + +**Step 4 — Auto-unlock on boot (keyfile approach — optional):** + +```bash +# Create keyfile — on HOST1 only, never sync this file +dd if=/dev/urandom bs=32 count=1 | base64 > /root/.zfs-keys/personal.key +chmod 600 /root/.zfs-keys/personal.key + +# Set dataset to use keyfile +zfs change-key -o keylocation=file:///root/.zfs-keys/personal.key \ + -o keyformat=raw poolname/Gmer4Lfe-Personal + +# Add to array start (via array_start.sh or User Scripts): +zfs load-key poolname/Gmer4Lfe-Personal +zfs mount poolname/Gmer4Lfe-Personal +``` + +**Manual unlock alternative (most secure):** + +```bash +zfs load-key poolname/Gmer4Lfe-Personal # prompts for passphrase +zfs mount poolname/Gmer4Lfe-Personal +``` + +**Step 5 — Add to Master.conf:** + +```bash +HOST1_PERSONAL_SHARES=( + /mnt/user/Gmer4Lfe-Personal +) +``` + +Personal shares sync automatically with the daily media share sync in `daily_sync_maintenance.sh`. The remote server receives encrypted blocks — content is unreadable without your key. + +--- + +## Step 8 — Set Up User Scripts + +The ecosystem uses a single orchestrator entry for array startup plus a small number of scheduled scripts. + +### At Startup of Array + +Create one script entry named `array start`: ```bash #!/bin/bash -bash /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-Failover/Arrs_Stack +bash /mnt/user/appdata/unraid_scripts/Orchestrators/array_start.sh ``` -Set the schedule to match your desired frequency: +Set schedule to: **At Startup of Array** -| Profile | Schedule | Notes | +This single entry launches everything configured in `ARRAY_START_SCRIPTS` in `Master.conf`: +- `ramdisk_setup.sh` — creates ramdisk before Emby starts +- `docker_syslog_filter.sh` — suppresses veth log noise +- `php_fpm_max_children.sh` — WebGUI tuning +- `docker_network_connect.sh` — connects containers to extra networks +- `system_watchdog.sh` — continuous system health monitor +- `docker_watchdog.sh` — continuous container health monitor +- `failover.sh` — continuous mutual failover + +### Cron Schedules + +| Script | Schedule | Purpose | |---|---|---| -| `emby-failover` | Every 30-60 min | Dirty sync — Emby running, critical data only | -| `emby` | Weekly Sunday via `nightly_critical_full_sync.sh` | Clean sync — Emby stopped, full mirror, cache stays warm all week | -| `Critical-Data` | Weekly Sunday via `nightly_critical_full_sync.sh` | Auth stack — clean weekly sync | -| `Important-Data` | Every 6-12 hours | NextCloud file changes | -| `Arrs_Stack` | Every 12-24 hours | Arr databases | -| `Gmer4Lfe` | Daily or weekly | Personal appdata, rarely changes | +| `transcode_management.sh` | `*/3 * * * *` | Transcode cleanup + manager | +| `arrs_failed_stalled_recovery.sh` | `0 */6 * * *` | Blocklist + re-search failed imports | +| `rsync.sh ... --profile=emby-failover` | `*/30 * * * *` | Emby dirty sync | +| `daily_sync_maintenance.sh` | `0 1 * * *` | Full daily maintenance window | +| `weekly_sync_maintenance.sh` | `30 2 * * 0` | Weekly sync + updates + restarts | +| `weekly_health_digest.sh` | `0 8 * * 6` | Saturday morning health digest | -> **Important:** Set each script to run as a **Background Task** — this ensures output streams correctly to the log rather than buffering in the browser. +### Emby Failover Dirty Sync -### Daily Sync Orchestrator - -Create one more script entry for the daily media sync: - -Name it `daily media sync`. - -In the script body paste: +Create a separate script entry named `rsync emby failover`: ```bash #!/bin/bash -bash /mnt/user/appdata/unraid_scripts/Orchestrators/daily_sync.sh +bash /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh \ + /mnt/user/Media_Server/Emby --profile=emby-failover ``` -Set the schedule to **Daily at 01:00**. +Set schedule to: `*/30 * * * *` + +### Appdata Profile Syncs + +Create one entry per appdata profile you want on a schedule: + +```bash +#!/bin/bash +bash /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh \ + /mnt/user/appdata-Failover/Arrs_Stack +``` + +| Profile | Recommended Schedule | +|---|---| +| `Arrs_Stack` | Every 12-24 hours | +| `Important-Data` | Every 6-12 hours | +| `Gmer4Lfe` | Daily or weekly | +| `emby` | Via `weekly_sync_maintenance.sh` only — do NOT schedule separately | +| `Critical-Data` | Via `weekly_sync_maintenance.sh` only — do NOT schedule separately | + +> Set all scripts to run as **Background Task** — output streams correctly rather than buffering in the browser. --- -## Step 7 — Verify the Setup +## Step 9 — Verify the Setup -Before letting the scheduled jobs run, do a manual test from the terminal on the primary: +Before letting scheduled jobs run, test manually from the terminal on HOST1: ```bash -bash /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-Failover/Arrs_Stack --log +# Test a single appdata profile sync +bash /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh \ + /mnt/user/appdata-Failover/Arrs_Stack --dry-run --log + +# Test the daily sync orchestrator +bash /mnt/user/appdata/unraid_scripts/Orchestrators/daily_sync_maintenance.sh --dry-run ``` A healthy run will show: ``` ━━━ ⚙️ Setup ━━━ -ℹ️ [INFO] 🖥️ Host: unRAID-Gmer4Lfe → unRAID-Jayred365 -ℹ️ [INFO] 🌐 Remote IP: 100.x.x.x +ℹ️ Host: unRAID-Gmer4Lfe → unRAID-Jayred365 +ℹ️ Remote IP: 100.x.x.x ━━━ 🛡️ Pre-flight Checks ━━━ -ℹ️ [INFO] 📡 unRAID-Jayred365 is reachable -ℹ️ [INFO] 🩺 Remote rootfs: 12% used (threshold: 75%) -ℹ️ [INFO] 🩺 Remote share verified: ... -ℹ️ [INFO] 💾 disk1 🟢 — share present -✅ [OK] All disks backing share are online +✅ Remote reachable +✅ Remote rootfs: 12% (threshold: 75%) +✅ All pre-flight checks passed ``` -If any pre-flight check fails the script will abort with a clear error and hint before touching anything. +If any pre-flight check fails the script aborts with a clear error before touching anything. --- -## Step 8 — Secondary Server Initial Setup +## Step 10 — Secondary Server Initial Sync -If setting up the secondary from scratch (no existing data): +If setting up HOST2 from scratch with empty shares: -1. Complete Steps 1–5 on the secondary +1. Complete Steps 1–8 on HOST2 2. Start the array and create your shares in the unRAID UI -3. Run the share recreation tool to create disk directories from your cfg files: +3. Run the share recreation tool to create disk directories from cfg files: ```bash bash /mnt/user/appdata/unraid_scripts/Tools/recreate_shares.sh ``` -4. Temporarily remove `--delete` from `DEFAULT_RSYNC_OPTS` in `Master.conf` -5. Run the initial push from the primary — the `.recovery` marker files allow rsync to populate empty shares without aborting -6. Once complete, restore `--delete` to `Master.conf` -7. The next nightly run will clean up the `.recovery` marker files automatically +4. Run the initial push from HOST1 — this populates HOST2's empty shares: + +```bash +bash /mnt/user/appdata/unraid_scripts/Orchestrators/daily_sync_maintenance.sh --log +``` + +5. Once complete, scheduled runs take over automatically. + +--- + +## Naming Consistency — Required + +The ecosystem is built on the assumption that containers and shares have identical names on both servers. This is not optional — it is what makes one codebase work on both servers without modification. + +``` +Container names must match exactly: + Emby ← HOST1 and HOST2 + NginxProxyManager ← HOST1 and HOST2 + Mariadb-Authelia ← HOST1 and HOST2 + +Share paths must match exactly: + /mnt/user/Movies ← HOST1 and HOST2 + /mnt/user/Tv_Shows ← HOST1 and HOST2 +``` + +If a container or share has a different name on one server the script skips it gracefully — but it will not do what you expect. Diverge from consistent naming and every script that touches containers or shares needs custom logic for each server. Keep naming consistent and one codebase covers both servers automatically. --- ## Troubleshooting ### SSH connection refused -- Verify SSH is enabled on the target server (Step 3) -- Check the correct key is referenced in `Master.conf` -- Confirm the Tailscale IP resolves: `tailscale ip -4 HOSTNAME` +- Verify SSH is enabled (Step 2) +- Confirm the correct key is referenced in `Master.conf` +- Test Tailscale: `tailscale ip -4 HOSTNAME` ### Pre-flight aborts on rootfs -- Remote rootfs is above `ROOTFS_WARN` threshold -- Check if the remote array is started and drives are mounted -- `df /` on the remote to see current usage +- Remote rootfs above `ROOTFS_WARN` threshold +- Check remote array is started and drives are mounted +- Run `df /` on the remote to see current usage ### Pre-flight aborts on empty share - Share exists but has no content — drives may not be mounted - Run `recreate_shares.sh` if setting up fresh -- Check array status on the remote server - -### Pre-flight aborts on disk check -- One or more disks backing the share are not mounted -- Check **Main → Array Devices** on the remote for offline disks -- Verify disk assignments are correct after any hardware changes - -### Script not found -- Verify the repo was cloned to `/mnt/user/appdata/unraid_scripts/` -- Check scripts are executable: `chmod +x /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh` ### Containers not stopping/starting -- Verify container names in `Master.conf` match exactly what Docker shows -- Check SSH key has access to run docker commands on the remote +- Verify container names in `Master.conf` match Docker exactly — case sensitive - Test manually: `ssh -i /root/.ssh/KEY root@REMOTE_IP "docker ps"` +### Script not found +- Verify repo was cloned to `/mnt/user/appdata/unraid_scripts/` +- Make scripts executable: `find /mnt/user/appdata/unraid_scripts -name "*.sh" -exec chmod +x {} \;` + +### Profile not matching +- Profile key is matched by directory basename lowercased +- `/mnt/user/appdata-Failover/Arrs_Stack` → basename `Arrs_Stack` → key `arrs_stack` +- Override with `--profile=name` if basename doesn't match + --- ## Available Flags -All scripts support the following flags: +All scripts support: | Flag | Description | |---|---| -| `--dry-run` or `-n` | Run without making any changes | +| `--dry-run` | Run without making any changes | | `--log` | Enable verbose logging output | -| `--no-log` | Disable logging (overrides Master.conf) | +| `--no-log` | Disable logging | | `--status` | Print resolved configuration and exit | -Example: - ```bash -# Preview what would be synced without transferring anything +# Preview what would be synced bash rsync.sh /mnt/user/Movies --dry-run --log -# Check what profile and settings resolved for a share +# Check resolved profile settings bash rsync.sh /mnt/user/appdata-Failover/Arrs_Stack --status + +# Test daily orchestrator without changes +bash daily_sync_maintenance.sh --dry-run ``` --- @@ -436,16 +560,69 @@ bash rsync.sh /mnt/user/appdata-Failover/Arrs_Stack --status ``` Unraid_Scripts/ -├── Master.conf # All user configuration — edit this file only -├── common.sh # Shared library — functions used by all scripts -├── Rsync/ -│ └── rsync.sh # Core rsync script — called per share +├── Master.conf # All user configuration — edit this file only +├── common.sh # Shared library — functions used by all scripts +│ ├── Orchestrators/ -│ └── daily_sync.sh # Daily media sync orchestrator +│ ├── array_start.sh # Single array-start entry point +│ ├── daily_sync_maintenance.sh # Daily maintenance window orchestrator +│ ├── weekly_sync_maintenance.sh # Weekly maintenance window orchestrator +│ ├── media_management.sh # Permissions + cleaners + arr cleanup +│ └── transcode_management.sh # Transcode cleanup + manager +│ +├── Rsync/ +│ └── rsync.sh # Core rsync script — called per share +│ +├── Failover/ +│ ├── failover.sh # Mutual container failover — continuous loop +│ ├── failover_test.sh # Controlled failover simulation +│ └── failover_state_reset.sh # Reset failover state manually +│ +├── Docker_Essentials/ +│ ├── docker_watchdog.sh # Two-tier container monitor — continuous loop +│ ├── docker_daily_restart.sh # Daily container restarts +│ ├── docker_weekly_restart.sh # Weekly container restarts +│ └── docker_network_connect.sh # Connect containers to extra networks +│ +├── unRAID_Essentials/ +│ ├── system_watchdog.sh # System health monitor — continuous loop +│ ├── ramdisk_setup.sh # Creates ramdisk + symlink at array start +│ ├── docker_syslog_filter.sh # Suppress veth log noise +│ ├── php_fpm_max_children.sh # WebGUI performance tuning +│ ├── server_reboot.sh # Graceful scheduled reboot +│ ├── mover_stop.sh # Stop mover cleanly +│ ├── clear_logs.sh # Weekly log cleanup +│ ├── webgui_restart.sh # nginx + emhttp restart escalation +│ └── git_pull_execute.sh # Pull latest scripts from Gitea +│ +├── Media/ +│ ├── media_shares_permissions.sh # Apply permissions to media shares +│ ├── media_cleaner.sh # Remove junk files from media shares +│ ├── lidarr_cleanup.sh # Remove orphaned music files +│ ├── sonarr_cleanup.sh # Remove orphaned TV files +│ ├── radarr_cleanup.sh # Remove orphaned movie files +│ └── arrs_failed_stalled_recovery.sh # Blocklist + re-search failed imports +│ +├── Transcodes/ +│ ├── transcode_manager.sh # Symlink direction management +│ ├── transcode_cleanup.sh # Remove old inactive transcode files +│ └── ramdisk_setup.sh # (also in unRAID_Essentials — symlinked) +│ +├── Monitors/ +│ ├── cert_monitor.sh # SSL certificate expiry monitoring +│ ├── backup_verify.sh # Checksum verification against remote +│ ├── smart_health.sh # Drive SMART attribute monitoring +│ ├── zfs_memory_snapshot.sh # Weekly ZFS health + memory report +│ ├── bandwidth_monitor.sh # Rsync transfer logging + weekly summary +│ ├── weekly_health_digest.sh # Aggregated health digest email +│ ├── emby_session_report.sh # Weekly Emby usage statistics +│ └── emby_database_repair.sh # Emby SQLite database repair +│ └── Tools/ - └── recreate_shares.sh # Share directory recreation from cfg files -``` - ---- - -*Guide version aligned with common.sh v1.6* \ No newline at end of file + ├── recreate_shares.sh # Recreate share directories from cfg files + ├── bulk_permissions_repair.sh # One-shot permission repair + ├── watchdog_skip_list_manager.sh # Manage docker watchdog skip list + ├── rsync_stop.sh # Stop active rsync jobs cleanly + ├── user_scripts_stop.sh # Stop running user scripts + └── container_data_export.sh # Export container configuration +``` \ No newline at end of file diff --git a/unRAID_Essentials/system_watchdog.sh b/unRAID_Essentials/system_watchdog.sh index 0ae5044..4b32e01 100644 --- a/unRAID_Essentials/system_watchdog.sh +++ b/unRAID_Essentials/system_watchdog.sh @@ -3,6 +3,7 @@ # --------------------------------- System Watchdog -------------------------------------------- # ----------------------------------------------------------------------------------------------- # Last line of defense — reboots the system cleanly if it is about to become unstable. +# Runs continuously as a background process — started by array_start.sh at array start. # Works alongside docker_watchdog.sh which handles container-level healing first. # # Checks (all toggleable in Master.conf): @@ -16,6 +17,432 @@ # Docker daemon — unresponsive daemon means containers cannot be managed # Required containers — stopped containers that should be running (after watchdog skip list) # +# Abort conditions (toggleable): +# ZFS pool unhealthy — reboot with bad pool risks data loss +# Parity running — aborting parity is better than crashing mid-check +# Mover running — aborting move is better than crashing mid-move +# +# Reboot loop protection: +# Tracks reboot timestamps in persistent log on /boot/ +# Rolling window — old entries purge automatically +# If reboot count hits limit in window → shutdown instead of reboot +# +# Continuous loop: +# Checks run every SYSTEM_WATCHDOG_INTERVAL seconds (default 300 = 5min) +# Master.conf re-sourced each cycle — config changes picked up without restart +# Silent when healthy — only verbose when trigger or reboot +# Clean shutdown on SIGTERM/SIGINT — sent by array stop +# +# All configuration in Master.conf under System Watchdog section. +# Supports --dry-run to show triggered conditions without rebooting. +# ----------------------------------------------------------------------------------------------- + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "$SCRIPT_DIR/../Master.conf" +source "$SCRIPT_DIR/../common.sh" + +parse_args "$@" + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_GEAR Setup — runs once at start ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_GEAR Setup ━━━" + +if [[ "$EUID" -ne 0 ]]; then + error "Must be run as root" + exit 1 +fi + +success "Running as root" + +acquire_lock + +TOTAL_CORES=$(nproc) + +# Ensure state and persistent files exist +touch "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null || { + error "Cannot create state file: $SYS_WATCHDOG_STATE_FILE" + exit 1 +} + +touch "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null || { + error "Cannot create reboot log: $SYS_WATCHDOG_REBOOT_LOG" + exit 1 +} + +touch "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null || { + error "Cannot create failed container list: $SYS_WATCHDOG_FAILED_FILE" + exit 1 +} + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_SUMMARY Status ━━━ +# ----------------------------------------------------------------------------------------------- +if [[ "$SHOW_STATUS" == true ]]; then + echo "" + echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" + echo "$ICON_HEALTH rootfs threshold: ${SYS_WATCHDOG_ROOTFS_PCT}% toggle: $SYS_WATCHDOG_CHECK_ROOTFS" + echo "$ICON_HEALTH log threshold: ${SYS_WATCHDOG_LOG_PCT}% toggle: $SYS_WATCHDOG_CHECK_LOG" + echo "$ICON_MEM RAM threshold: ${SYS_WATCHDOG_MEM_GB}GB free toggle: $SYS_WATCHDOG_CHECK_RAM" + echo "$ICON_ZFS ARC pinned: ${SYS_WATCHDOG_ARC_PINNED_PCT}% toggle: $SYS_WATCHDOG_CHECK_ARC" + echo "$ICON_ZFS ARC release: ${SYS_WATCHDOG_ARC_RELEASE_PCT}%" + echo "$ICON_GEAR CPU temp max: ${SYS_WATCHDOG_CPU_TEMP_MAX}°C toggle: $SYS_WATCHDOG_CHECK_CPU_TEMP" + echo "$ICON_GEAR Load multiplier: ${SYS_WATCHDOG_LOAD_MULTIPLIER}x cores toggle: $SYS_WATCHDOG_CHECK_LOAD" + echo "$ICON_GEAR Zombie limit: ${SYS_WATCHDOG_ZOMBIE_LIMIT} toggle: $SYS_WATCHDOG_CHECK_ZOMBIES" + echo "$ICON_CONTAINERS Docker daemon: toggle: $SYS_WATCHDOG_CHECK_DOCKER_DAEMON" + echo "$ICON_CONTAINERS Containers: toggle: $SYS_WATCHDOG_CHECK_CONTAINERS" + echo "$ICON_SHIELD Strike limit: $SYS_WATCHDOG_STRIKE_LIMIT" + echo "$ICON_TIME Interval: ${SYSTEM_WATCHDOG_INTERVAL}s" + echo "$ICON_REBOOT_SMART Reboot limit: $SYS_WATCHDOG_REBOOT_LIMIT in ${SYS_WATCHDOG_REBOOT_WINDOW_HRS}hrs" + echo "$ICON_ZFS Abort ZFS unhealthy: $SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY" + echo "$ICON_GEAR Abort on parity: $SYS_WATCHDOG_ABORT_ON_PARITY" + echo "$ICON_MOVER Abort on mover: $SYS_WATCHDOG_ABORT_ON_MOVER" + echo "$ICON_GEAR Dry Run: $DRY_RUN" + echo "━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no reboot will be executed" + +# ----------------------------------------------------------------------------------------------- +# STATE HELPERS — defined once, used every cycle +# ----------------------------------------------------------------------------------------------- + +get_strikes() { + local key="$1" + grep -E "^${key}:" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | cut -d':' -f2 +} + +set_strikes() { + local key="$1" count="$2" + grep -vE "^${key}:" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null > "${SYS_WATCHDOG_STATE_FILE}.tmp" + echo "${key}:${count}" >> "${SYS_WATCHDOG_STATE_FILE}.tmp" + mv "${SYS_WATCHDOG_STATE_FILE}.tmp" "$SYS_WATCHDOG_STATE_FILE" +} + +increment_strikes() { + local key="$1" + local current + current=$(get_strikes "$key") + [[ -z "$current" ]] && current=0 + ((current++)) + set_strikes "$key" "$current" + echo "$current" +} + +reset_strikes() { + local key="$1" + set_strikes "$key" 0 +} + +purge_old_reboots() { + local now + now=$(date +%s) + local cutoff=$(( now - SYS_WATCHDOG_REBOOT_WINDOW )) + grep -v "^$" "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null | while IFS= read -r ts; do + [[ "$ts" -gt "$cutoff" ]] && echo "$ts" + done > "${SYS_WATCHDOG_REBOOT_LOG}.tmp" + mv "${SYS_WATCHDOG_REBOOT_LOG}.tmp" "$SYS_WATCHDOG_REBOOT_LOG" +} + +count_recent_reboots() { + purge_old_reboots + grep -c "." "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null || echo 0 +} + +log_reboot() { + date +%s >> "$SYS_WATCHDOG_REBOOT_LOG" +} + +check_abort_conditions() { + local should_abort=false + + if command -v zpool >/dev/null 2>&1; then + local unhealthy + unhealthy=$(zpool list -H -o health 2>/dev/null | grep -v ONLINE || true) + if [[ -n "$unhealthy" ]]; then + if [[ "$SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY" == true ]]; then + error "$ICON_ZFS ZFS pool unhealthy — aborting reboot" + notify "System watchdog aborted reboot on $(hostname) — ZFS pool unhealthy" "System Watchdog" "warning" + should_abort=true + else + warn "$ICON_ZFS ZFS pool unhealthy — continuing reboot" + fi + fi + fi + + if [[ -f /var/local/emhttp/parity-date.txt ]]; then + if grep -q "progress" /var/local/emhttp/parity-date.txt 2>/dev/null; then + if [[ "$SYS_WATCHDOG_ABORT_ON_PARITY" == true ]]; then + error "$ICON_DISK Parity check running — aborting reboot" + notify "System watchdog aborted reboot on $(hostname) — parity running" "System Watchdog" "warning" + should_abort=true + else + warn "$ICON_DISK Parity check running — continuing reboot" + fi + fi + fi + + if pgrep -f "mover" >/dev/null 2>&1; then + if [[ "$SYS_WATCHDOG_ABORT_ON_MOVER" == true ]]; then + error "$ICON_MOVER Mover running — aborting reboot" + notify "System watchdog aborted reboot on $(hostname) — mover running" "System Watchdog" "warning" + should_abort=true + else + warn "$ICON_MOVER Mover running — continuing reboot" + fi + fi + + [[ "$should_abort" == true ]] && return 1 + return 0 +} + +run_strike_check() { + local key="$1" triggered="$2" description="$3" + if [[ "$triggered" == true ]]; then + local strikes + strikes=$(increment_strikes "$key") + warn "$description — strike $strikes/$SYS_WATCHDOG_STRIKE_LIMIT" + if (( strikes >= SYS_WATCHDOG_STRIKE_LIMIT )); then + error "$description hit strike limit — reboot triggered" + reset_strikes "$key" + return 0 + fi + else + local current + current=$(get_strikes "$key") + if [[ -n "$current" && "$current" -gt 0 ]]; then + reset_strikes "$key" + fi + fi + return 1 +} + +do_reboot() { + local triggers=("$@") + + if ! check_abort_conditions; then + return + fi + + RECENT_REBOOTS=$(count_recent_reboots) + info "Recent reboots in ${SYS_WATCHDOG_REBOOT_WINDOW_HRS}hr window: $RECENT_REBOOTS / $SYS_WATCHDOG_REBOOT_LIMIT" + + if [[ "$RECENT_REBOOTS" -ge "$SYS_WATCHDOG_REBOOT_LIMIT" ]]; then + error "Reboot limit hit — shutting down instead" + notify "Reboot loop detected on $(hostname) — shutting down after $RECENT_REBOOTS reboots — conditions: ${triggers[*]}" "System Watchdog" "warning" + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would shutdown now" + return + fi + sync + /sbin/poweroff + return + fi + + notify "System watchdog reboot triggered on $(hostname) — conditions: ${triggers[*]}" "System Watchdog" "warning" + + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — reboot sequence would begin now" + return + fi + + log_reboot + + info "Shutting down VMs..." + if command -v virsh >/dev/null 2>&1; then + for VM in $(virsh list --name 2>/dev/null); do + [[ -z "$VM" ]] && continue + virsh shutdown "$VM" >/dev/null 2>&1 + done + sleep 30 + fi + + info "Stopping Docker containers..." + if command -v docker >/dev/null 2>&1; then + docker ps -q | xargs -r docker stop >/dev/null 2>&1 + fi + + info "Stopping User Scripts..." + pkill -f "/tmp/user.scripts" 2>/dev/null || true + + info "Syncing disks..." + sync + + echo "" + echo "$ICON_REBOOT_SMART Rebooting system NOW..." + sleep 5 + /sbin/reboot +} + +# ----------------------------------------------------------------------------------------------- +# CLEAN SHUTDOWN — trap SIGTERM/SIGINT from array stop +# ----------------------------------------------------------------------------------------------- +WATCHDOG_RUNNING=true + +cleanup() { + echo "" + info "System watchdog received shutdown signal — stopping cleanly" + WATCHDOG_RUNNING=false + exit 0 +} + +trap cleanup SIGTERM SIGINT + +# ----------------------------------------------------------------------------------------------- +# ━━━ CONTINUOUS MONITORING LOOP ━━━ +# ----------------------------------------------------------------------------------------------- +info "System watchdog started — checking every ${SYSTEM_WATCHDOG_INTERVAL}s" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + +CYCLE=0 + +while [[ "$WATCHDOG_RUNNING" == true ]]; do + ((CYCLE++)) + + # Re-source Master.conf each cycle — picks up config changes without restart + source "$SCRIPT_DIR/../Master.conf" + SYS_WATCHDOG_REBOOT_WINDOW=$(( SYS_WATCHDOG_REBOOT_WINDOW_HRS * 3600 )) + TOTAL_CORES=$(nproc) + + # Per-cycle triggers — cleared each iteration + TRIGGERS=() + + # ── rootfs usage ───────────────────────────────────────────────────────────────────────── + if [[ "$SYS_WATCHDOG_CHECK_ROOTFS" == true ]]; then + ROOTFS_USED=$(df / --output=pcent | tail -1 | tr -d ' %') + TRIGGERED=false + [[ "$ROOTFS_USED" -ge "$SYS_WATCHDOG_ROOTFS_PCT" ]] && TRIGGERED=true + run_strike_check "rootfs" "$TRIGGERED" "rootfs ${ROOTFS_USED}%" && \ + TRIGGERS+=("rootfs=${ROOTFS_USED}%") + fi + + # ── /var/log usage ──────────────────────────────────────────────────────────────────────── + if [[ "$SYS_WATCHDOG_CHECK_LOG" == true ]]; then + LOG_USED=$(df -P /var/log | awk 'NR==2 {print $5}' | tr -d '%') + TRIGGERED=false + [[ "$LOG_USED" -ge "$SYS_WATCHDOG_LOG_PCT" ]] && TRIGGERED=true + run_strike_check "log" "$TRIGGERED" "/var/log ${LOG_USED}%" && \ + TRIGGERS+=("log=${LOG_USED}%") + fi + + # ── Free RAM ───────────────────────────────────────────────────────────────────────────── + if [[ "$SYS_WATCHDOG_CHECK_RAM" == true ]]; then + MEM_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo) + MEM_GB=$((MEM_KB / 1024 / 1024)) + TRIGGERED=false + [[ "$MEM_GB" -lt "$SYS_WATCHDOG_MEM_GB" ]] && TRIGGERED=true + run_strike_check "ram" "$TRIGGERED" "RAM ${MEM_GB}GB free" && \ + TRIGGERS+=("low_ram=${MEM_GB}GB") + fi + + # ── ZFS ARC ────────────────────────────────────────────────────────────────────────────── + if [[ "$SYS_WATCHDOG_CHECK_ARC" == true ]] && [[ -f /proc/spl/kstat/zfs/arcstats ]]; then + ARC_SIZE=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats) + ARC_MAX=$(awk '/^c_max / {print $3}' /proc/spl/kstat/zfs/arcstats) + ARC_PCT=$(( ARC_SIZE * 100 / ARC_MAX )) + TRIGGERED=false + if [[ "$ARC_PCT" -ge "$SYS_WATCHDOG_ARC_PINNED_PCT" ]]; then + sync; echo 3 > /proc/sys/vm/drop_caches; sleep 5 + ARC_AFTER=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats) + ARC_AFTER_PCT=$(( ARC_AFTER * 100 / ARC_MAX )) + [[ "$ARC_AFTER_PCT" -ge "$SYS_WATCHDOG_ARC_RELEASE_PCT" ]] && TRIGGERED=true + fi + run_strike_check "arc" "$TRIGGERED" "ZFS ARC pinned" && \ + TRIGGERS+=("arc_pinned=${ARC_PCT}%") + fi + + # ── CPU temperature ────────────────────────────────────────────────────────────────────── + if [[ "$SYS_WATCHDOG_CHECK_CPU_TEMP" == true ]]; then + CPU_TEMP="" + if command -v sensors >/dev/null 2>&1; then + CPU_TEMP=$(sensors 2>/dev/null | grep -i "Package id 0\|Tctl\|CPU Temp" | \ + awk '{print $NF}' | tr -d '+°C' | head -1) + fi + if [[ -n "$CPU_TEMP" ]]; then + CPU_TEMP_INT=$(printf "%.0f" "$CPU_TEMP") + TRIGGERED=false + [[ "$CPU_TEMP_INT" -ge "$SYS_WATCHDOG_CPU_TEMP_MAX" ]] && TRIGGERED=true + run_strike_check "cpu_temp" "$TRIGGERED" "CPU temp ${CPU_TEMP_INT}°C" && \ + TRIGGERS+=("cpu_temp=${CPU_TEMP_INT}C") + fi + fi + + # ── Load average ───────────────────────────────────────────────────────────────────────── + if [[ "$SYS_WATCHDOG_CHECK_LOAD" == true ]]; then + LOAD=$(awk '{print $1}' /proc/loadavg) + LOAD_INT=$(printf "%.0f" "$LOAD") + LOAD_THRESHOLD=$(( TOTAL_CORES * SYS_WATCHDOG_LOAD_MULTIPLIER )) + TRIGGERED=false + [[ "$LOAD_INT" -ge "$LOAD_THRESHOLD" ]] && TRIGGERED=true + run_strike_check "load" "$TRIGGERED" "load avg ${LOAD}" && \ + TRIGGERS+=("load=${LOAD}") + fi + + # ── Zombie processes ───────────────────────────────────────────────────────────────────── + if [[ "$SYS_WATCHDOG_CHECK_ZOMBIES" == true ]]; then + ZOMBIE_COUNT=$(ps aux | awk '{print $8}' | grep -c "^Z$" || echo 0) + TRIGGERED=false + [[ "$ZOMBIE_COUNT" -ge "$SYS_WATCHDOG_ZOMBIE_LIMIT" ]] && TRIGGERED=true + run_strike_check "zombies" "$TRIGGERED" "zombies ${ZOMBIE_COUNT}" && \ + TRIGGERS+=("zombies=${ZOMBIE_COUNT}") + fi + + # ── Docker daemon ──────────────────────────────────────────────────────────────────────── + if [[ "$SYS_WATCHDOG_CHECK_DOCKER_DAEMON" == true ]]; then + TRIGGERED=false + ! timeout 10 docker ps >/dev/null 2>&1 && TRIGGERED=true + run_strike_check "docker_daemon" "$TRIGGERED" "Docker daemon unresponsive" && \ + TRIGGERS+=("docker_daemon") + fi + + # ── Required containers from skip list ─────────────────────────────────────────────────── + if [[ "$SYS_WATCHDOG_CHECK_CONTAINERS" == true ]] && [[ -s "$SYS_WATCHDOG_FAILED_FILE" ]]; then + FAILED_CONTAINERS=() + while IFS= read -r container; do + [[ -z "$container" ]] && continue + STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null || echo "unknown") + [[ "$STATUS" != "true" ]] && FAILED_CONTAINERS+=("$container") + done < "$SYS_WATCHDOG_FAILED_FILE" + + if [[ ${#FAILED_CONTAINERS[@]} -gt 0 ]]; then + run_strike_check "failed_containers" "true" "required containers stopped" && \ + TRIGGERS+=("containers=${FAILED_CONTAINERS[*]}") + fi + fi + + # ── Evaluate triggers ──────────────────────────────────────────────────────────────────── + if [[ ${#TRIGGERS[@]} -gt 0 ]]; then + echo "" + echo "━━━ $ICON_REBOOT_SMART System Watchdog — Cycle $CYCLE — $(date '+%Y-%m-%d %H:%M:%S') ━━━" + for t in "${TRIGGERS[@]}"; do + echo " $ICON_REBOOT_SMART $t" + done + echo "" + do_reboot "${TRIGGERS[@]}" + else + log "Cycle $CYCLE — system healthy ($(date '+%H:%M:%S'))" + fi + + # Sleep until next cycle — interruptible by SIGTERM + sleep "$SYSTEM_WATCHDOG_INTERVAL" & + wait $! + +done +# +# Checks (all toggleable in Master.conf): +# rootfs usage — high rootfs fills rapidly when array is down, crash imminent +# /var/log usage — log spam can fill rootfs, indicates something is broken +# free RAM — critically low RAM means OOM or swap imminent +# ZFS ARC pinned — ARC not releasing after reclaim means memory is stuck +# CPU temperature — sustained tjmax causes throttling or kernel panic +# load average — sustained high load means something is stuck or runaway +# zombie processes — large zombie count indicates serious process management failure +# Docker daemon — unresponsive daemon means containers cannot be managed +# Required containers — stopped containers that should be running (after watchdog skip list) +# # Abort conditions (toggleable — true = abort, false = reboot anyway): # ZFS pool unhealthy — reboot with bad pool risks data loss # Parity running — aborting parity is better than crashing mid-check diff --git a/user_script_plug-in.sh b/user_script_plug-in.sh index ddfa517..70076fd 100644 --- a/user_script_plug-in.sh +++ b/user_script_plug-in.sh @@ -65,10 +65,13 @@ # │ └── README-Monitors.md # │ # ├── Orchestrators/ -# │ ├── media_shares_sync.sh # Runs all daily media share syncs sequentially -# │ ├── critical_shares_maintenance.sh # Maintenance window — stop, update, sync, restart -# │ ├── media_management.sh # Runs permissions + cleaners + arr cleanup -# │ ├── transcode_management.sh # Runs cleanup then manager every 3min + daily stats +# │ ├── array_start.sh # Single entry point — launches all array-start scripts +# │ ├── daily_sync_maintenance.sh # Daily — git pull, media sync, media mgmt, docker restart +# │ ├── weekly_sync_maintenance.sh # Weekly — critical sync + updates, docker weekly restart +# │ ├── daily_sync_maintenance.sh # Media shares sync both directions +# │ ├── weekly_sync_maintenance.sh # Clean sync + container updates (Emby + auth stack) +# │ ├── media_management.sh # Permissions + cleaners + arr cleanup — run manually +# │ ├── transcode_management.sh # Cleanup then manager every 3min + daily stats # │ └── README-Orchestrators.md # │ # ├── Rsync/ @@ -160,13 +163,16 @@ #/mnt/user/appdata/unraid_scripts/Monitors/zfs_memory_snapshot.sh # # ━━━ Orchestrators ━━━ -# Orchestrators run their child scripts in the correct order. -# Individual scripts below are still accessible for manual runs or testing. +# array_start.sh is the single User Scripts entry for array startup. +# All other orchestrators are scheduled via cron — not started at array start. # -#/mnt/user/appdata/unraid_scripts/Orchestrators/media_shares_sync.sh -#/mnt/user/appdata/unraid_scripts/Orchestrators/critical_shares_maintenance.sh -#/mnt/user/appdata/unraid_scripts/Orchestrators/media_management.sh +#/mnt/user/appdata/unraid_scripts/Orchestrators/array_start.sh +# ^^ set to: At Startup of Array +#/mnt/user/appdata/unraid_scripts/Orchestrators/daily_sync_maintenance.sh +#/mnt/user/appdata/unraid_scripts/Orchestrators/weekly_sync_maintenance.sh #/mnt/user/appdata/unraid_scripts/Orchestrators/transcode_management.sh +# ^^ media_management.sh is absorbed into daily_sync_maintenance.sh via MEDIA_MANAGEMENT_JOBS +# ^^ run manually: bash Orchestrators/media_management.sh --dry-run # # ━━━ Rsync — Appdata Profiles ━━━ #/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-Failover/Arrs_Stack @@ -175,7 +181,7 @@ #/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Media_Server/Emby --profile=emby-failover # ^^ schedule every 30-60min — dirty sync, Emby running, critical data only #/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Media_Server/Emby -# ^^ do NOT schedule — called by critical_shares_maintenance.sh Sunday 2:30am only +# ^^ do NOT schedule — called by weekly_sync_maintenance.sh Sunday 2:30am only #/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-Failover/Gmer4Lfe # # ━━━ Rsync — Individual Media Shares (ad hoc) ━━━ @@ -202,8 +208,8 @@ #/mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_network_connect.sh # # ━━━ Media ━━━ -# media_management.sh runs all of these in order — individual entries for manual use. -# Always --dry-run --log first on arr cleanup scripts before running live. +# media_management.sh is absorbed into daily_sync_maintenance.sh via MEDIA_MANAGEMENT_JOBS. +# Scripts below available for manual runs only — always --dry-run first on arr cleanup scripts. # #/mnt/user/appdata/unraid_scripts/Media/media_shares_permissions.sh #/mnt/user/appdata/unraid_scripts/Media/media_cleaner.sh anime --dry-run @@ -286,37 +292,40 @@ # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ # # ━━━ At Startup of Array ━━━ -# failover.sh — background task (continuous loop) -# ramdisk_setup.sh — creates ramdisk before Emby starts -# docker_network_connect.sh — connect containers to extra networks -# docker_syslog_filter.sh — suppress veth noise before logs fill -# php_fpm_max_children.sh — WebGUI performance tuning +# array_start.sh — single entry point, launches everything below +# configure what runs in ARRAY_START_SCRIPTS in Master.conf +# +# Launched by array_start.sh: +# ramdisk_setup.sh — creates ramdisk before Emby starts (one-shot) +# docker_syslog_filter.sh — suppress veth noise before logs fill (one-shot) +# php_fpm_max_children.sh — WebGUI performance tuning (one-shot) +# docker_network_connect.sh — connect containers to extra networks (one-shot) +# system_watchdog.sh — system health monitor (continuous) +# docker_watchdog.sh — container health monitor (continuous) +# failover.sh — mutual failover (continuous) # # ━━━ Frequent (cron) ━━━ # */3 * * * * transcode_management.sh (cleanup + manager every 3min) -# */30 * * * * rsync.sh /mnt/user/Media_Server/Emby (emby-failover dirty sync) +# */30 * * * * rsync.sh /mnt/user/Media_Server/Emby (emby-failover dirty sync) # --profile=emby-failover -# */10 * * * * webgui_restart.sh -# */15 * * * * docker_watchdog.sh +# */6 * * * * arrs_failed_stalled_recovery.sh (blocklist + re-search) # */15 * * * * system_watchdog.sh # # ━━━ Daily ━━━ -# 0 1 * * * media_shares_sync.sh (media shares both directions) -# 0 2 * * * media_management.sh (permissions + cleaners + arrs) -# 0 3 * * * docker_daily_restart.sh -# 0 5 * * * arrs_failed_stalled_recovery.sh (blocklist + re-search failed imports + stalled) -# 0 8 * * * weekly_health_digest.sh (profile controls if it sends) +# 0 1 * * * daily_sync_maintenance.sh (git pull + media sync + media_management + docker restart) +# 0 5 * * * arrs_failed_stalled_recovery.sh (blocklist + re-search failed imports + stalled) +# 0 8 * * * weekly_health_digest.sh (profile controls if it sends) # # ━━━ Rsync profiles — schedule individually ━━━ # Arrs_Stack — daily or every few days (arr databases change on every download) -# Critical-Data — handled by critical_shares_maintenance.sh — no separate schedule needed +# Critical-Data — handled by weekly_sync_maintenance.sh — no separate schedule needed # Important-Data — daily (NextCloud file changes) # Gmer4Lfe — daily or weekly (personal appdata, rarely changes) -# Emby — handled by critical_shares_maintenance.sh — no separate schedule needed +# Emby — handled by weekly_sync_maintenance.sh — no separate schedule needed # emby-failover — every 30-60min via frequent cron above # # ━━━ Weekly — Sunday morning block ━━━ -# 30 2 * * 0 critical_shares_maintenance.sh (clean Emby + auth stack — cache resets weekly) +# 30 2 * * 0 weekly_sync_maintenance.sh (clean Emby + auth stack — cache resets weekly) # 0 3 * * 0 docker_weekly_restart.sh # 0 5 * * 0 clear_logs.sh # 0 6 * * 0 zfs_memory_snapshot.sh