created loop for both watchdogs and consolidated orch lists
This commit is contained in:
@@ -2,7 +2,576 @@
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- Docker Watchdog --------------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Two-tier self-healing container monitoring system.
|
||||
# Two-tier self-healing container monitoring system — runs continuously as a background process.
|
||||
# Started by array_start.sh at array start — runs until array stops or SIGTERM received.
|
||||
#
|
||||
# Tier 1 — Strict monitoring (configured containers only)
|
||||
# Memory hard limits — immediate restart if exceeded
|
||||
# CPU thresholds — strike system, restart after CPU_FAIL_LIMIT strikes
|
||||
# HTTP responsiveness — strike system, restart after RESP_FAIL_LIMIT strikes
|
||||
# Required containers — must always be running, strike system with skip list
|
||||
#
|
||||
# Tier 2 — Global health scan (all running containers)
|
||||
# Unhealthy status — Docker HEALTHCHECK unhealthy → restart
|
||||
# OOM killed — kernel killed container → restart + notify
|
||||
# Crash loop detection — RestartCount climbing → notify, critical above limit
|
||||
# Dead containers — remove and restart
|
||||
# Unexpected exits — non-zero exit code → restart
|
||||
#
|
||||
# Cross-cutting intelligence:
|
||||
# Startup grace period — skip restarts while system is still booting
|
||||
# Dependency ordering — restart database before app
|
||||
# Restart loop protect — stop restarting after X restarts in X hours → skip list
|
||||
# Skip list auto-clear — clears when container recovers
|
||||
# Notification batching — one clean summary per cycle, not one ping per event
|
||||
# Quiet when healthy — only logs when something needs attention
|
||||
# Parity awareness — skips restarts during parity check
|
||||
#
|
||||
# Continuous loop:
|
||||
# Checks run every DOCKER_WATCHDOG_INTERVAL seconds (default 900 = 15min)
|
||||
# Clean shutdown on SIGTERM/SIGINT — sent by array stop
|
||||
# Variables scoped per-cycle — no state accumulation between cycles
|
||||
#
|
||||
# State files:
|
||||
# WATCHDOG_STATE_FILE — strike counts (/tmp — resets on reboot)
|
||||
# SYS_WATCHDOG_FAILED_FILE — persistent skip list (/boot — survives reboots)
|
||||
# WATCHDOG_CONTAINER_RESTART_LOG — restart history for loop detection
|
||||
#
|
||||
# All configuration in Master.conf under Docker Watchdog section.
|
||||
# Supports --dry-run and --status.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup — runs once at start ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Running as root"
|
||||
|
||||
acquire_lock
|
||||
|
||||
# Select correct per-host watchdog lists — done once at startup
|
||||
detect_hosts
|
||||
|
||||
if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then
|
||||
WATCHDOG_REQUIRED_CONTAINERS=("${HOST1_WATCHDOG_REQUIRED_CONTAINERS[@]}")
|
||||
declare -A WATCHDOG_CONTAINER_URLS
|
||||
for key in "${!HOST1_WATCHDOG_CONTAINER_URLS[@]}"; do
|
||||
WATCHDOG_CONTAINER_URLS["$key"]="${HOST1_WATCHDOG_CONTAINER_URLS[$key]}"
|
||||
done
|
||||
else
|
||||
WATCHDOG_REQUIRED_CONTAINERS=("${HOST2_WATCHDOG_REQUIRED_CONTAINERS[@]}")
|
||||
declare -A WATCHDOG_CONTAINER_URLS
|
||||
for key in "${!HOST2_WATCHDOG_CONTAINER_URLS[@]}"; do
|
||||
WATCHDOG_CONTAINER_URLS["$key"]="${HOST2_WATCHDOG_CONTAINER_URLS[$key]}"
|
||||
done
|
||||
fi
|
||||
|
||||
info "Watchdog running as: $LOCAL_SERVER_NAME"
|
||||
info "Check interval: ${DOCKER_WATCHDOG_INTERVAL}s"
|
||||
|
||||
if ! command -v docker >/dev/null 2>&1; then
|
||||
error "Docker not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Docker found"
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be restarted"
|
||||
|
||||
# Ensure state files exist
|
||||
touch "$WATCHDOG_STATE_FILE" "$WATCHDOG_CONTAINER_RESTART_LOG" \
|
||||
"$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Status ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_CONTAINERS Watched: ${!WATCHDOG_CONTAINERS[@]}"
|
||||
echo "$ICON_CONTAINERS Required: ${WATCHDOG_REQUIRED_CONTAINERS[*]}"
|
||||
echo "$ICON_WATCHDOG Scan all: $WATCHDOG_SCAN_ALL"
|
||||
echo "$ICON_WATCHDOG Interval: ${DOCKER_WATCHDOG_INTERVAL}s"
|
||||
echo "$ICON_WATCHDOG Startup grace: ${WATCHDOG_STARTUP_GRACE}s"
|
||||
echo "$ICON_WATCHDOG Restart limit: $WATCHDOG_CONTAINER_RESTART_LIMIT in ${WATCHDOG_CONTAINER_RESTART_WINDOW}h"
|
||||
echo "$ICON_WATCHDOG Batch notify: $WATCHDOG_BATCH_NOTIFY"
|
||||
echo "$ICON_WATCHDOG Ignore list: ${WATCHDOG_SCAN_IGNORE[*]:-none}"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
UPTIME_S=$(awk '{print int($1)}' /proc/uptime)
|
||||
echo "$ICON_TIME System uptime: $(format_duration $UPTIME_S)"
|
||||
[[ "$UPTIME_S" -lt "$WATCHDOG_STARTUP_GRACE" ]] && \
|
||||
warn "Within startup grace period — restarts suppressed"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# HELPERS — defined once, used every cycle
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
get_strikes() {
|
||||
grep "^${1}:" "${2}" 2>/dev/null | cut -d: -f2 || echo "0"
|
||||
}
|
||||
|
||||
set_strikes() {
|
||||
local container="$1" count="$2" file="$3"
|
||||
if grep -q "^${container}:" "$file" 2>/dev/null; then
|
||||
sed -i "s/^${container}:.*/${container}:${count}/" "$file"
|
||||
else
|
||||
echo "${container}:${count}" >> "$file"
|
||||
fi
|
||||
}
|
||||
|
||||
is_skipped() {
|
||||
grep -q "^${1}$" "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null
|
||||
}
|
||||
|
||||
add_to_skip_list() {
|
||||
local container="$1" reason="$2"
|
||||
if ! is_skipped "$container"; then
|
||||
echo "$container" >> "$SYS_WATCHDOG_FAILED_FILE"
|
||||
error "$container added to skip list — $reason"
|
||||
queue_notify "$container added to skip list on $(hostname) — $reason — manual intervention needed" "critical"
|
||||
fi
|
||||
}
|
||||
|
||||
remove_from_skip_list() {
|
||||
sed -i "/^${1}$/d" "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null
|
||||
success "$1 removed from skip list — recovered"
|
||||
}
|
||||
|
||||
log_restart() {
|
||||
local container="$1"
|
||||
local now
|
||||
now=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
local cutoff
|
||||
cutoff=$(date -d "${WATCHDOG_CONTAINER_RESTART_WINDOW} hours ago" '+%Y-%m-%d %H:%M:%S')
|
||||
echo "${container}|${now}" >> "$WATCHDOG_CONTAINER_RESTART_LOG"
|
||||
local tmp="${WATCHDOG_CONTAINER_RESTART_LOG}.tmp"
|
||||
awk -F'|' -v cutoff="$cutoff" '$2 >= cutoff' \
|
||||
"$WATCHDOG_CONTAINER_RESTART_LOG" > "$tmp" && \
|
||||
mv "$tmp" "$WATCHDOG_CONTAINER_RESTART_LOG"
|
||||
}
|
||||
|
||||
get_restart_count() {
|
||||
local container="$1"
|
||||
local cutoff
|
||||
cutoff=$(date -d "${WATCHDOG_CONTAINER_RESTART_WINDOW} hours ago" '+%Y-%m-%d %H:%M:%S')
|
||||
awk -F'|' -v c="$container" -v cutoff="$cutoff" \
|
||||
'$1==c && $2>=cutoff' "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | wc -l
|
||||
}
|
||||
|
||||
dependencies_satisfied() {
|
||||
local container="$1"
|
||||
local deps="${WATCHDOG_DEPENDENCIES[$container]:-}"
|
||||
[[ -z "$deps" ]] && return 0
|
||||
for dep in $deps; do
|
||||
local status
|
||||
status=$(docker inspect -f '{{.State.Running}}' "$dep" 2>/dev/null)
|
||||
if [[ "$status" != "true" ]]; then
|
||||
warn "$container — dependency $dep is not running — skipping restart this cycle"
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
return 0
|
||||
}
|
||||
|
||||
safe_restart() {
|
||||
local container="$1" reason="$2"
|
||||
local restart_count
|
||||
restart_count=$(get_restart_count "$container")
|
||||
if [[ "$restart_count" -ge "$WATCHDOG_CONTAINER_RESTART_LIMIT" ]]; then
|
||||
add_to_skip_list "$container" \
|
||||
"restarted $restart_count times in ${WATCHDOG_CONTAINER_RESTART_WINDOW}h"
|
||||
return 2
|
||||
fi
|
||||
dependencies_satisfied "$container" || return 1
|
||||
local uptime_s
|
||||
uptime_s=$(awk '{print int($1)}' /proc/uptime)
|
||||
if [[ "$uptime_s" -lt "$WATCHDOG_STARTUP_GRACE" ]]; then
|
||||
warn "$container — within startup grace period — skipping restart"
|
||||
return 1
|
||||
fi
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would restart $container ($reason)"
|
||||
return 0
|
||||
fi
|
||||
info "Restarting $container ($reason) [restart $((restart_count + 1))/$WATCHDOG_CONTAINER_RESTART_LIMIT in window]..."
|
||||
if docker restart "$container" >/dev/null 2>&1; then
|
||||
success "$ICON_STARTED $container restarted"
|
||||
log_restart "$container"
|
||||
return 0
|
||||
else
|
||||
error "Failed to restart $container"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
queue_notify() {
|
||||
local message="$1" severity="${2:-warning}"
|
||||
NOTIFY_EVENTS+=("${severity}|${message}")
|
||||
log "Queued: $message"
|
||||
}
|
||||
|
||||
flush_notify() {
|
||||
[[ ${#NOTIFY_EVENTS[@]} -eq 0 ]] && return
|
||||
if [[ "$WATCHDOG_BATCH_NOTIFY" == "true" ]]; then
|
||||
local highest_severity="normal"
|
||||
local messages=()
|
||||
for event in "${NOTIFY_EVENTS[@]}"; do
|
||||
local sev="${event%%|*}" msg="${event#*|}"
|
||||
messages+=("$msg")
|
||||
[[ "$sev" == "critical" ]] && highest_severity="warning"
|
||||
[[ "$sev" == "warning" && "$highest_severity" == "normal" ]] && highest_severity="warning"
|
||||
done
|
||||
local summary
|
||||
summary=$(printf '%s. ' "${messages[@]}")
|
||||
notify "Docker Watchdog on $(hostname) — ${#NOTIFY_EVENTS[@]} event(s): $summary" \
|
||||
"Docker Watchdog" "$highest_severity"
|
||||
else
|
||||
for event in "${NOTIFY_EVENTS[@]}"; do
|
||||
local sev="${event%%|*}" msg="${event#*|}"
|
||||
[[ "$sev" == "critical" ]] && sev="warning"
|
||||
notify "$msg" "Docker Watchdog" "$sev"
|
||||
done
|
||||
fi
|
||||
NOTIFY_EVENTS=()
|
||||
}
|
||||
|
||||
is_parity_running() {
|
||||
grep -q "progress" /var/local/emhttp/parity-date.txt 2>/dev/null
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# CLEAN SHUTDOWN — trap SIGTERM/SIGINT from array stop
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
WATCHDOG_RUNNING=true
|
||||
|
||||
cleanup() {
|
||||
echo ""
|
||||
info "Docker watchdog received shutdown signal — stopping cleanly"
|
||||
WATCHDOG_RUNNING=false
|
||||
exit 0
|
||||
}
|
||||
|
||||
trap cleanup SIGTERM SIGINT
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ CONTINUOUS MONITORING LOOP ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
info "Docker watchdog started — checking every ${DOCKER_WATCHDOG_INTERVAL}s"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
CYCLE=0
|
||||
|
||||
while [[ "$WATCHDOG_RUNNING" == true ]]; do
|
||||
((CYCLE++))
|
||||
CYCLE_START=$(date +%s)
|
||||
|
||||
# Re-source Master.conf each cycle — picks up any config changes without restart
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
|
||||
# Rebuild per-host lists after re-source in case they changed
|
||||
if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then
|
||||
WATCHDOG_REQUIRED_CONTAINERS=("${HOST1_WATCHDOG_REQUIRED_CONTAINERS[@]}")
|
||||
for key in "${!HOST1_WATCHDOG_CONTAINER_URLS[@]}"; do
|
||||
WATCHDOG_CONTAINER_URLS["$key"]="${HOST1_WATCHDOG_CONTAINER_URLS[$key]}"
|
||||
done
|
||||
else
|
||||
WATCHDOG_REQUIRED_CONTAINERS=("${HOST2_WATCHDOG_REQUIRED_CONTAINERS[@]}")
|
||||
for key in "${!HOST2_WATCHDOG_CONTAINER_URLS[@]}"; do
|
||||
WATCHDOG_CONTAINER_URLS["$key"]="${HOST2_WATCHDOG_CONTAINER_URLS[$key]}"
|
||||
done
|
||||
fi
|
||||
|
||||
# Per-cycle variables — cleared each iteration, no accumulation
|
||||
NOTIFY_EVENTS=()
|
||||
T1_RESTARTS=0
|
||||
T1_WARNINGS=0
|
||||
T2_RESTARTS=0
|
||||
T2_WARNINGS=0
|
||||
|
||||
# Rebuild ignore map each cycle in case config was updated
|
||||
declare -A IGNORE_MAP
|
||||
for c in "${WATCHDOG_SCAN_IGNORE[@]}"; do
|
||||
[[ -n "$c" ]] && IGNORE_MAP["$c"]=1
|
||||
done
|
||||
|
||||
# Startup grace check
|
||||
UPTIME_SECONDS=$(awk '{print int($1)}' /proc/uptime)
|
||||
IN_GRACE_PERIOD=false
|
||||
if [[ "$UPTIME_SECONDS" -lt "$WATCHDOG_STARTUP_GRACE" ]]; then
|
||||
IN_GRACE_PERIOD=true
|
||||
fi
|
||||
|
||||
# Parity check — skip restarts during parity to avoid I/O interference
|
||||
if is_parity_running; then
|
||||
log "Parity check in progress — skipping restart actions this cycle"
|
||||
sleep "$DOCKER_WATCHDOG_INTERVAL"
|
||||
continue
|
||||
fi
|
||||
|
||||
# ── TIER 1 — Strict Monitoring ──────────────────────────────────────────────────────────
|
||||
|
||||
# Required containers
|
||||
if [[ ${#WATCHDOG_REQUIRED_CONTAINERS[@]} -gt 0 ]]; then
|
||||
for container in "${WATCHDOG_REQUIRED_CONTAINERS[@]}"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
|
||||
|
||||
if is_skipped "$container"; then
|
||||
if [[ "$STATUS" == "true" ]]; then
|
||||
remove_from_skip_list "$container"
|
||||
set_strikes "$container" 0 "$WATCHDOG_STATE_FILE"
|
||||
sed -i "/^${container}|/d" "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null
|
||||
queue_notify "$container recovered on $(hostname) — removed from skip list" "normal"
|
||||
else
|
||||
warn "$container — on skip list, manual intervention needed"
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ "$STATUS" == "true" ]]; then
|
||||
set_strikes "$container" 0 "$WATCHDOG_STATE_FILE"
|
||||
else
|
||||
STRIKES=$(get_strikes "$container" "$WATCHDOG_STATE_FILE")
|
||||
STRIKES=$(( STRIKES + 1 ))
|
||||
set_strikes "$container" "$STRIKES" "$WATCHDOG_STATE_FILE"
|
||||
warn "$container — not running (strike $STRIKES/$SYS_WATCHDOG_STRIKE_LIMIT)"
|
||||
((T1_WARNINGS++))
|
||||
|
||||
if [[ "$STRIKES" -ge "$SYS_WATCHDOG_STRIKE_LIMIT" ]]; then
|
||||
result=0
|
||||
safe_restart "$container" "required container down" || result=$?
|
||||
case $result in
|
||||
0) set_strikes "$container" 0 "$WATCHDOG_STATE_FILE"
|
||||
((T1_RESTARTS++))
|
||||
queue_notify "$container was down and restarted on $(hostname)" "warning" ;;
|
||||
2) : ;;
|
||||
*) queue_notify "$container failed to restart on $(hostname)" "warning" ;;
|
||||
esac
|
||||
fi
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
# Memory and CPU monitoring
|
||||
if [[ ${#WATCHDOG_CONTAINERS[@]} -gt 0 ]]; then
|
||||
STATS=$(docker stats --no-stream \
|
||||
--format "{{.Name}}|{{.MemUsage}}|{{.CPUPerc}}" 2>/dev/null)
|
||||
TOTAL_CORES=$(nproc 2>/dev/null || echo 1)
|
||||
|
||||
for container in "${!WATCHDOG_CONTAINERS[@]}"; do
|
||||
MEM_LIMIT_MB="${WATCHDOG_CONTAINERS[$container]}"
|
||||
CONTAINER_STATS=$(echo "$STATS" | grep "^${container}|" | head -1)
|
||||
[[ -z "$CONTAINER_STATS" ]] && continue
|
||||
|
||||
MEM_USAGE=$(echo "$CONTAINER_STATS" | cut -d'|' -f2 | awk '{print $1}')
|
||||
MEM_UNIT=$(echo "$MEM_USAGE" | grep -oE '[A-Za-z]+')
|
||||
MEM_VALUE=$(echo "$MEM_USAGE" | grep -oE '[0-9.]+')
|
||||
|
||||
case "$MEM_UNIT" in
|
||||
GiB|GB) MEM_MB=$(awk "BEGIN {printf \"%.0f\", $MEM_VALUE * 1024}") ;;
|
||||
MiB|MB) MEM_MB=$(awk "BEGIN {printf \"%.0f\", $MEM_VALUE}") ;;
|
||||
KiB|KB) MEM_MB=$(awk "BEGIN {printf \"%.0f\", $MEM_VALUE / 1024}") ;;
|
||||
*) MEM_MB=0 ;;
|
||||
esac
|
||||
|
||||
CPU_RAW=$(echo "$CONTAINER_STATS" | cut -d'|' -f3 | tr -d '%')
|
||||
CPU_NORM=$(awk "BEGIN {printf \"%.1f\", $CPU_RAW / $TOTAL_CORES}")
|
||||
CPU_INT=$(printf "%.0f" "$CPU_NORM")
|
||||
|
||||
if [[ "$MEM_MB" -ge "$MEM_LIMIT_MB" ]]; then
|
||||
error "$container — memory exceeded hard limit ${MEM_LIMIT_MB}MB"
|
||||
safe_restart "$container" "memory hard limit exceeded"
|
||||
((T1_RESTARTS++))
|
||||
queue_notify "$container exceeded memory limit on $(hostname) — restarted" "warning"
|
||||
fi
|
||||
|
||||
if [[ "$CPU_INT" -ge "$HARD_CPU_THRESHOLD" ]]; then
|
||||
CPU_STRIKES=$(get_strikes "${container}_cpu" "$WATCHDOG_STATE_FILE")
|
||||
CPU_STRIKES=$(( CPU_STRIKES + 1 ))
|
||||
set_strikes "${container}_cpu" "$CPU_STRIKES" "$WATCHDOG_STATE_FILE"
|
||||
if [[ "$CPU_STRIKES" -ge "$CPU_FAIL_LIMIT" ]]; then
|
||||
safe_restart "$container" "CPU threshold exceeded"
|
||||
set_strikes "${container}_cpu" 0 "$WATCHDOG_STATE_FILE"
|
||||
((T1_RESTARTS++))
|
||||
queue_notify "$container CPU ${CPU_NORM}% on $(hostname) — restarted" "warning"
|
||||
fi
|
||||
else
|
||||
set_strikes "${container}_cpu" 0 "$WATCHDOG_STATE_FILE"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
# HTTP responsiveness
|
||||
if [[ ${#WATCHDOG_CONTAINER_URLS[@]} -gt 0 ]]; then
|
||||
for container in "${!WATCHDOG_CONTAINER_URLS[@]}"; do
|
||||
URL="${WATCHDOG_CONTAINER_URLS[$container]}"
|
||||
if curl -sf --max-time "$CURL_TIMEOUT" "$URL" >/dev/null 2>&1; then
|
||||
set_strikes "${container}_http" 0 "$WATCHDOG_STATE_FILE"
|
||||
else
|
||||
HTTP_STRIKES=$(get_strikes "${container}_http" "$WATCHDOG_STATE_FILE")
|
||||
HTTP_STRIKES=$(( HTTP_STRIKES + 1 ))
|
||||
set_strikes "${container}_http" "$HTTP_STRIKES" "$WATCHDOG_STATE_FILE"
|
||||
warn "$container — not responding at $URL (strike $HTTP_STRIKES/$RESP_FAIL_LIMIT)"
|
||||
((T1_WARNINGS++))
|
||||
if [[ "$HTTP_STRIKES" -ge "$RESP_FAIL_LIMIT" ]]; then
|
||||
safe_restart "$container" "HTTP unresponsive"
|
||||
set_strikes "${container}_http" 0 "$WATCHDOG_STATE_FILE"
|
||||
((T1_RESTARTS++))
|
||||
queue_notify "$container unresponsive at $URL on $(hostname) — restarted" "warning"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
# ── TIER 2 — Global Health Scan ─────────────────────────────────────────────────────────
|
||||
if [[ "$WATCHDOG_SCAN_ALL" == "true" ]]; then
|
||||
ALL_CONTAINERS=$(docker ps --format "{{.Names}}" 2>/dev/null)
|
||||
|
||||
# Unhealthy containers
|
||||
if [[ "$WATCHDOG_RESTART_UNHEALTHY" == "true" ]]; then
|
||||
UNHEALTHY=$(docker ps --filter health=unhealthy --format "{{.Names}}" 2>/dev/null)
|
||||
while IFS= read -r container; do
|
||||
[[ -z "$container" ]] && continue
|
||||
[[ -n "${IGNORE_MAP[$container]:-}" ]] && continue
|
||||
is_skipped "$container" && continue
|
||||
error "$container — unhealthy"
|
||||
((T2_WARNINGS++))
|
||||
result=0
|
||||
safe_restart "$container" "unhealthy health status" || result=$?
|
||||
[[ $result -eq 0 ]] && ((T2_RESTARTS++)) && \
|
||||
queue_notify "$container unhealthy on $(hostname) — restarted" "warning"
|
||||
done <<< "$UNHEALTHY"
|
||||
fi
|
||||
|
||||
# OOM killed
|
||||
if [[ "$WATCHDOG_NOTIFY_OOM" == "true" ]]; then
|
||||
while IFS= read -r container; do
|
||||
[[ -z "$container" ]] && continue
|
||||
[[ -n "${IGNORE_MAP[$container]:-}" ]] && continue
|
||||
is_skipped "$container" && continue
|
||||
OOM=$(docker inspect -f '{{.State.OOMKilled}}' "$container" 2>/dev/null)
|
||||
if [[ "$OOM" == "true" ]]; then
|
||||
error "$container — OOM killed"
|
||||
((T2_WARNINGS++))
|
||||
result=0
|
||||
safe_restart "$container" "OOM killed" || result=$?
|
||||
[[ $result -eq 0 ]] && ((T2_RESTARTS++)) && \
|
||||
queue_notify "$container OOM killed on $(hostname) — restarted" "warning"
|
||||
fi
|
||||
done <<< "$ALL_CONTAINERS"
|
||||
fi
|
||||
|
||||
# Crash loop detection
|
||||
if [[ "$WATCHDOG_NOTIFY_CRASHLOOP" == "true" ]]; then
|
||||
while IFS= read -r container; do
|
||||
[[ -z "$container" ]] && continue
|
||||
[[ -n "${IGNORE_MAP[$container]:-}" ]] && continue
|
||||
is_skipped "$container" && continue
|
||||
RESTART_COUNT=$(docker inspect -f '{{.RestartCount}}' "$container" 2>/dev/null || echo 0)
|
||||
PREV_COUNT=$(grep "^${container}_docker:" "$WATCHDOG_STATE_FILE" 2>/dev/null | cut -d: -f2 || echo 0)
|
||||
set_strikes "${container}_docker" "$RESTART_COUNT" "$WATCHDOG_STATE_FILE"
|
||||
if [[ "$RESTART_COUNT" -gt "$PREV_COUNT" && "$RESTART_COUNT" -gt 0 ]]; then
|
||||
((T2_WARNINGS++))
|
||||
if [[ "$RESTART_COUNT" -ge "$WATCHDOG_CRASH_LIMIT" ]]; then
|
||||
error "$container — crash loop CRITICAL: $RESTART_COUNT restarts"
|
||||
queue_notify "$container crash loop CRITICAL on $(hostname) — manual intervention needed" "critical"
|
||||
else
|
||||
warn "$container — restarted since last check (total: $RESTART_COUNT)"
|
||||
queue_notify "$container restarted on $(hostname) — count: $RESTART_COUNT" "warning"
|
||||
fi
|
||||
fi
|
||||
done <<< "$ALL_CONTAINERS"
|
||||
fi
|
||||
|
||||
# Dead containers
|
||||
if [[ "$WATCHDOG_RESTART_DEAD" == "true" ]]; then
|
||||
DEAD=$(docker ps -a --filter status=dead --format "{{.Names}}" 2>/dev/null)
|
||||
while IFS= read -r container; do
|
||||
[[ -z "$container" ]] && continue
|
||||
[[ -n "${IGNORE_MAP[$container]:-}" ]] && continue
|
||||
is_skipped "$container" && continue
|
||||
error "$container — dead"
|
||||
((T2_WARNINGS++))
|
||||
RESTART_COUNT=$(get_restart_count "$container")
|
||||
if [[ "$RESTART_COUNT" -ge "$WATCHDOG_CONTAINER_RESTART_LIMIT" ]]; then
|
||||
add_to_skip_list "$container" "dead — restarted $RESTART_COUNT times"
|
||||
elif [[ "$DRY_RUN" == false ]]; then
|
||||
docker rm "$container" >/dev/null 2>&1
|
||||
if docker start "$container" >/dev/null 2>&1; then
|
||||
success "$container removed from dead state and restarted"
|
||||
log_restart "$container"
|
||||
((T2_RESTARTS++))
|
||||
queue_notify "$container was dead on $(hostname) — restarted" "warning"
|
||||
fi
|
||||
fi
|
||||
done <<< "$DEAD"
|
||||
fi
|
||||
|
||||
# Unexpected exits
|
||||
if [[ "$WATCHDOG_RESTART_CRASHED" == "true" ]]; then
|
||||
CRASHED=$(docker ps -a \
|
||||
--filter status=exited \
|
||||
--format "{{.Names}}|{{.Status}}" 2>/dev/null | \
|
||||
grep -v "Exited (0)")
|
||||
while IFS='|' read -r container status; do
|
||||
[[ -z "$container" ]] && continue
|
||||
[[ -n "${IGNORE_MAP[$container]:-}" ]] && continue
|
||||
is_skipped "$container" && continue
|
||||
SKIP=false
|
||||
for req in "${WATCHDOG_REQUIRED_CONTAINERS[@]}"; do
|
||||
[[ "$container" == "$req" ]] && SKIP=true && break
|
||||
done
|
||||
[[ "$SKIP" == true ]] && continue
|
||||
error "$container — $status (unexpected exit)"
|
||||
((T2_WARNINGS++))
|
||||
result=0
|
||||
safe_restart "$container" "unexpected exit" || result=$?
|
||||
[[ $result -eq 0 ]] && ((T2_RESTARTS++)) && \
|
||||
queue_notify "$container crashed on $(hostname) ($status) — restarted" "warning"
|
||||
done <<< "$CRASHED"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Send notifications if any events this cycle
|
||||
flush_notify
|
||||
|
||||
# Only log summary if something happened — quiet when all healthy
|
||||
TOTAL_RESTARTS=$(( T1_RESTARTS + T2_RESTARTS ))
|
||||
TOTAL_WARNINGS=$(( T1_WARNINGS + T2_WARNINGS ))
|
||||
|
||||
if [[ "$TOTAL_RESTARTS" -gt 0 || "$TOTAL_WARNINGS" -gt 0 ]]; then
|
||||
CYCLE_END=$(date +%s)
|
||||
echo ""
|
||||
echo "━━━ $ICON_WATCHDOG Cycle $CYCLE — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo "$ICON_WATCHDOG T1: $T1_RESTARTS restarts / $T1_WARNINGS warnings"
|
||||
echo "$ICON_WATCHDOG T2: $T2_RESTARTS restarts / $T2_WARNINGS warnings"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( CYCLE_END - CYCLE_START )))"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
else
|
||||
log "Cycle $CYCLE — all healthy ($(date '+%H:%M:%S'))"
|
||||
fi
|
||||
|
||||
# Sleep until next cycle — interruptible by SIGTERM
|
||||
sleep "$DOCKER_WATCHDOG_INTERVAL" &
|
||||
wait $!
|
||||
|
||||
done
|
||||
#
|
||||
# Tier 1 — Strict monitoring (configured containers only)
|
||||
# Memory hard limits — immediate restart if exceeded
|
||||
|
||||
+373
-723
File diff suppressed because it is too large
Load Diff
@@ -13,14 +13,68 @@ Without orchestrators, each script runs independently on its own schedule. This
|
||||
- **Race conditions** — two scripts running simultaneously on the same data
|
||||
- **Order dependency failures** — media cleaner runs before permissions, finds wrong ownership
|
||||
- **No combined summary** — 6 separate notifications instead of one clean report
|
||||
- **Scheduling complexity** — 6+ cron entries instead of one
|
||||
- **Scheduling complexity** — many cron entries instead of a few clean ones
|
||||
|
||||
Orchestrators solve this by making a set of related scripts into a single scheduled unit with a defined execution order and a unified summary.
|
||||
|
||||
---
|
||||
|
||||
## The Orchestrator Model
|
||||
|
||||
The ecosystem is designed so the User Scripts plugin contains only a small number of entries — each one an orchestrator that owns a domain:
|
||||
|
||||
```
|
||||
At Startup of Array:
|
||||
array_start.sh ← single entry, launches everything
|
||||
|
||||
Cron:
|
||||
transcode_management.sh ← */3 * * * *
|
||||
arrs_failed_stalled_recovery.sh ← 0 */6 * * *
|
||||
rsync.sh ... emby-failover ← */30 * * * *
|
||||
daily_sync_maintenance.sh ← 0 1 * * *
|
||||
weekly_sync_maintenance.sh ← 30 2 * * 0
|
||||
weekly_health_digest.sh ← Saturday morning
|
||||
|
||||
Manual only:
|
||||
failover_test.sh, emby_database_repair.sh, repair tools
|
||||
```
|
||||
|
||||
All job lists are configured in the `ORCHESTRATORS` section of `Master.conf`. No changes to orchestrator scripts needed when adding or removing jobs.
|
||||
|
||||
---
|
||||
|
||||
## Scripts
|
||||
|
||||
### `array_start.sh`
|
||||
|
||||
Single entry point for the User Scripts "At Startup of Array" schedule. Launches all array-start scripts in order — each as a background process.
|
||||
|
||||
```bash
|
||||
# Scheduled as: At Startup of Array
|
||||
/mnt/user/appdata/unraid_scripts/Orchestrators/array_start.sh
|
||||
```
|
||||
|
||||
One-shot scripts (ramdisk, syslog filter, php-fpm, network connect) run and exit naturally. Continuous scripts (system watchdog, docker watchdog, failover) run until the array stops.
|
||||
|
||||
**Configuration:**
|
||||
|
||||
```bash
|
||||
# Master.conf — ORCHESTRATORS section
|
||||
ARRAY_START_SCRIPTS=(
|
||||
"unRAID_Essentials/ramdisk_setup.sh" # creates ramdisk before Emby starts
|
||||
"unRAID_Essentials/docker_syslog_filter.sh" # suppress veth log noise
|
||||
"unRAID_Essentials/php_fpm_max_children.sh" # WebGUI tuning
|
||||
"Docker_Essentials/docker_network_connect.sh" # connect containers to extra networks
|
||||
"unRAID_Essentials/system_watchdog.sh" # continuous system health monitor
|
||||
"Docker_Essentials/docker_watchdog.sh" # continuous container health monitor
|
||||
"Failover/failover.sh" # continuous mutual failover
|
||||
)
|
||||
```
|
||||
|
||||
Add or remove scripts from `ARRAY_START_SCRIPTS` — no changes to `array_start.sh` needed. Order matters — ramdisk before Emby, network before watchdogs, watchdogs before failover.
|
||||
|
||||
---
|
||||
|
||||
### `transcode_management.sh`
|
||||
|
||||
Runs `transcode_cleanup.sh` then `transcode_manager.sh` in the correct order every 3 minutes. Replaces two separate cron entries with one.
|
||||
@@ -32,13 +86,13 @@ Runs `transcode_cleanup.sh` then `transcode_manager.sh` in the correct order eve
|
||||
|
||||
**Why cleanup must run before manager:**
|
||||
|
||||
If the manager runs first it may see inflated ramdisk usage from stale segment files left by ended sessions — and trigger an unnecessary flip to SSD. Cleanup runs first to clear those files, then the manager makes its threshold decision based on real active session usage.
|
||||
If the manager runs first it sees inflated ramdisk usage from stale segment files left by ended sessions — and triggers an unnecessary flip to SSD. Cleanup runs first to clear those files, then the manager decides based on real active session usage.
|
||||
|
||||
```
|
||||
Without correct order:
|
||||
Manager checks usage → 6.8GB (includes stale files) → flips to SSD
|
||||
Cleanup runs → removes stale files → actual usage 2.1GB
|
||||
Manager was wrong — unnecessary flip
|
||||
Unnecessary flip — sessions now on SSD
|
||||
|
||||
With correct order:
|
||||
Cleanup runs → removes stale files → actual usage 2.1GB
|
||||
@@ -47,7 +101,7 @@ With correct order:
|
||||
|
||||
**Daily statistics tracking:**
|
||||
|
||||
Every cycle `transcode_management.sh` records stats to `/boot/config/transcode_daily.db`:
|
||||
Every cycle `transcode_management.sh` records stats to `TRANSCODE_DAILY_LOG`:
|
||||
- Peak ramdisk usage for the day
|
||||
- Total flip count for the day
|
||||
- Ramdisk vs SSD session counts
|
||||
@@ -57,193 +111,114 @@ Every cycle `transcode_management.sh` records stats to `/boot/config/transcode_d
|
||||
|
||||
---
|
||||
|
||||
### `media_shares_sync.sh`
|
||||
### `arrs_failed_stalled_recovery.sh`
|
||||
|
||||
Syncs each server's source-of-truth media shares to the remote server sequentially. Each server only pushes the shares it owns — direction and share list are automatic based on which server is running the script.
|
||||
Automatically detects and recovers from failed imports and stalled downloads across Sonarr, Radarr, and Lidarr. Blocklists the bad release and triggers a new search — hands-free recovery while you sleep.
|
||||
|
||||
```bash
|
||||
# Scheduled as: 0 */6 * * * (every 6 hours)
|
||||
/mnt/user/appdata/unraid_scripts/Media/arrs_failed_stalled_recovery.sh
|
||||
```
|
||||
|
||||
Targets four problem types: `importFailed`, `importPending`, `error` status, and `stalled` downloads. Items newer than `ARR_IMPORT_RECOVERY_AGE` (6 hours) are skipped — gives the arr time to retry on its own first.
|
||||
|
||||
**API versions:** Sonarr v4 → `/api/v3/` — Radarr v6 → `/api/v3/` — Lidarr v3 → `/api/v1/`
|
||||
|
||||
Lidarr runs on HOST1 only — exits cleanly on HOST2.
|
||||
|
||||
**Configuration:**
|
||||
|
||||
```bash
|
||||
# Master.conf — MEDIA section
|
||||
ARR_IMPORT_RECOVERY_AGE=6 # hours — skip items newer than this
|
||||
|
||||
HOST1_SONARR_RECOVERY=true
|
||||
HOST1_RADARR_RECOVERY=true
|
||||
HOST1_LIDARR_RECOVERY=true
|
||||
HOST2_SONARR_RECOVERY=true
|
||||
HOST2_RADARR_RECOVERY=true
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### `daily_sync_maintenance.sh`
|
||||
|
||||
Full daily maintenance window orchestrator — git pull, media share sync, media management, and docker daily restarts. All driven by `Master.conf` arrays.
|
||||
|
||||
```bash
|
||||
# Scheduled as: 0 1 * * * (1am daily — on both servers)
|
||||
/mnt/user/appdata/unraid_scripts/Orchestrators/media_shares_sync.sh
|
||||
/mnt/user/appdata/unraid_scripts/Orchestrators/daily_sync_maintenance.sh
|
||||
```
|
||||
|
||||
**Execution order:**
|
||||
|
||||
```
|
||||
1. Pre-sync jobs (DAILY_MAINTENANCE_SCRIPTS — git pull first):
|
||||
git_pull_execute.sh ← always runs first — pulls latest scripts
|
||||
|
||||
2. Media share sync:
|
||||
HOST*_DAILY_SYNC_SHARES ← each server pushes its own truth shares
|
||||
HOST*_PERSONAL_SHARES ← personal encrypted shares appended after
|
||||
|
||||
3. Post-sync jobs (DAILY_MAINTENANCE_SCRIPTS — remaining):
|
||||
media_management.sh ← permissions + cleaners + arr cleanup
|
||||
docker_daily_restart.sh ← daily container restarts
|
||||
```
|
||||
|
||||
**Bidirectional — same script, correct direction automatically:**
|
||||
|
||||
```
|
||||
HOST1 runs media_shares_sync.sh → pushes HOST1_DAILY_SYNC_SHARES → TO HOST2
|
||||
Movies, Tv_Shows, Music, Books etc. — HOST1 is source of truth
|
||||
HOST1 runs daily_sync_maintenance.sh:
|
||||
git pull → sync HOST1_DAILY_SYNC_SHARES → TO HOST2 → media_management → docker restart
|
||||
|
||||
HOST2 runs media_shares_sync.sh → pushes HOST2_DAILY_SYNC_SHARES → TO HOST1
|
||||
Anime_Shows, Anime_Movies — HOST2 is source of truth
|
||||
HOST2 runs daily_sync_maintenance.sh:
|
||||
git pull → sync HOST2_DAILY_SYNC_SHARES → TO HOST1 → media_management → docker restart
|
||||
```
|
||||
|
||||
`detect_hosts()` determines which server is local at runtime and selects the correct share list. No script changes needed to reconfigure who syncs what — only `Master.conf` changes required.
|
||||
|
||||
**What it does:**
|
||||
1. Detects local server via `detect_hosts()` — determines HOST1 or HOST2
|
||||
2. Resolves remote Tailscale IP
|
||||
3. Runs a single pre-flight check — connectivity + remote rootfs
|
||||
4. Builds share list from `HOST1_DAILY_SYNC_SHARES` or `HOST2_DAILY_SYNC_SHARES`
|
||||
5. Appends personal shares (`HOST1_PERSONAL_SHARES` or `HOST2_PERSONAL_SHARES`)
|
||||
6. Calls `Rsync/rsync.sh` for each share
|
||||
7. Tracks pass/fail and duration per share
|
||||
8. Reports a combined summary
|
||||
`detect_hosts()` determines which server is local at runtime and selects the correct share list. No script changes needed — only `Master.conf` changes required.
|
||||
|
||||
**Why one pre-flight check upfront:**
|
||||
Connectivity and rootfs are checked once before the loop starts — not once per share. If the remote is unreachable or the rootfs is nearly full, the whole run fails fast. Individual share existence and disk checks still run per-share inside `rsync.sh`.
|
||||
|
||||
Connectivity and rootfs are checked once before the loop starts — not once per share. If the remote is unreachable or rootfs is nearly full, the whole run fails fast. Individual share checks still run per-share inside `rsync.sh`.
|
||||
|
||||
**Configuration:**
|
||||
|
||||
```bash
|
||||
# Master.conf — per-host share lists
|
||||
# HOST1 truth shares — pushed from HOST1 to HOST2 nightly
|
||||
# Master.conf — ORCHESTRATORS section
|
||||
|
||||
DAILY_MAINTENANCE_SCRIPTS=(
|
||||
"git_pull_execute.sh" # always first
|
||||
"Docker_Essentials/docker_daily_restart.sh" # after sync and media jobs
|
||||
)
|
||||
|
||||
# Media jobs run between sync and docker restart
|
||||
# Permissions first, cleaners second, arr cleanup last
|
||||
MEDIA_MANAGEMENT_JOBS=(
|
||||
"Media/media_shares_permissions.sh" # permissions — everything depends on this
|
||||
"Media/media_cleaner.sh anime" # clean junk before arr scripts scan
|
||||
"Media/media_cleaner.sh media"
|
||||
"Media/lidarr_cleanup.sh" # arr cleanup last — depends on clean folders
|
||||
"Media/sonarr_cleanup.sh"
|
||||
"Media/radarr_cleanup.sh"
|
||||
)
|
||||
|
||||
HOST1_DAILY_SYNC_SHARES=(
|
||||
/mnt/user/Movies
|
||||
/mnt/user/Tv_Shows
|
||||
/mnt/user/Music
|
||||
# ...
|
||||
# all HOST1-owned shares
|
||||
)
|
||||
|
||||
# HOST2 truth shares — pushed from HOST2 to HOST1 nightly
|
||||
HOST2_DAILY_SYNC_SHARES=(
|
||||
/mnt/user/Anime_Shows
|
||||
/mnt/user/Anime_Movies
|
||||
# ...
|
||||
)
|
||||
```
|
||||
|
||||
These shares use global rsync defaults — no profile needed. For shares requiring custom bandwidth limits, container stops, or different rsync options, create a named profile in the Rsync profile system and call `rsync.sh` directly on a separate schedule instead.
|
||||
|
||||
**Relationship to failover writeback:**
|
||||
|
||||
The same share lists are used by `failover.sh` for Tier 4 writeback — but in the opposite direction. If HOST1 was down for 18hr+ and HOST2's arrs downloaded new content, Tier 4 writeback pushes `HOST1_DAILY_SYNC_SHARES` FROM HOST2 back TO HOST1. No duplicate configuration needed.
|
||||
|
||||
**Example output:**
|
||||
```
|
||||
━━━ 🔄 Daily Sync Starting — 2026-04-14 01:00:00 ━━━
|
||||
📋 Shares: 11
|
||||
|
||||
━━━ [1/11] Movies ━━━
|
||||
...rsync output...
|
||||
✅ Movies — 4m32s
|
||||
|
||||
━━━ [2/11] Tv_Shows ━━━
|
||||
...
|
||||
━━━━━ 📋 DAILY SYNC SUMMARY ━━━━━
|
||||
✅ Pass: 10 ❌ Fail: 1
|
||||
⏱️ Duration: 47m12s
|
||||
❌ Failed: Anime_Shows-Old
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### `critical_shares_full_sync.sh`
|
||||
|
||||
Runs a clean nightly sync for Emby and the auth stack (Critical-Data) with containers stopped. This is the companion to the hourly dirty sync — it provides a fully consistent state on HOST2 once per night.
|
||||
**Adding a media job:**
|
||||
|
||||
```bash
|
||||
# Scheduled as: 30 2 * * 0 (2:30am Sunday — weekly clean sync)
|
||||
/mnt/user/appdata/unraid_scripts/Orchestrators/critical_shares_full_sync.sh
|
||||
```
|
||||
|
||||
**Why two Emby syncs:**
|
||||
|
||||
The hourly dirty sync runs with Emby up — WAL files excluded, watch states pushed continuously. This means HOST2 is never more than an hour behind on watch state. But it's not a clean database snapshot.
|
||||
|
||||
The nightly clean sync stops Emby, syncs the full clean database state, then restarts. HOST2 gets a fully consistent Emby state every night. The two syncs work together:
|
||||
|
||||
```
|
||||
Hourly dirty sync (Emby running):
|
||||
users.db, library.db, authentication.db, config/
|
||||
WAL excluded — safe mid-write
|
||||
HOST2 always within 1hr of HOST1 on watch state
|
||||
|
||||
Nightly clean sync (Emby stopped):
|
||||
Full clean snapshot — all databases flushed
|
||||
No WAL files in flight
|
||||
HOST2 gets gold-standard state once per night
|
||||
```
|
||||
|
||||
**Why clean auth sync matters:**
|
||||
|
||||
The auth stack runs warm on both servers continuously. During normal operation HOST2's auth stack serves its own domain — it doesn't receive dirty updates from HOST1. The nightly clean sync is the only time auth state propagates.
|
||||
|
||||
This means:
|
||||
- New user added on HOST1 → propagates to HOST2 overnight automatically
|
||||
- Proxy rule changes → propagated overnight
|
||||
- No manual intervention needed for most auth changes
|
||||
|
||||
For users who just want failover to work — this script handles it. No thinking required about dirty writes, WAL files, or when to sync.
|
||||
|
||||
**What it syncs:**
|
||||
|
||||
```
|
||||
Emby appdata:
|
||||
users.db, library.db, authentication.db, config/
|
||||
Containers stopped → clean flush → safe copy
|
||||
|
||||
Critical-Data (auth stack):
|
||||
NPM proxy rules + SSL certs
|
||||
Authelia config + database
|
||||
Mariadb-Authelia data
|
||||
Redis-Authelia session store
|
||||
LLDAP users and groups database
|
||||
All auth containers stopped → clean databases → safe copy
|
||||
Authelia delayed start on restart — Mariadb + Redis must be ready first
|
||||
```
|
||||
|
||||
**What it excludes (per rsync profile):**
|
||||
|
||||
```
|
||||
Emby: logs, transcodes, cache, metadata, *.db-wal, *.db-shm
|
||||
Auth: logs, *.tmp, nginx/temp, nginx/cache, notification.txt
|
||||
```
|
||||
|
||||
### `media_management.sh`
|
||||
|
||||
Runs all media maintenance scripts sequentially in the order defined in `Master.conf`. Scheduled once daily, typically after the nightly sync.
|
||||
|
||||
```bash
|
||||
# Scheduled as: 0 2 * * * (2am daily — after media_shares_sync.sh)
|
||||
/mnt/user/appdata/unraid_scripts/Orchestrators/media_management.sh
|
||||
```
|
||||
|
||||
**What it does:**
|
||||
1. Reads `MEDIA_MAINTENANCE_JOBS` from `Master.conf`
|
||||
2. Runs each job in order — script path + optional argument
|
||||
3. Tracks pass/fail per job
|
||||
4. Reports a combined summary
|
||||
5. A failure in one job does not stop the others
|
||||
|
||||
**Why order matters:**
|
||||
|
||||
```
|
||||
1. media_shares_permissions.sh ← permissions first — everything else depends on correct ownership
|
||||
2. media_cleaner.sh anime ← clean junk before arr scripts scan
|
||||
3. media_cleaner.sh media ← same
|
||||
4. lidarr_cleanup.sh ← arr cleanup last — depends on clean folders
|
||||
5. sonarr_cleanup.sh
|
||||
6. radarr_cleanup.sh
|
||||
```
|
||||
|
||||
If arr cleanup runs before permissions, it may fail to delete files it doesn't have access to. If it runs before the cleaner, it finds junk files mixed in with real content. The order is intentional.
|
||||
|
||||
**Configuration:**
|
||||
```bash
|
||||
# Master.conf — add, remove, or reorder jobs here
|
||||
# Format: "folder/script.sh optional_argument"
|
||||
MEDIA_MAINTENANCE_JOBS=(
|
||||
"Media/media_shares_permissions.sh"
|
||||
"Media/media_cleaner.sh anime"
|
||||
"Media/media_cleaner.sh media"
|
||||
"Media/lidarr_cleanup.sh"
|
||||
"Media/sonarr_cleanup.sh"
|
||||
"Media/radarr_cleanup.sh"
|
||||
)
|
||||
```
|
||||
|
||||
**Adding a new job:**
|
||||
```bash
|
||||
# Add a line to MEDIA_MAINTENANCE_JOBS — no script changes needed
|
||||
MEDIA_MAINTENANCE_JOBS=(
|
||||
MEDIA_MANAGEMENT_JOBS=(
|
||||
"Media/media_shares_permissions.sh"
|
||||
"Media/media_cleaner.sh anime"
|
||||
"Media/media_cleaner.sh media"
|
||||
@@ -254,10 +229,10 @@ MEDIA_MAINTENANCE_JOBS=(
|
||||
)
|
||||
```
|
||||
|
||||
**Disabling a job temporarily:**
|
||||
**Disabling a media job temporarily:**
|
||||
|
||||
```bash
|
||||
# Comment it out — easy to re-enable
|
||||
MEDIA_MAINTENANCE_JOBS=(
|
||||
MEDIA_MANAGEMENT_JOBS=(
|
||||
"Media/media_shares_permissions.sh"
|
||||
# "Media/media_cleaner.sh anime" # ← disabled, not deleted
|
||||
"Media/media_cleaner.sh media"
|
||||
@@ -267,31 +242,152 @@ MEDIA_MAINTENANCE_JOBS=(
|
||||
)
|
||||
```
|
||||
|
||||
**--dry-run support:**
|
||||
`media_management.sh --dry-run` passes `--dry-run` through to every child script. All scripts report what they would do without making changes. Useful for testing a new job before adding it to the live schedule.
|
||||
**Relationship to failover writeback:**
|
||||
|
||||
The same share lists are used by `failover.sh` for Tier 4 writeback — but in the opposite direction. If HOST1 was down for 24hr+ and HOST2's arrs accumulated content, writeback pushes `HOST1_DAILY_SYNC_SHARES` FROM HOST2 BACK TO HOST1. No duplicate configuration needed.
|
||||
|
||||
---
|
||||
|
||||
### `weekly_sync_maintenance.sh`
|
||||
|
||||
Weekly maintenance window orchestrator — critical appdata clean sync, container updates, and weekly docker restarts. Runs Sunday 2:30am, fits before the 3am network reboot.
|
||||
|
||||
```bash
|
||||
/mnt/user/appdata/unraid_scripts/Orchestrators/media_management.sh --dry-run
|
||||
# Scheduled as: 30 2 * * 0 (Sunday 2:30am)
|
||||
/mnt/user/appdata/unraid_scripts/Orchestrators/weekly_sync_maintenance.sh
|
||||
```
|
||||
|
||||
**Execution order:**
|
||||
|
||||
```
|
||||
1. Stop local containers — auth stack + Emby stopped locally
|
||||
2. Stop remote containers — auth stack + Emby stopped remotely via SSH
|
||||
3. Pull updates locally — if CRITICAL_SYNC_UPDATES=true
|
||||
4. Pull updates remotely — if CRITICAL_SYNC_UPDATES_REMOTE=true
|
||||
5. rsync WEEKLY_SYNC_JOBS — Emby + Critical-Data clean sync
|
||||
6. Start remote containers — starts on new images, correct order
|
||||
7. Start local containers — starts on new images, correct order
|
||||
|
||||
Post-sync jobs (WEEKLY_MAINTENANCE_SCRIPTS):
|
||||
8. docker_weekly_restart.sh
|
||||
```
|
||||
|
||||
**Why two Emby syncs:**
|
||||
|
||||
The emby-failover dirty sync runs every 30-60 minutes with Emby running — WAL files excluded, watch states and library pushed continuously. HOST2 stays current on what users are watching. But it is not a clean database snapshot.
|
||||
|
||||
The weekly clean sync stops Emby on both sides, checkpoints the WAL, and pushes a full consistent mirror. HOST2 gets a gold-standard Emby state once per week.
|
||||
|
||||
```
|
||||
emby-failover every 30-60min (Emby running):
|
||||
users.db, library.db, authentication.db, config/
|
||||
WAL excluded — safe mid-write
|
||||
HOST2 always within 30-60min of HOST1 on watch state
|
||||
|
||||
weekly clean sync Sunday 2:30am (Emby stopped):
|
||||
Full clean mirror — all databases flushed
|
||||
metadata, plugins, config all included
|
||||
~30s downtime — both Emby instances down during sync only
|
||||
Cache stays warm on HOST2 all week — only reset Sunday
|
||||
```
|
||||
|
||||
**Why weekly instead of nightly:**
|
||||
|
||||
Emby builds a warm image cache on HOST2 naturally throughout the week. Syncing nightly resets this cache — users experience slow image loads every morning. Weekly sync lets the cache stay warm for 6 days and only resets on Sunday night when most users are asleep.
|
||||
|
||||
**What it syncs:**
|
||||
|
||||
```
|
||||
WEEKLY_SYNC_JOBS (configurable in Master.conf):
|
||||
/mnt/user/Media_Server/Emby ← emby profile — full clean mirror
|
||||
/mnt/user/appdata-Failover/Critical-Data ← critical-data profile — auth stack
|
||||
|
||||
Emby excludes: logs, transcodes, cache, crash*
|
||||
Auth excludes: logs, *.tmp, nginx/temp, nginx/cache, notification.txt
|
||||
```
|
||||
|
||||
**Container update window:**
|
||||
|
||||
Containers are already stopped for the sync — container image updates pull at zero extra downtime. Both servers start on the same new image version after the sync.
|
||||
|
||||
```bash
|
||||
# Master.conf toggles
|
||||
CRITICAL_SYNC_UPDATES=true # pull updates locally
|
||||
CRITICAL_SYNC_UPDATES_REMOTE=true # pull updates on remote via SSH
|
||||
|
||||
# Toggle false to skip updates without changing the schedule
|
||||
CRITICAL_SYNC_UPDATES=false
|
||||
```
|
||||
|
||||
**Why auth stack matters:**
|
||||
|
||||
The auth stack (Authelia, NPM, Mariadb, Redis, LLDAP) runs warm on both servers. During normal operation HOST2 serves its own domain independently. The weekly clean sync is the only time auth state propagates from HOST1 to HOST2.
|
||||
|
||||
- New user added on HOST1 → propagates to HOST2 on Sunday automatically
|
||||
- Proxy rule changes → propagated Sunday
|
||||
- No manual intervention needed for routine auth changes
|
||||
|
||||
**Sunday maintenance window:**
|
||||
|
||||
```
|
||||
2:30am weekly_sync_maintenance.sh ← clean sync + updates (~3-5min)
|
||||
2:50am CA Auto Update plugin ← plugin updates
|
||||
2:55am CA container updates ← docker container updates
|
||||
3:00am Network reboot ← router/switch restart
|
||||
|
||||
Everything comes back clean:
|
||||
Network fresh, Emby updated, auth stack updated
|
||||
All in one maintenance window while users sleep
|
||||
```
|
||||
|
||||
**Configuration:**
|
||||
|
||||
```bash
|
||||
# Master.conf — ORCHESTRATORS section
|
||||
|
||||
WEEKLY_SYNC_JOBS=(
|
||||
"/mnt/user/Media_Server/Emby"
|
||||
"/mnt/user/appdata-Failover/Critical-Data"
|
||||
)
|
||||
|
||||
WEEKLY_MAINTENANCE_SCRIPTS=(
|
||||
"Docker_Essentials/docker_weekly_restart.sh"
|
||||
)
|
||||
|
||||
CRITICAL_SYNC_UPDATES=true
|
||||
CRITICAL_SYNC_UPDATES_REMOTE=true
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### `media_management.sh`
|
||||
|
||||
Runs all media maintenance scripts sequentially in the order defined in `Master.conf`. Absorbed into `daily_sync_maintenance.sh` via `MEDIA_MANAGEMENT_JOBS` — not scheduled separately. Available for manual runs.
|
||||
|
||||
```bash
|
||||
# Manual use only — called automatically by daily_sync_maintenance.sh
|
||||
bash /mnt/user/appdata/unraid_scripts/Orchestrators/media_management.sh --dry-run
|
||||
bash /mnt/user/appdata/unraid_scripts/Orchestrators/media_management.sh
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## The Orchestrator Pattern
|
||||
|
||||
Both orchestrators follow the same pattern. This is by design — any script that needs to coordinate multiple operations should follow it:
|
||||
All orchestrators follow the same pattern:
|
||||
|
||||
```
|
||||
1. Setup — validate config, detect hosts if needed
|
||||
1. Setup — validate config, detect hosts if needed, acquire lock
|
||||
2. Pre-flight — fail fast checks before doing any work
|
||||
3. Job loop — run each job, track pass/fail, continue on failure
|
||||
4. Summary — one clean report of all results
|
||||
5. Notification — one notification per run, not one per job
|
||||
```
|
||||
|
||||
This pattern means:
|
||||
- **Consistent output** — every orchestrator looks the same in the logs
|
||||
This means:
|
||||
- **Consistent output** — every orchestrator looks the same in logs
|
||||
- **No silent failures** — pass/fail tracked per job, reported in summary
|
||||
- **Single notification** — one bell ring, not six
|
||||
- **Single notification** — one bell ring per run
|
||||
- **Resilient** — one job failing doesn't stop the rest
|
||||
|
||||
---
|
||||
@@ -299,23 +395,39 @@ This pattern means:
|
||||
## Scheduling
|
||||
|
||||
```bash
|
||||
# Recommended schedule
|
||||
*/3 * * * * transcode_management.sh # cleanup then manager — every 3 minutes
|
||||
0 1 * * * media_shares_sync.sh # 1am — media shares to remote
|
||||
0 2 * * * media_management.sh # 2am — permissions, cleaners, arr cleanup
|
||||
30 2 * * 0 critical_shares_full_sync.sh # 2:30am Sunday — clean Emby + auth stack
|
||||
# At Startup of Array
|
||||
array_start.sh # single entry — launches all startup scripts
|
||||
|
||||
# Every 3 minutes
|
||||
*/3 * * * * transcode_management.sh
|
||||
|
||||
# Every 6 hours
|
||||
0 */6 * * * arrs_failed_stalled_recovery.sh
|
||||
|
||||
# Every 30-60 minutes
|
||||
*/30 * * * * rsync.sh /mnt/user/Media_Server/Emby --profile=emby-failover
|
||||
|
||||
# Daily 1am — full daily maintenance window:
|
||||
# git pull → media sync → permissions → cleaners → arr cleanup → docker restart
|
||||
0 1 * * * daily_sync_maintenance.sh
|
||||
|
||||
# Weekly — Sunday morning
|
||||
30 2 * * 0 weekly_sync_maintenance.sh # clean sync + updates + docker weekly restart
|
||||
50 2 * * 0 CA plugin update
|
||||
55 2 * * 0 CA container updates
|
||||
```
|
||||
|
||||
`media_shares_sync.sh` and `media_management.sh` run nightly — media shares and maintenance. `critical_shares_full_sync.sh` runs weekly on Sunday — it stops Emby and the auth stack for a clean consistent sync. Running it weekly instead of nightly lets Emby's image cache stay warm on HOST2 throughout the week. The emby-failover dirty sync handles watch states, library structure, and auth every 30-60 minutes — the weekly clean sync covers metadata, plugins, and a full database flush.
|
||||
`daily_sync_maintenance.sh` owns the entire daily window — git pull, media sync, permissions, cleaners, arr cleanup, and docker restarts in one scheduled run. Everything configured in `Master.conf` via `DAILY_MAINTENANCE_SCRIPTS` and `MEDIA_MANAGEMENT_JOBS`.
|
||||
|
||||
---
|
||||
|
||||
## Adding a New Orchestrator
|
||||
|
||||
If you find yourself running 3 or more related scripts on the same schedule, consider wrapping them in a new orchestrator. The pattern is simple:
|
||||
If you find yourself running 3 or more related scripts on the same schedule, consider wrapping them in a new orchestrator. Model it directly on `media_management.sh` which handles dry-run passthrough, status display, pass/fail tracking and summary reporting.
|
||||
|
||||
Minimal skeleton:
|
||||
|
||||
```bash
|
||||
# Minimal orchestrator skeleton
|
||||
JOBS=(
|
||||
"Folder/script1.sh"
|
||||
"Folder/script2.sh arg"
|
||||
@@ -335,5 +447,3 @@ for JOB in "${JOBS[@]}"; do
|
||||
fi
|
||||
done
|
||||
```
|
||||
|
||||
Better yet — model it directly on `media_management.sh` which already handles dry-run passthrough, status display, pass/fail tracking and summary reporting.
|
||||
@@ -0,0 +1,106 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- Array Start Orchestrator -----------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Launches all scripts configured in ARRAY_START_SCRIPTS when the unRAID array comes online.
|
||||
# Set this script to run at "Startup of Array" in the User Scripts plugin.
|
||||
#
|
||||
# Each script is launched as a background process:
|
||||
# One-shot scripts (ramdisk_setup, syslog_filter etc.) run and exit naturally
|
||||
# Continuous scripts (system_watchdog, docker_watchdog, failover) run until array stops
|
||||
#
|
||||
# Scripts are launched in the order defined in ARRAY_START_SCRIPTS in Master.conf.
|
||||
# Order matters — ramdisk before Emby, network before watchdogs, watchdogs before failover.
|
||||
#
|
||||
# To add or remove a script: edit ARRAY_START_SCRIPTS in Master.conf.
|
||||
# No changes to this script needed.
|
||||
#
|
||||
# Logs: each script logs its own output independently.
|
||||
# This orchestrator exits after launching all scripts — unRAID sees it complete normally.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/Master.conf"
|
||||
source "$SCRIPT_DIR/common.sh"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Array Start — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Running as root"
|
||||
|
||||
SCRIPT_COUNT=${#ARRAY_START_SCRIPTS[@]}
|
||||
info "Launching $SCRIPT_COUNT script(s)..."
|
||||
echo ""
|
||||
|
||||
LAUNCHED=0
|
||||
FAILED=0
|
||||
|
||||
for relative_path in "${ARRAY_START_SCRIPTS[@]}"; do
|
||||
[[ -z "$relative_path" ]] && continue
|
||||
|
||||
SCRIPT_PATH="$SCRIPT_DIR/$relative_path"
|
||||
SCRIPT_NAME=$(basename "$SCRIPT_PATH")
|
||||
|
||||
if [[ ! -f "$SCRIPT_PATH" ]]; then
|
||||
error "$SCRIPT_NAME — not found at $SCRIPT_PATH"
|
||||
((FAILED++))
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ ! -x "$SCRIPT_PATH" ]]; then
|
||||
error "$SCRIPT_NAME — not executable"
|
||||
((FAILED++))
|
||||
continue
|
||||
fi
|
||||
|
||||
info "$ICON_START Launching $SCRIPT_NAME..."
|
||||
bash "$SCRIPT_PATH" &
|
||||
PID=$!
|
||||
|
||||
# Brief pause to let script initialize and catch immediate failures
|
||||
sleep 1
|
||||
|
||||
if kill -0 "$PID" 2>/dev/null; then
|
||||
success "$SCRIPT_NAME — running (PID $PID)"
|
||||
((LAUNCHED++))
|
||||
else
|
||||
# Script exited — check if it was a one-shot (exit 0) or a failure
|
||||
wait "$PID"
|
||||
EXIT_CODE=$?
|
||||
if [[ "$EXIT_CODE" -eq 0 ]]; then
|
||||
success "$SCRIPT_NAME — completed (one-shot)"
|
||||
((LAUNCHED++))
|
||||
else
|
||||
error "$SCRIPT_NAME — exited with code $EXIT_CODE"
|
||||
((FAILED++))
|
||||
fi
|
||||
fi
|
||||
|
||||
done
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Summary ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY ARRAY START SUMMARY ━━━━━"
|
||||
echo "$ICON_SUCCESS Launched: $LAUNCHED"
|
||||
echo "$ICON_ERROR Failed: $FAILED"
|
||||
echo "$ICON_TIME Time: $(date '+%H:%M:%S')"
|
||||
echo ""
|
||||
|
||||
if [[ "$FAILED" -gt 0 ]]; then
|
||||
echo "$ICON_WARN Status: $FAILED script(s) failed to launch — check logs"
|
||||
notify "Array start on $(hostname) — $FAILED script(s) failed to launch" "Array Start" "warning"
|
||||
else
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS All scripts launched"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
@@ -0,0 +1,284 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- Daily Sync Maintenance ------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Daily orchestrator — runs the full daily maintenance window in the correct order.
|
||||
#
|
||||
# What it does:
|
||||
# 1. Iterates DAILY_MAINTENANCE_SCRIPTS — runs git pull first, then additional jobs
|
||||
# 2. Syncs all media shares in the correct direction for the local server
|
||||
# 3. Additional scripts in DAILY_MAINTENANCE_SCRIPTS run after the sync completes
|
||||
#
|
||||
# Media share sync:
|
||||
# Each server pushes only the shares it owns (source of truth) — direction is automatic.
|
||||
# HOST1 pushes: Movies, Tv_Shows, Music, Books etc. → HOST2
|
||||
# HOST2 pushes: Anime_Shows, Anime_Movies → HOST1
|
||||
# Personal encrypted shares synced after media shares.
|
||||
# detect_hosts() determines which server is running — no script changes needed.
|
||||
# Share lists configured in Master.conf ORCHESTRATORS section.
|
||||
#
|
||||
# Per-share rsync handled by rsync.sh — this script tracks pass/fail and total time.
|
||||
# All job lists configured in Master.conf — no script changes needed to add or remove jobs.
|
||||
# Schedule: 0 1 * * * (1am daily — configured in User Scripts plugin)
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
|
||||
RSYNC_SCRIPT="$SCRIPT_DIR/../Rsync/rsync.sh"
|
||||
SCRIPTS_ROOT="$SCRIPT_DIR/.."
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
|
||||
detect_hosts
|
||||
resolve_remote_ip
|
||||
|
||||
acquire_lock
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SHIELD Pre-flight Checks ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_SHIELD Pre-flight Checks ━━━"
|
||||
|
||||
check_connectivity
|
||||
check_remote_rootfs
|
||||
|
||||
WINDOW_START=$(date +%s)
|
||||
JOB_PASS=()
|
||||
JOB_FAIL=()
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GIT Pre-sync Jobs ━━━
|
||||
# git_pull_execute.sh runs first — pulls latest scripts before anything else runs
|
||||
# Identified by script name — all other DAILY_MAINTENANCE_SCRIPTS run after sync
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Pre-sync Jobs ━━━"
|
||||
|
||||
PRE_SYNC_SCRIPTS=()
|
||||
POST_SYNC_SCRIPTS=()
|
||||
|
||||
for script_entry in "${DAILY_MAINTENANCE_SCRIPTS[@]}"; do
|
||||
[[ -z "$script_entry" ]] && continue
|
||||
script_name=$(basename "${script_entry%% *}")
|
||||
if [[ "$script_name" == "git_pull_execute.sh" ]]; then
|
||||
PRE_SYNC_SCRIPTS+=("$script_entry")
|
||||
else
|
||||
POST_SYNC_SCRIPTS+=("$script_entry")
|
||||
fi
|
||||
done
|
||||
|
||||
for script_entry in "${PRE_SYNC_SCRIPTS[@]}"; do
|
||||
[[ -z "$script_entry" ]] && continue
|
||||
script_args=($script_entry)
|
||||
script_path="$SCRIPTS_ROOT/${script_args[0]}"
|
||||
script_name=$(basename "${script_args[0]}")
|
||||
extra_args=("${script_args[@]:1}")
|
||||
|
||||
echo ""
|
||||
info "$ICON_START Running: $script_name"
|
||||
|
||||
if [[ ! -f "$script_path" ]]; then
|
||||
error "$script_name — not found at $script_path"
|
||||
JOB_FAIL+=("$script_name")
|
||||
continue
|
||||
fi
|
||||
|
||||
if bash "$script_path" "${extra_args[@]}"; then
|
||||
success "$script_name — done"
|
||||
JOB_PASS+=("$script_name")
|
||||
else
|
||||
error "$script_name — failed"
|
||||
JOB_FAIL+=("$script_name")
|
||||
fi
|
||||
done
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Build share list — host-specific truth shares + personal shares
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
PASS=()
|
||||
FAIL=()
|
||||
SHARE_TIMES=()
|
||||
TOTAL_START=$(date +%s)
|
||||
|
||||
ALL_SHARES=()
|
||||
|
||||
if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then
|
||||
for share in "${HOST1_DAILY_SYNC_SHARES[@]}"; do
|
||||
[[ -n "$share" ]] && ALL_SHARES+=("$share")
|
||||
done
|
||||
for share in "${HOST1_PERSONAL_SHARES[@]}"; do
|
||||
[[ -n "$share" ]] && ALL_SHARES+=("$share")
|
||||
done
|
||||
elif [[ "$LOCAL_SERVER_NAME" == "$HOST2" ]]; then
|
||||
for share in "${HOST2_DAILY_SYNC_SHARES[@]}"; do
|
||||
[[ -n "$share" ]] && ALL_SHARES+=("$share")
|
||||
done
|
||||
for share in "${HOST2_PERSONAL_SHARES[@]}"; do
|
||||
[[ -n "$share" ]] && ALL_SHARES+=("$share")
|
||||
done
|
||||
fi
|
||||
|
||||
SHARE_COUNT=${#ALL_SHARES[@]}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SYNC Media Share Sync ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Media Share Sync — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo "$ICON_SUMMARY Shares: $SHARE_COUNT"
|
||||
echo ""
|
||||
|
||||
SHARE_INDEX=0
|
||||
|
||||
for SHARE in "${ALL_SHARES[@]}"; do
|
||||
SHARE_INDEX=$((SHARE_INDEX + 1))
|
||||
SHARE_NAME=$(basename "$SHARE")
|
||||
SHARE_START=$(date +%s)
|
||||
|
||||
echo "━━━ $ICON_SYNC Share $SHARE_INDEX of $SHARE_COUNT: $SHARE_NAME ━━━"
|
||||
|
||||
if bash "$RSYNC_SCRIPT" "$SHARE"; then
|
||||
SHARE_END=$(date +%s)
|
||||
SHARE_TIMES+=("$SHARE_NAME:$((SHARE_END - SHARE_START))")
|
||||
PASS+=("$SHARE_NAME")
|
||||
echo "$ICON_DONE $SHARE_NAME complete"
|
||||
else
|
||||
SHARE_END=$(date +%s)
|
||||
SHARE_TIMES+=("$SHARE_NAME:$((SHARE_END - SHARE_START))")
|
||||
FAIL+=("$SHARE_NAME")
|
||||
error "$SHARE_NAME failed — continuing to next share"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
done
|
||||
|
||||
TOTAL_END=$(date +%s)
|
||||
TOTAL_DURATION=$((TOTAL_END - TOTAL_START))
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_CLEAN Post-sync Media Jobs ━━━
|
||||
# Reads MEDIA_MANAGEMENT_JOBS from Master.conf — permissions, cleaners, arr cleanup
|
||||
# Runs after sync completes — correct ownership available, clean folders guaranteed
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_CLEAN Post-sync Media Jobs ━━━"
|
||||
|
||||
if [[ ${#MEDIA_MANAGEMENT_JOBS[@]} -gt 0 ]]; then
|
||||
for script_entry in "${MEDIA_MANAGEMENT_JOBS[@]}"; do
|
||||
[[ -z "$script_entry" ]] && continue
|
||||
script_args=($script_entry)
|
||||
script_path="$SCRIPTS_ROOT/${script_args[0]}"
|
||||
script_name=$(basename "${script_args[0]}")
|
||||
extra_args=("${script_args[@]:1}")
|
||||
|
||||
echo ""
|
||||
info "$ICON_START Running: $script_name ${extra_args[*]}"
|
||||
|
||||
if [[ ! -f "$script_path" ]]; then
|
||||
error "$script_name — not found at $script_path"
|
||||
JOB_FAIL+=("$script_name ${extra_args[*]}")
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
if bash "$script_path" "${extra_args[@]}" --dry-run; then
|
||||
success "$script_name — done (dry run)"
|
||||
JOB_PASS+=("$script_name ${extra_args[*]}")
|
||||
else
|
||||
error "$script_name — failed"
|
||||
JOB_FAIL+=("$script_name ${extra_args[*]}")
|
||||
fi
|
||||
else
|
||||
if bash "$script_path" "${extra_args[@]}"; then
|
||||
success "$script_name — done"
|
||||
JOB_PASS+=("$script_name ${extra_args[*]}")
|
||||
else
|
||||
error "$script_name — failed"
|
||||
JOB_FAIL+=("$script_name ${extra_args[*]}")
|
||||
fi
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Post-sync System Jobs ━━━
|
||||
# Reads remaining DAILY_MAINTENANCE_SCRIPTS — docker restart etc.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Post-sync Jobs ━━━"
|
||||
|
||||
for script_entry in "${POST_SYNC_SCRIPTS[@]}"; do
|
||||
[[ -z "$script_entry" ]] && continue
|
||||
script_args=($script_entry)
|
||||
script_path="$SCRIPTS_ROOT/${script_args[0]}"
|
||||
script_name=$(basename "${script_args[0]}")
|
||||
extra_args=("${script_args[@]:1}")
|
||||
|
||||
echo ""
|
||||
info "$ICON_START Running: $script_name"
|
||||
|
||||
if [[ ! -f "$script_path" ]]; then
|
||||
error "$script_name — not found at $script_path"
|
||||
JOB_FAIL+=("$script_name")
|
||||
continue
|
||||
fi
|
||||
|
||||
if bash "$script_path" "${extra_args[@]}"; then
|
||||
success "$script_name — done"
|
||||
JOB_PASS+=("$script_name")
|
||||
else
|
||||
error "$script_name — failed"
|
||||
JOB_FAIL+=("$script_name")
|
||||
fi
|
||||
done
|
||||
|
||||
WINDOW_END=$(date +%s)
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Summary ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY DAILY SYNC MAINTENANCE SUMMARY ━━━━━"
|
||||
echo "$ICON_TIME Window: $(date -d @$WINDOW_START '+%Y-%m-%d %H:%M:%S') → $(date -d @$WINDOW_END '+%H:%M:%S')"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( WINDOW_END - WINDOW_START )))"
|
||||
echo ""
|
||||
|
||||
echo "$ICON_SYNC Media shares:"
|
||||
for entry in "${SHARE_TIMES[@]}"; do
|
||||
SHARE_NAME="${entry%%:*}"
|
||||
DURATION="${entry##*:}"
|
||||
if printf '%s\n' "${FAIL[@]}" | grep -qx "$SHARE_NAME"; then
|
||||
echo " $ICON_ERROR $SHARE_NAME — $(format_duration $DURATION)"
|
||||
else
|
||||
echo " $ICON_DONE $SHARE_NAME — $(format_duration $DURATION)"
|
||||
fi
|
||||
done
|
||||
echo " Passed: ${#PASS[@]}/$SHARE_COUNT Failed: ${#FAIL[@]}/$SHARE_COUNT"
|
||||
echo ""
|
||||
|
||||
if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then
|
||||
echo "$ICON_GEAR Jobs (media + system):"
|
||||
for job in "${JOB_PASS[@]}"; do echo " $ICON_DONE $job"; done
|
||||
for job in "${JOB_FAIL[@]}"; do echo " $ICON_ERROR $job"; done
|
||||
echo ""
|
||||
fi
|
||||
|
||||
TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} ))
|
||||
|
||||
if [[ "$TOTAL_FAIL" -gt 0 ]]; then
|
||||
echo "$ICON_WARN Status: $TOTAL_FAIL failure(s) — check logs"
|
||||
notify "Daily sync maintenance completed with failures on $(hostname) — shares: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" "Daily Maintenance" "warning"
|
||||
exit 1
|
||||
else
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS ALL COMPLETE"
|
||||
notify "Daily sync maintenance complete on $(hostname) — ${#PASS[@]} shares synced, ${#JOB_PASS[@]} jobs run in $(format_duration $(( WINDOW_END - WINDOW_START )))" "Daily Maintenance" "normal"
|
||||
exit 0
|
||||
fi
|
||||
@@ -1,177 +0,0 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- Media Management Orchestrator ------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Runs all media maintenance scripts sequentially in the order defined in Master.conf.
|
||||
# Each job in MEDIA_MAINTENANCE_JOBS is a script path with an optional argument.
|
||||
# Scripts are resolved relative to the ecosystem root directory.
|
||||
#
|
||||
# To add a new job — edit MEDIA_MAINTENANCE_JOBS in Master.conf:
|
||||
# "Media/my_new_script.sh" — script with no argument
|
||||
# "Media/media_cleaner.sh profile" — script with argument
|
||||
#
|
||||
# Order matters — permissions runs before cleaners so files are correctly owned first.
|
||||
# Each job runs independently — a failure in one does not stop the others.
|
||||
# Supports --dry-run — passes through to all child scripts.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
|
||||
source "$ECOSYSTEM_ROOT/Master.conf"
|
||||
source "$ECOSYSTEM_ROOT/common.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Running as root"
|
||||
|
||||
acquire_lock
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — passing --dry-run to all child scripts"
|
||||
|
||||
# Validate job list
|
||||
if [[ ${#MEDIA_MAINTENANCE_JOBS[@]} -eq 0 ]]; then
|
||||
warn "MEDIA_MAINTENANCE_JOBS is empty in Master.conf — nothing to run"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Status ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_CLEAN Jobs to run: ${#MEDIA_MAINTENANCE_JOBS[@]}"
|
||||
local_idx=1
|
||||
for job in "${MEDIA_MAINTENANCE_JOBS[@]}"; do
|
||||
echo " $local_idx. $job"
|
||||
((local_idx++))
|
||||
done
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Tracking
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
PASS=()
|
||||
FAIL=()
|
||||
JOB_TIMES=()
|
||||
TOTAL_START=$(date +%s)
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# JOB RUNNER
|
||||
# Splits each MEDIA_MAINTENANCE_JOBS entry into script path + optional argument.
|
||||
# Resolves script relative to ecosystem root. Passes --dry-run if active.
|
||||
# Records pass/fail and duration for summary.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
run_job() {
|
||||
local entry="$1"
|
||||
|
||||
# Split entry into script path and optional argument
|
||||
local script_rel arg=""
|
||||
read -r script_rel arg <<< "$entry"
|
||||
|
||||
local script="$ECOSYSTEM_ROOT/$script_rel"
|
||||
local label
|
||||
label="$(basename "$script_rel" .sh)${arg:+ $arg}"
|
||||
|
||||
local job_start
|
||||
job_start=$(date +%s)
|
||||
|
||||
echo ""
|
||||
echo "━━━ $ICON_CLEAN $label ━━━"
|
||||
|
||||
if [[ ! -f "$script" ]]; then
|
||||
error "$script not found — skipping"
|
||||
FAIL+=("$label")
|
||||
JOB_TIMES+=("$label:0")
|
||||
return
|
||||
fi
|
||||
|
||||
if [[ ! -x "$script" ]]; then
|
||||
warn "$script is not executable — attempting to fix"
|
||||
chmod +x "$script"
|
||||
fi
|
||||
|
||||
local dry_flag=""
|
||||
[[ "$DRY_RUN" == true ]] && dry_flag="--dry-run"
|
||||
|
||||
# Run script with optional argument and optional dry-run flag
|
||||
if bash "$script" $arg $dry_flag; then
|
||||
local job_end
|
||||
job_end=$(date +%s)
|
||||
PASS+=("$label")
|
||||
JOB_TIMES+=("$label:$((job_end - job_start))")
|
||||
success "$label complete"
|
||||
else
|
||||
local job_end
|
||||
job_end=$(date +%s)
|
||||
FAIL+=("$label")
|
||||
JOB_TIMES+=("$label:$((job_end - job_start))")
|
||||
error "$label failed — continuing to next job"
|
||||
fi
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_CLEAN Media Management ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_CLEAN Media Management — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo "$ICON_SUMMARY Jobs: ${#MEDIA_MAINTENANCE_JOBS[@]}"
|
||||
echo ""
|
||||
|
||||
JOB_INDEX=1
|
||||
for job in "${MEDIA_MAINTENANCE_JOBS[@]}"; do
|
||||
info "Job $JOB_INDEX of ${#MEDIA_MAINTENANCE_JOBS[@]}: $job"
|
||||
run_job "$job"
|
||||
((JOB_INDEX++))
|
||||
done
|
||||
|
||||
TOTAL_END=$(date +%s)
|
||||
TOTAL_DURATION=$((TOTAL_END - TOTAL_START))
|
||||
JOB_COUNT=$(( ${#PASS[@]} + ${#FAIL[@]} ))
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Summary ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY MEDIA MANAGEMENT SUMMARY ━━━━━"
|
||||
echo "$ICON_TIME Started: $(date -d @$TOTAL_START '+%Y-%m-%d %H:%M:%S')"
|
||||
echo "$ICON_TIME Finished: $(date -d @$TOTAL_END '+%Y-%m-%d %H:%M:%S')"
|
||||
echo ""
|
||||
|
||||
for entry in "${JOB_TIMES[@]}"; do
|
||||
label="${entry%%:*}"
|
||||
duration="${entry##*:}"
|
||||
if printf '%s\n' "${FAIL[@]}" | grep -qx "$label"; then
|
||||
echo " $ICON_ERROR $label — $(format_duration $duration)"
|
||||
else
|
||||
echo " $ICON_DONE $label — $(format_duration $duration)"
|
||||
fi
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo " $ICON_SUCCESS Passed: ${#PASS[@]}/$JOB_COUNT"
|
||||
echo " $ICON_ERROR Failed: ${#FAIL[@]}/$JOB_COUNT"
|
||||
echo " $ICON_TIME Duration: $(format_duration $TOTAL_DURATION)"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
if [[ ${#FAIL[@]} -gt 0 ]]; then
|
||||
notify "Media management completed with failures on $(hostname) — failed: ${FAIL[*]}" "Media Management" "warning"
|
||||
exit 1
|
||||
else
|
||||
notify "Media management complete on $(hostname) — ${#PASS[@]}/$JOB_COUNT jobs in $(format_duration $TOTAL_DURATION)" "Media Management" "normal"
|
||||
exit 0
|
||||
fi
|
||||
@@ -1,141 +0,0 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- Media Shares Sync ------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Runs all media shares sequentially in the correct direction for the local server.
|
||||
# Each server pushes its own source-of-truth shares to the remote — direction is automatic.
|
||||
#
|
||||
# HOST1 pushes: its truth shares (Movies, Tv_Shows, Music etc.) + personal → HOST2
|
||||
# HOST2 pushes: its truth shares (Anime_Shows, Anime_Movies etc.) + personal → HOST1
|
||||
#
|
||||
# detect_hosts() determines which server is running the script at runtime.
|
||||
# Share lists are configured per host in Master.conf — no script changes needed
|
||||
# to add, remove, or reconfigure shares.
|
||||
#
|
||||
# Per-share rsync handled by rsync.sh — this script tracks pass/fail and total time only.
|
||||
# Scheduled via unRAID User Scripts plugin at 1am on both servers.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
|
||||
RSYNC_SCRIPT="$SCRIPT_DIR/../Rsync/rsync.sh"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
|
||||
detect_hosts
|
||||
resolve_remote_ip
|
||||
|
||||
acquire_lock
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SHIELD Pre-flight Checks ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_SHIELD Pre-flight Checks ━━━"
|
||||
|
||||
# Single connectivity and rootfs check upfront — fail fast before attempting all shares
|
||||
# Individual share and disk checks run per-share inside rsync.sh
|
||||
check_connectivity
|
||||
check_remote_rootfs
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Build share list — host-specific truth shares + personal shares
|
||||
# Each server only syncs the shares it is source of truth for
|
||||
# Personal shares appended after media shares
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
PASS=()
|
||||
FAIL=()
|
||||
SHARE_TIMES=()
|
||||
TOTAL_START=$(date +%s)
|
||||
|
||||
ALL_SHARES=()
|
||||
|
||||
if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then
|
||||
for share in "${HOST1_DAILY_SYNC_SHARES[@]}"; do
|
||||
[[ -n "$share" ]] && ALL_SHARES+=("$share")
|
||||
done
|
||||
for share in "${HOST1_PERSONAL_SHARES[@]}"; do
|
||||
[[ -n "$share" ]] && ALL_SHARES+=("$share")
|
||||
done
|
||||
elif [[ "$LOCAL_SERVER_NAME" == "$HOST2" ]]; then
|
||||
for share in "${HOST2_DAILY_SYNC_SHARES[@]}"; do
|
||||
[[ -n "$share" ]] && ALL_SHARES+=("$share")
|
||||
done
|
||||
for share in "${HOST2_PERSONAL_SHARES[@]}"; do
|
||||
[[ -n "$share" ]] && ALL_SHARES+=("$share")
|
||||
done
|
||||
fi
|
||||
|
||||
SHARE_COUNT=${#ALL_SHARES[@]}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SYNC Transfer ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Daily Sync Starting — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo "$ICON_SUMMARY Shares: $SHARE_COUNT"
|
||||
echo ""
|
||||
|
||||
SHARE_INDEX=0
|
||||
|
||||
for SHARE in "${ALL_SHARES[@]}"; do
|
||||
SHARE_INDEX=$((SHARE_INDEX + 1))
|
||||
SHARE_NAME=$(basename "$SHARE")
|
||||
SHARE_START=$(date +%s)
|
||||
|
||||
echo "━━━ $ICON_SYNC Share $SHARE_INDEX of $SHARE_COUNT: $SHARE_NAME ━━━"
|
||||
|
||||
if bash "$RSYNC_SCRIPT" "$SHARE"; then
|
||||
SHARE_END=$(date +%s)
|
||||
SHARE_TIMES+=("$SHARE_NAME:$((SHARE_END - SHARE_START))")
|
||||
PASS+=("$SHARE_NAME")
|
||||
echo "$ICON_DONE $SHARE_NAME complete"
|
||||
else
|
||||
SHARE_END=$(date +%s)
|
||||
SHARE_TIMES+=("$SHARE_NAME:$((SHARE_END - SHARE_START))")
|
||||
FAIL+=("$SHARE_NAME")
|
||||
error "$SHARE_NAME failed — continuing to next share"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
done
|
||||
|
||||
TOTAL_END=$(date +%s)
|
||||
TOTAL_DURATION=$((TOTAL_END - TOTAL_START))
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Summary ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo "━━━━━ $ICON_SUMMARY DAILY SYNC SUMMARY ━━━━━"
|
||||
echo "$ICON_TIME Started: $(date -d @$TOTAL_START '+%Y-%m-%d %H:%M:%S')"
|
||||
echo "$ICON_TIME Finished: $(date -d @$TOTAL_END '+%Y-%m-%d %H:%M:%S')"
|
||||
echo ""
|
||||
for entry in "${SHARE_TIMES[@]}"; do
|
||||
SHARE_NAME="${entry%%:*}"
|
||||
DURATION="${entry##*:}"
|
||||
if printf '%s\n' "${FAIL[@]}" | grep -qx "$SHARE_NAME"; then
|
||||
echo " $ICON_ERROR $SHARE_NAME — $(format_duration $DURATION)"
|
||||
else
|
||||
echo " $ICON_DONE $SHARE_NAME — $(format_duration $DURATION)"
|
||||
fi
|
||||
done
|
||||
echo ""
|
||||
echo " $ICON_SUCCESS Passed: ${#PASS[@]}/$SHARE_COUNT"
|
||||
echo " $ICON_ERROR Failed: ${#FAIL[@]}/$SHARE_COUNT"
|
||||
echo " $ICON_TIME Duration: $(format_duration $TOTAL_DURATION)"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
if [[ ${#FAIL[@]} -gt 0 ]]; then
|
||||
notify "Daily sync completed with failures — ${#FAIL[@]}/$SHARE_COUNT failed: ${FAIL[*]}" "Daily Sync" "warning"
|
||||
exit 1
|
||||
else
|
||||
notify "Daily sync complete — ${#PASS[@]}/$SHARE_COUNT shares in $(format_duration $TOTAL_DURATION)" "Daily Sync" "normal"
|
||||
exit 0
|
||||
fi
|
||||
+62
-13
@@ -1,6 +1,6 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ----------------------------- Critical Shares Maintenance ------------------------------------
|
||||
# ----------------------------- Weekly Sync Maintenance ------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Maintenance window orchestrator for Emby and the auth stack (Critical-Data).
|
||||
# Both local and remote containers are stopped for the entire window — clean state
|
||||
@@ -161,11 +161,7 @@ PASS=()
|
||||
FAIL=()
|
||||
TOTAL_START=$(date +%s)
|
||||
|
||||
SYNC_JOBS=(
|
||||
"/mnt/user/Media_Server/Emby"
|
||||
"/mnt/user/appdata-Failover/Critical-Data"
|
||||
)
|
||||
|
||||
SYNC_JOBS=("${WEEKLY_SYNC_JOBS[@]}")
|
||||
SHARE_COUNT=${#SYNC_JOBS[@]}
|
||||
|
||||
echo ""
|
||||
@@ -221,32 +217,85 @@ fi
|
||||
TOTAL_END=$(date +%s)
|
||||
TOTAL_DURATION=$(format_duration $(( TOTAL_END - TOTAL_START )))
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Post-sync Jobs ━━━
|
||||
# docker_weekly_restart.sh and any other WEEKLY_MAINTENANCE_SCRIPTS run after sync
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
JOB_PASS=()
|
||||
JOB_FAIL=()
|
||||
SCRIPTS_ROOT="$SCRIPT_DIR/.."
|
||||
|
||||
if [[ ${#WEEKLY_MAINTENANCE_SCRIPTS[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Post-sync Jobs ━━━"
|
||||
|
||||
for script_entry in "${WEEKLY_MAINTENANCE_SCRIPTS[@]}"; do
|
||||
[[ -z "$script_entry" ]] && continue
|
||||
script_args=($script_entry)
|
||||
script_path="$SCRIPTS_ROOT/${script_args[0]}"
|
||||
script_name=$(basename "${script_args[0]}")
|
||||
extra_args=("${script_args[@]:1}")
|
||||
|
||||
echo ""
|
||||
info "$ICON_START Running: $script_name"
|
||||
|
||||
if [[ ! -f "$script_path" ]]; then
|
||||
error "$script_name — not found at $script_path"
|
||||
JOB_FAIL+=("$script_name")
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would run: $script_name"
|
||||
JOB_PASS+=("$script_name (dry run)")
|
||||
elif bash "$script_path" "${extra_args[@]}"; then
|
||||
success "$script_name — done"
|
||||
JOB_PASS+=("$script_name")
|
||||
else
|
||||
error "$script_name — failed"
|
||||
JOB_FAIL+=("$script_name")
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
WINDOW_END=$(date +%s)
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Summary ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY CRITICAL SHARES MAINTENANCE SUMMARY ━━━━━"
|
||||
echo "━━━━━ $ICON_SUMMARY WEEKLY SYNC MAINTENANCE SUMMARY ━━━━━"
|
||||
echo "$ICON_TIME Duration: $TOTAL_DURATION"
|
||||
echo "$ICON_GEAR Updates: local=$CRITICAL_SYNC_UPDATES remote=$CRITICAL_SYNC_UPDATES_REMOTE"
|
||||
echo "$ICON_SUCCESS Passed: ${#PASS[@]} $ICON_ERROR Failed: ${#FAIL[@]}"
|
||||
echo ""
|
||||
|
||||
echo "$ICON_SYNC Sync jobs:"
|
||||
if [[ ${#PASS[@]} -gt 0 ]]; then
|
||||
for job in "${PASS[@]}"; do echo " $ICON_SUCCESS $job"; done
|
||||
fi
|
||||
if [[ ${#FAIL[@]} -gt 0 ]]; then
|
||||
for job in "${FAIL[@]}"; do echo " $ICON_ERROR $job"; done
|
||||
fi
|
||||
echo " Passed: ${#PASS[@]} Failed: ${#FAIL[@]}"
|
||||
|
||||
if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
echo "$ICON_GEAR Post-sync jobs:"
|
||||
for job in "${JOB_PASS[@]}"; do echo " $ICON_SUCCESS $job"; done
|
||||
for job in "${JOB_FAIL[@]}"; do echo " $ICON_ERROR $job"; done
|
||||
fi
|
||||
|
||||
echo ""
|
||||
|
||||
TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} ))
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "$ICON_WARN Status: DRY RUN — no changes made"
|
||||
elif [[ ${#FAIL[@]} -eq 0 ]]; then
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS ALL JOBS COMPLETE"
|
||||
notify "Critical maintenance complete on $(hostname) — synced + updated (local=$CRITICAL_SYNC_UPDATES remote=$CRITICAL_SYNC_UPDATES_REMOTE)" "Critical Maintenance" "normal"
|
||||
elif [[ "$TOTAL_FAIL" -eq 0 ]]; then
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS ALL COMPLETE"
|
||||
notify "Weekly sync maintenance complete on $(hostname) — synced + updated (local=$CRITICAL_SYNC_UPDATES remote=$CRITICAL_SYNC_UPDATES_REMOTE)" "Weekly Maintenance" "normal"
|
||||
else
|
||||
echo "$ICON_ERROR Status: $ICON_ERROR ${#FAIL[@]} JOB(S) FAILED"
|
||||
notify "Critical maintenance failed on $(hostname) — failed: ${FAIL[*]}" "Critical Maintenance" "warning"
|
||||
echo "$ICON_ERROR Status: $TOTAL_FAIL failure(s) — check logs"
|
||||
notify "Weekly sync maintenance failed on $(hostname) — sync: ${#FAIL[@]} failed, jobs: ${#JOB_FAIL[@]} failed" "Weekly Maintenance" "warning"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
+370
-193
@@ -1,6 +1,6 @@
|
||||
# Rsync Setup Guide
|
||||
> **Status:** Work in Progress
|
||||
> For the unRAID Rsync Ecosystem — `common.sh` · `Master.conf` · `rsync.sh` · `daily_sync.sh`
|
||||
|
||||
> For the unRAID Script Ecosystem — `Master.conf` · `common.sh` · `rsync.sh` · `daily_sync_maintenance.sh` · `weekly_sync_maintenance.sh`
|
||||
|
||||
---
|
||||
|
||||
@@ -8,11 +8,13 @@
|
||||
|
||||
This guide walks through setting up the rsync ecosystem on both your primary and secondary unRAID 7.x servers. By the end you will have:
|
||||
|
||||
- A Gitea repository cloned to both servers
|
||||
- SSH keys configured for server-to-server communication
|
||||
- Tailscale running on both servers for secure networking
|
||||
- Scripts scheduled and running via the User Scripts plugin
|
||||
- Automated daily sync of media shares and appdata profiles
|
||||
- A Gitea repository cloned to both servers
|
||||
- All scripts scheduled via the User Scripts plugin
|
||||
- Automated daily sync of media shares driven by orchestrators
|
||||
- Automated weekly clean sync of critical appdata (Emby + auth stack)
|
||||
- Optional personal encrypted shares synced for offsite backup
|
||||
|
||||
---
|
||||
|
||||
@@ -21,141 +23,130 @@ This guide walks through setting up the rsync ecosystem on both your primary and
|
||||
Both servers need the following before starting:
|
||||
|
||||
- unRAID 7.x
|
||||
- [Community Applications plugin](https://forums.unraid.net/topic/38582-plug-in-community-applications/) installed
|
||||
- [User Scripts plugin](https://forums.unraid.net/topic/48286-plugin-user-scripts/) installed via Community Applications
|
||||
- [Tailscale plugin](https://forums.unraid.net/topic/136889-tailscale-plugin/) installed via Community Applications
|
||||
- Access to a Gitea instance (self-hosted or remote)
|
||||
- Terminal access to both servers (via unRAID UI → Tools → Terminal, or SSH)
|
||||
- Access to a Gitea instance (self-hosted recommended — Gitea runs as a Docker container on HOST1)
|
||||
- Terminal access to both servers (unRAID UI → Tools → Terminal, or SSH)
|
||||
|
||||
---
|
||||
|
||||
## Step 1 — Tailscale Setup
|
||||
|
||||
Tailscale provides the secure network tunnel between your two servers. The scripts resolve the remote server's IP via Tailscale at runtime.
|
||||
Tailscale provides the secure network tunnel between your two servers. Scripts resolve the remote server's IP via Tailscale at runtime — no hardcoded IPs needed.
|
||||
|
||||
### On Both Servers
|
||||
|
||||
1. Open **Apps** in the unRAID UI
|
||||
2. Search for **Tailscale** and install the plugin
|
||||
3. Once installed go to **Settings → Tailscale**
|
||||
3. Go to **Settings → Tailscale**
|
||||
4. Click **Connect** and authenticate with your Tailscale account
|
||||
5. Verify both servers appear in your [Tailscale admin console](https://login.tailscale.com/admin/machines)
|
||||
|
||||
### Verify Connectivity
|
||||
|
||||
Run this on the primary to confirm it can see the secondary:
|
||||
Run this on HOST1 to confirm it can reach HOST2:
|
||||
|
||||
```bash
|
||||
tailscale ip -4 unRAID-Jayred365
|
||||
```
|
||||
|
||||
You should get back a `100.x.x.x` IP. If not, check that both machines are authenticated and connected in the Tailscale admin console.
|
||||
You should get back a `100.x.x.x` IP. If not, check both machines are authenticated in the Tailscale admin console.
|
||||
|
||||
> **Note:** The hostnames used in `Master.conf` (`HOST1` and `HOST2`) must match the Tailscale machine names exactly — these are case sensitive.
|
||||
> **Important:** The hostnames in `Master.conf` (`HOST1` and `HOST2`) must match the Tailscale machine names exactly — case sensitive.
|
||||
|
||||
---
|
||||
|
||||
## Step 2 — Generate SSH Keys
|
||||
## Step 2 — Enable SSH on unRAID
|
||||
|
||||
unRAID 7.x has SSH disabled by default. Enable it on both servers so scripts can connect between them.
|
||||
|
||||
1. Go to **Settings → Management Access**
|
||||
2. Under **Secure Shell** set **SSH** to `Enabled`
|
||||
3. Set **SSH port** to `22`
|
||||
4. Click **Apply**
|
||||
|
||||
> SSH is only exposed on your local network and Tailscale interface. Scripts connect via Tailscale IP — traffic is encrypted end-to-end.
|
||||
|
||||
---
|
||||
|
||||
## Step 3 — Generate SSH Keys
|
||||
|
||||
The scripts use SSH keys for two purposes:
|
||||
- **Server-to-server rsync** — primary authenticates to secondary (and vice versa)
|
||||
- **Server-to-server rsync and failover** — each server authenticates to the other
|
||||
- **Gitea access** — both servers pull from the git repository
|
||||
|
||||
### 2a — Server-to-Server Keys
|
||||
### 3a — Server-to-Server Keys
|
||||
|
||||
Run the following on **each server** to generate its rsync key. Replace the filename with the appropriate server name.
|
||||
|
||||
**On Primary (unRAID-Gmer4Lfe):**
|
||||
**On HOST1 (unRAID-Gmer4Lfe):**
|
||||
|
||||
```bash
|
||||
ssh-keygen -t ed25519 -f /root/.ssh/Gmer4Lfe-rsync-key -C "gmer4lfe-rsync" -N ""
|
||||
```
|
||||
|
||||
**On Secondary (unRAID-Jayred365):**
|
||||
**On HOST2 (unRAID-Jayred365):**
|
||||
|
||||
```bash
|
||||
ssh-keygen -t ed25519 -f /root/.ssh/Jayred365-rsync-key -C "jayred365-rsync" -N ""
|
||||
```
|
||||
|
||||
### 2b — Copy Public Keys to Each Server
|
||||
### 3b — Authorise Keys on Each Server
|
||||
|
||||
The primary's public key must be authorised on the secondary, and vice versa.
|
||||
HOST1's public key must be authorised on HOST2, and vice versa.
|
||||
|
||||
**Copy primary key → secondary:**
|
||||
**Copy HOST1 key → HOST2:**
|
||||
|
||||
```bash
|
||||
# Run on primary
|
||||
# On HOST1 — print the public key
|
||||
cat /root/.ssh/Gmer4Lfe-rsync-key.pub
|
||||
```
|
||||
|
||||
Copy the output. Then on the secondary:
|
||||
|
||||
```bash
|
||||
# Run on secondary
|
||||
# On HOST2 — paste and authorise
|
||||
mkdir -p /root/.ssh
|
||||
echo "PASTE_PUBLIC_KEY_HERE" >> /root/.ssh/authorized_keys
|
||||
chmod 600 /root/.ssh/authorized_keys
|
||||
```
|
||||
|
||||
**Copy secondary key → primary:**
|
||||
**Copy HOST2 key → HOST1:**
|
||||
|
||||
```bash
|
||||
# Run on secondary
|
||||
# On HOST2 — print the public key
|
||||
cat /root/.ssh/Jayred365-rsync-key.pub
|
||||
```
|
||||
|
||||
Copy the output. Then on the primary:
|
||||
|
||||
```bash
|
||||
# Run on primary
|
||||
mkdir -p /root/.ssh
|
||||
# On HOST1 — paste and authorise
|
||||
echo "PASTE_PUBLIC_KEY_HERE" >> /root/.ssh/authorized_keys
|
||||
chmod 600 /root/.ssh/authorized_keys
|
||||
```
|
||||
|
||||
### 2c — Test the Connection
|
||||
### 3c — Test the Connection
|
||||
|
||||
From the primary, test that it can SSH to the secondary without a password prompt:
|
||||
From HOST1, verify it can SSH to HOST2 without a password prompt:
|
||||
|
||||
```bash
|
||||
ssh -i /root/.ssh/Gmer4Lfe-rsync-key root@$(tailscale ip -4 unRAID-Jayred365) "echo connected"
|
||||
```
|
||||
|
||||
You should see `connected`. If prompted for a password the key was not authorised correctly — recheck Step 2b.
|
||||
You should see `connected`. If prompted for a password the key was not authorised correctly — recheck Step 3b.
|
||||
|
||||
### 2d — Gitea SSH Key
|
||||
### 3d — Gitea SSH Key
|
||||
|
||||
Generate a separate key for Gitea access on each server:
|
||||
|
||||
```bash
|
||||
ssh-keygen -t ed25519 -f /root/.ssh/id_gitea_rsync -C "unraid-gitea" -N ""
|
||||
ssh-keygen -t ed25519 -f /root/.ssh/unraid_gitea -C "unraid-gitea" -N ""
|
||||
```
|
||||
|
||||
Add the public key to your Gitea account:
|
||||
|
||||
```bash
|
||||
cat /root/.ssh/id_gitea_rsync.pub
|
||||
cat /root/.ssh/unraid_gitea.pub
|
||||
```
|
||||
|
||||
Copy the output and add it in Gitea under **Settings → SSH / GPG Keys → Add Key**.
|
||||
|
||||
---
|
||||
|
||||
## Step 3 — Enable SSH on unRAID
|
||||
|
||||
unRAID 7.x has SSH disabled by default. Enable it on both servers so the scripts can connect between them.
|
||||
|
||||
1. Go to **Settings → Management Access**
|
||||
2. Under **Secure Shell** set **SSH** to `Enabled`
|
||||
3. Set **SSH port** to `22` (default)
|
||||
4. Click **Apply**
|
||||
|
||||
> **Security note:** SSH is only exposed on your local network and Tailscale interface. The rsync scripts connect via the Tailscale IP so traffic is encrypted end-to-end.
|
||||
|
||||
---
|
||||
|
||||
## Step 4 — Clone the Git Repository
|
||||
|
||||
The scripts live in a Gitea repository. Both servers clone from the same repo so updates propagate everywhere via a single git pull.
|
||||
Both servers clone from the same Gitea repository. Updates pushed to the repo propagate to both servers on the next daily git pull.
|
||||
|
||||
### On Both Servers
|
||||
|
||||
@@ -164,12 +155,12 @@ The scripts live in a Gitea repository. Both servers clone from the same repo so
|
||||
mkdir -p /mnt/user/appdata/unraid_scripts
|
||||
|
||||
# Clone the repository
|
||||
GIT_SSH_COMMAND="ssh -i /root/.ssh/id_gitea_rsync" \
|
||||
git clone git@YOUR_GITEA_HOST:YOUR_USER/Unraid_Scripts.git \
|
||||
GIT_SSH_COMMAND="ssh -i /root/.ssh/unraid_gitea" \
|
||||
git clone git@YOUR_GITEA_HOST:FailedProxy/Unraid_Scripts.git \
|
||||
/mnt/user/appdata/unraid_scripts
|
||||
```
|
||||
|
||||
Replace `YOUR_GITEA_HOST` and `YOUR_USER` with your Gitea server address and username.
|
||||
Replace `YOUR_GITEA_HOST` with your Gitea server address and port.
|
||||
|
||||
### Verify the Structure
|
||||
|
||||
@@ -182,252 +173,385 @@ You should see:
|
||||
```
|
||||
Master.conf
|
||||
common.sh
|
||||
Rsync/
|
||||
rsync.sh
|
||||
Orchestrators/
|
||||
daily_sync.sh
|
||||
Rsync/
|
||||
Failover/
|
||||
Docker_Essentials/
|
||||
unRAID_Essentials/
|
||||
Media/
|
||||
Transcodes/
|
||||
Monitors/
|
||||
Tools/
|
||||
recreate_shares.sh
|
||||
```
|
||||
|
||||
### Make Scripts Executable
|
||||
|
||||
```bash
|
||||
chmod +x /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh
|
||||
chmod +x /mnt/user/appdata/unraid_scripts/Orchestrators/daily_sync.sh
|
||||
chmod +x /mnt/user/appdata/unraid_scripts/Tools/recreate_shares.sh
|
||||
find /mnt/user/appdata/unraid_scripts -name "*.sh" -exec chmod +x {} \;
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Step 5 — Configure Master.conf
|
||||
|
||||
All user configuration lives in `Master.conf`. Open it and adjust the following to match your setup:
|
||||
All user configuration lives in `Master.conf`. Open it and fill in your values:
|
||||
|
||||
```bash
|
||||
nano /mnt/user/appdata/unraid_scripts/Master.conf
|
||||
```
|
||||
|
||||
### Required Changes
|
||||
|
||||
| Variable | Description | Example |
|
||||
|---|---|---|
|
||||
| `HOST1` | Hostname of your primary server | `unRAID-Gmer4Lfe` |
|
||||
| `HOST2` | Hostname of your secondary server | `unRAID-Jayred365` |
|
||||
| `HOST1_SSH_KEY` | Path to primary's rsync private key | `/root/.ssh/Gmer4Lfe-rsync-key` |
|
||||
| `HOST2_SSH_KEY` | Path to secondary's rsync private key | `/root/.ssh/Jayred365-rsync-key` |
|
||||
| `REPO_SSH` | SSH URL of your Gitea repository | `git@192.168.50.2:User/Unraid_Scripts.git` |
|
||||
| `GITEA_SSH_KEY` | Path to Gitea private key | `/root/.ssh/id_gitea_rsync` |
|
||||
| `BW_LIMIT` | Global bandwidth limit in KB/s | `12500` |
|
||||
| `ROOTFS_WARN` | Remote rootfs % threshold before aborting | `75` |
|
||||
|
||||
### Daily Sync Shares
|
||||
|
||||
Add the full paths of all media shares you want synced nightly:
|
||||
### Host Configuration
|
||||
|
||||
```bash
|
||||
DAILY_SYNC_SHARES=(
|
||||
HOST1="unRAID-Gmer4Lfe" # must match Tailscale machine name exactly
|
||||
HOST2="unRAID-Jayred365"
|
||||
|
||||
HOST1_SSH_KEY="/root/.ssh/Gmer4Lfe-rsync-key"
|
||||
HOST2_SSH_KEY="/root/.ssh/Jayred365-rsync-key"
|
||||
|
||||
HOST1_EMBY_CONTAINER="Emby"
|
||||
HOST1_EMBY_URL="http://localhost:8096"
|
||||
HOST1_EMBY_API_KEY="your-host1-emby-api-key" # Emby Dashboard → API Keys → + New Key
|
||||
|
||||
HOST2_EMBY_CONTAINER="Emby-Jayred365"
|
||||
HOST2_EMBY_URL="http://localhost:8096"
|
||||
HOST2_EMBY_API_KEY="your-host2-emby-api-key"
|
||||
```
|
||||
|
||||
### Git / Repo
|
||||
|
||||
```bash
|
||||
GITEA_CONTAINER="Gitea" # exact Docker container name
|
||||
GITEA_REPO_PATH="FailedProxy/Unraid_Scripts.git"
|
||||
GITEA_DOMAIN="" # optional public domain fallback
|
||||
TARGET_DIR="/mnt/user/appdata/unraid_scripts"
|
||||
GITEA_SSH_KEY="/root/.ssh/unraid_gitea"
|
||||
SSH_PORT=221
|
||||
```
|
||||
|
||||
### Orchestrators — Daily Sync Shares
|
||||
|
||||
Define which shares each server owns. Each server only pushes the shares it is source of truth for — the other server mirrors and treats them as read-only.
|
||||
|
||||
```bash
|
||||
HOST1_DAILY_SYNC_SHARES=(
|
||||
/mnt/user/Movies
|
||||
/mnt/user/Tv_Shows
|
||||
/mnt/user/Music
|
||||
# add more here
|
||||
# add all HOST1-managed shares here
|
||||
)
|
||||
|
||||
HOST2_DAILY_SYNC_SHARES=(
|
||||
/mnt/user/Anime_Shows
|
||||
/mnt/user/Anime_Movies
|
||||
# add all HOST2-managed shares here
|
||||
)
|
||||
```
|
||||
|
||||
### Profiles
|
||||
> Never put the same share in both lists. One server is always the truth holder for each share.
|
||||
|
||||
Profiles control per-share rsync behaviour for your frequently synced appdata shares. Each profile is matched by the directory basename (lowercased) — or overridden with `--profile=name`.
|
||||
### Orchestrators — Weekly Sync Jobs
|
||||
|
||||
Shares synced during the Sunday maintenance window with containers stopped both sides:
|
||||
|
||||
```bash
|
||||
# One array drives both local and remote container stops
|
||||
# Local stops first (flush databases) then remote stops (clean receive)
|
||||
# Same container names on both HOST1 and HOST2 — no duplication needed
|
||||
# Containers not found on a server are skipped gracefully
|
||||
declare -A PROFILE_CRITICAL_CONTAINER_NAMES=(
|
||||
[critical-data]="Mariadb-Authelia Mariadb-Authelia-Secondary Redis-Authelia Redis-Authelia-Secondary Lldap-Gmer4Lfe NginxProxyManager Authelia Authelia-Secondary"
|
||||
[important-data]="Postgres-NextCloud NextCloud"
|
||||
[emby]="Emby" # nightly clean sync — Emby stopped both sides
|
||||
[emby-failover]="" # dirty sync — Emby stays running
|
||||
WEEKLY_SYNC_JOBS=(
|
||||
"/mnt/user/Media_Server/Emby" # full clean Emby mirror
|
||||
"/mnt/user/appdata-Failover/Critical-Data" # auth stack
|
||||
)
|
||||
```
|
||||
|
||||
Any share with no matching profile falls through to the global `DEFAULT_RSYNC_OPTS`. Containers not found on a server are skipped gracefully — only containers that were actually running get restarted.
|
||||
---
|
||||
|
||||
**Two Emby profiles:**
|
||||
## Step 6 — Rsync Profiles
|
||||
|
||||
Profiles control per-share rsync behaviour for appdata syncs. The profile key is matched automatically by the basename of the directory passed to `rsync.sh` (lowercased). Override with `--profile=name`.
|
||||
|
||||
One array drives both local and remote container stops. Same container names on both servers — consistent naming is a requirement of this ecosystem.
|
||||
|
||||
### Current Profiles
|
||||
|
||||
| Profile | Purpose | Containers Stopped |
|
||||
|---|---|---|
|
||||
| `arrs_stack` | Arr databases | Sonarr, Radarr, Lidarr, Prowlarr, Bazarr, Pinchflat |
|
||||
| `critical-data` | Auth stack | Mariadb, Redis, LLDAP, NPM, Authelia (delayed start) |
|
||||
| `important-data` | NextCloud + Postgres | Postgres, NextCloud (delayed start) |
|
||||
| `gmer4lfe` | Server-specific appdata | Organizr, UptimeKuma, VaultWarden |
|
||||
| `emby` | Weekly clean sync | Emby both sides — WAL checkpointed |
|
||||
| `emby-failover` | Frequent dirty sync | None — Emby stays running |
|
||||
|
||||
### Two Emby Profiles
|
||||
|
||||
```
|
||||
emby-failover — frequent dirty sync (every 30-60min):
|
||||
Emby stays running on both sides
|
||||
emby-failover — every 30-60min, Emby stays running:
|
||||
WAL and SHM excluded — safe while Emby is active
|
||||
Only critical failover data: users.db, library.db, authentication.db, config/
|
||||
Fast, small dataset, high bandwidth
|
||||
This is also what gets written back during failover handback
|
||||
Critical failover data only: users.db, library.db, authentication.db, config/
|
||||
Fast, small dataset — users continue watching without interruption on failover
|
||||
Also used for failover writeback on handback
|
||||
|
||||
emby — weekly clean sync (via nightly_critical_full_sync.sh Sunday 2:30am):
|
||||
Emby stopped on both sides — WAL checkpointed on shutdown
|
||||
emby — weekly Sunday 2:30am, both Emby instances stopped:
|
||||
WAL checkpointed on shutdown — full consistent mirror
|
||||
Full mirror: metadata, plugins, config all included
|
||||
Minimal excludes: logs, transcodes, cache, crash files only
|
||||
Complete faithful state pushed once per week
|
||||
Cache stays warm on HOST2 all week — only reset on Sunday
|
||||
emby-failover handles the critical state between weekly syncs
|
||||
```
|
||||
|
||||
Usage with profile override:
|
||||
```bash
|
||||
# Dirty sync — Emby stays running
|
||||
bash rsync.sh /mnt/user/Media_Server/Emby --profile=emby-failover
|
||||
|
||||
# Clean sync — called by nightly_critical_full_sync.sh, Emby stopped via profile
|
||||
bash rsync.sh /mnt/user/Media_Server/Emby
|
||||
emby-failover covers the critical state between weekly syncs
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Step 6 — Set Up User Scripts
|
||||
## Step 7 — Personal Encrypted Shares
|
||||
|
||||
The User Scripts plugin is how unRAID schedules and runs the scripts. Each sync job is its own script entry in the plugin.
|
||||
Personal shares can be synced to the remote server for offsite backup. ZFS encrypts at the dataset level — the remote server receives encrypted blocks and cannot read the content without your passphrase or keyfile.
|
||||
|
||||
### Frequent Sync Jobs (Scheduled Individually)
|
||||
### ZFS Encryption Setup (unRAID 7)
|
||||
|
||||
Create one script entry per appdata profile. Go to **Plugins → User Scripts → Add New Script**.
|
||||
**Step 1 — Create an encrypted dataset:**
|
||||
|
||||
Name it descriptively — e.g. `rsync appdata arrs_stack`.
|
||||
1. In the unRAID UI go to **Main** → click your ZFS pool name
|
||||
2. Click **+ Dataset** to create a new dataset
|
||||
3. Name it — e.g. `Gmer4Lfe-Personal`
|
||||
4. Enable **Encryption** → set your passphrase
|
||||
> ⚠️ Write your passphrase down — if lost, data is unrecoverable
|
||||
|
||||
In the script body paste:
|
||||
**Step 2 — Create the share:**
|
||||
|
||||
1. Go to **Settings → Shares → Add Share**
|
||||
2. Set the share path to your new encrypted dataset
|
||||
3. Set **Use cache:** `Only` — keeps data on ZFS pool, not array
|
||||
|
||||
**Step 3 — Verify encryption is active before syncing:**
|
||||
|
||||
```bash
|
||||
zfs get encryption poolname/Gmer4Lfe-Personal
|
||||
# Should show: encryption aes-256-gcm
|
||||
```
|
||||
|
||||
**Step 4 — Auto-unlock on boot (keyfile approach — optional):**
|
||||
|
||||
```bash
|
||||
# Create keyfile — on HOST1 only, never sync this file
|
||||
dd if=/dev/urandom bs=32 count=1 | base64 > /root/.zfs-keys/personal.key
|
||||
chmod 600 /root/.zfs-keys/personal.key
|
||||
|
||||
# Set dataset to use keyfile
|
||||
zfs change-key -o keylocation=file:///root/.zfs-keys/personal.key \
|
||||
-o keyformat=raw poolname/Gmer4Lfe-Personal
|
||||
|
||||
# Add to array start (via array_start.sh or User Scripts):
|
||||
zfs load-key poolname/Gmer4Lfe-Personal
|
||||
zfs mount poolname/Gmer4Lfe-Personal
|
||||
```
|
||||
|
||||
**Manual unlock alternative (most secure):**
|
||||
|
||||
```bash
|
||||
zfs load-key poolname/Gmer4Lfe-Personal # prompts for passphrase
|
||||
zfs mount poolname/Gmer4Lfe-Personal
|
||||
```
|
||||
|
||||
**Step 5 — Add to Master.conf:**
|
||||
|
||||
```bash
|
||||
HOST1_PERSONAL_SHARES=(
|
||||
/mnt/user/Gmer4Lfe-Personal
|
||||
)
|
||||
```
|
||||
|
||||
Personal shares sync automatically with the daily media share sync in `daily_sync_maintenance.sh`. The remote server receives encrypted blocks — content is unreadable without your key.
|
||||
|
||||
---
|
||||
|
||||
## Step 8 — Set Up User Scripts
|
||||
|
||||
The ecosystem uses a single orchestrator entry for array startup plus a small number of scheduled scripts.
|
||||
|
||||
### At Startup of Array
|
||||
|
||||
Create one script entry named `array start`:
|
||||
|
||||
```bash
|
||||
#!/bin/bash
|
||||
bash /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-Failover/Arrs_Stack
|
||||
bash /mnt/user/appdata/unraid_scripts/Orchestrators/array_start.sh
|
||||
```
|
||||
|
||||
Set the schedule to match your desired frequency:
|
||||
Set schedule to: **At Startup of Array**
|
||||
|
||||
| Profile | Schedule | Notes |
|
||||
This single entry launches everything configured in `ARRAY_START_SCRIPTS` in `Master.conf`:
|
||||
- `ramdisk_setup.sh` — creates ramdisk before Emby starts
|
||||
- `docker_syslog_filter.sh` — suppresses veth log noise
|
||||
- `php_fpm_max_children.sh` — WebGUI tuning
|
||||
- `docker_network_connect.sh` — connects containers to extra networks
|
||||
- `system_watchdog.sh` — continuous system health monitor
|
||||
- `docker_watchdog.sh` — continuous container health monitor
|
||||
- `failover.sh` — continuous mutual failover
|
||||
|
||||
### Cron Schedules
|
||||
|
||||
| Script | Schedule | Purpose |
|
||||
|---|---|---|
|
||||
| `emby-failover` | Every 30-60 min | Dirty sync — Emby running, critical data only |
|
||||
| `emby` | Weekly Sunday via `nightly_critical_full_sync.sh` | Clean sync — Emby stopped, full mirror, cache stays warm all week |
|
||||
| `Critical-Data` | Weekly Sunday via `nightly_critical_full_sync.sh` | Auth stack — clean weekly sync |
|
||||
| `Important-Data` | Every 6-12 hours | NextCloud file changes |
|
||||
| `Arrs_Stack` | Every 12-24 hours | Arr databases |
|
||||
| `Gmer4Lfe` | Daily or weekly | Personal appdata, rarely changes |
|
||||
| `transcode_management.sh` | `*/3 * * * *` | Transcode cleanup + manager |
|
||||
| `arrs_failed_stalled_recovery.sh` | `0 */6 * * *` | Blocklist + re-search failed imports |
|
||||
| `rsync.sh ... --profile=emby-failover` | `*/30 * * * *` | Emby dirty sync |
|
||||
| `daily_sync_maintenance.sh` | `0 1 * * *` | Full daily maintenance window |
|
||||
| `weekly_sync_maintenance.sh` | `30 2 * * 0` | Weekly sync + updates + restarts |
|
||||
| `weekly_health_digest.sh` | `0 8 * * 6` | Saturday morning health digest |
|
||||
|
||||
> **Important:** Set each script to run as a **Background Task** — this ensures output streams correctly to the log rather than buffering in the browser.
|
||||
### Emby Failover Dirty Sync
|
||||
|
||||
### Daily Sync Orchestrator
|
||||
|
||||
Create one more script entry for the daily media sync:
|
||||
|
||||
Name it `daily media sync`.
|
||||
|
||||
In the script body paste:
|
||||
Create a separate script entry named `rsync emby failover`:
|
||||
|
||||
```bash
|
||||
#!/bin/bash
|
||||
bash /mnt/user/appdata/unraid_scripts/Orchestrators/daily_sync.sh
|
||||
bash /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh \
|
||||
/mnt/user/Media_Server/Emby --profile=emby-failover
|
||||
```
|
||||
|
||||
Set the schedule to **Daily at 01:00**.
|
||||
Set schedule to: `*/30 * * * *`
|
||||
|
||||
### Appdata Profile Syncs
|
||||
|
||||
Create one entry per appdata profile you want on a schedule:
|
||||
|
||||
```bash
|
||||
#!/bin/bash
|
||||
bash /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh \
|
||||
/mnt/user/appdata-Failover/Arrs_Stack
|
||||
```
|
||||
|
||||
| Profile | Recommended Schedule |
|
||||
|---|---|
|
||||
| `Arrs_Stack` | Every 12-24 hours |
|
||||
| `Important-Data` | Every 6-12 hours |
|
||||
| `Gmer4Lfe` | Daily or weekly |
|
||||
| `emby` | Via `weekly_sync_maintenance.sh` only — do NOT schedule separately |
|
||||
| `Critical-Data` | Via `weekly_sync_maintenance.sh` only — do NOT schedule separately |
|
||||
|
||||
> Set all scripts to run as **Background Task** — output streams correctly rather than buffering in the browser.
|
||||
|
||||
---
|
||||
|
||||
## Step 7 — Verify the Setup
|
||||
## Step 9 — Verify the Setup
|
||||
|
||||
Before letting the scheduled jobs run, do a manual test from the terminal on the primary:
|
||||
Before letting scheduled jobs run, test manually from the terminal on HOST1:
|
||||
|
||||
```bash
|
||||
bash /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-Failover/Arrs_Stack --log
|
||||
# Test a single appdata profile sync
|
||||
bash /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh \
|
||||
/mnt/user/appdata-Failover/Arrs_Stack --dry-run --log
|
||||
|
||||
# Test the daily sync orchestrator
|
||||
bash /mnt/user/appdata/unraid_scripts/Orchestrators/daily_sync_maintenance.sh --dry-run
|
||||
```
|
||||
|
||||
A healthy run will show:
|
||||
|
||||
```
|
||||
━━━ ⚙️ Setup ━━━
|
||||
ℹ️ [INFO] 🖥️ Host: unRAID-Gmer4Lfe → unRAID-Jayred365
|
||||
ℹ️ [INFO] 🌐 Remote IP: 100.x.x.x
|
||||
ℹ️ Host: unRAID-Gmer4Lfe → unRAID-Jayred365
|
||||
ℹ️ Remote IP: 100.x.x.x
|
||||
|
||||
━━━ 🛡️ Pre-flight Checks ━━━
|
||||
ℹ️ [INFO] 📡 unRAID-Jayred365 is reachable
|
||||
ℹ️ [INFO] 🩺 Remote rootfs: 12% used (threshold: 75%)
|
||||
ℹ️ [INFO] 🩺 Remote share verified: ...
|
||||
ℹ️ [INFO] 💾 disk1 🟢 — share present
|
||||
✅ [OK] All disks backing share are online
|
||||
✅ Remote reachable
|
||||
✅ Remote rootfs: 12% (threshold: 75%)
|
||||
✅ All pre-flight checks passed
|
||||
```
|
||||
|
||||
If any pre-flight check fails the script will abort with a clear error and hint before touching anything.
|
||||
If any pre-flight check fails the script aborts with a clear error before touching anything.
|
||||
|
||||
---
|
||||
|
||||
## Step 8 — Secondary Server Initial Setup
|
||||
## Step 10 — Secondary Server Initial Sync
|
||||
|
||||
If setting up the secondary from scratch (no existing data):
|
||||
If setting up HOST2 from scratch with empty shares:
|
||||
|
||||
1. Complete Steps 1–5 on the secondary
|
||||
1. Complete Steps 1–8 on HOST2
|
||||
2. Start the array and create your shares in the unRAID UI
|
||||
3. Run the share recreation tool to create disk directories from your cfg files:
|
||||
3. Run the share recreation tool to create disk directories from cfg files:
|
||||
|
||||
```bash
|
||||
bash /mnt/user/appdata/unraid_scripts/Tools/recreate_shares.sh
|
||||
```
|
||||
|
||||
4. Temporarily remove `--delete` from `DEFAULT_RSYNC_OPTS` in `Master.conf`
|
||||
5. Run the initial push from the primary — the `.recovery` marker files allow rsync to populate empty shares without aborting
|
||||
6. Once complete, restore `--delete` to `Master.conf`
|
||||
7. The next nightly run will clean up the `.recovery` marker files automatically
|
||||
4. Run the initial push from HOST1 — this populates HOST2's empty shares:
|
||||
|
||||
```bash
|
||||
bash /mnt/user/appdata/unraid_scripts/Orchestrators/daily_sync_maintenance.sh --log
|
||||
```
|
||||
|
||||
5. Once complete, scheduled runs take over automatically.
|
||||
|
||||
---
|
||||
|
||||
## Naming Consistency — Required
|
||||
|
||||
The ecosystem is built on the assumption that containers and shares have identical names on both servers. This is not optional — it is what makes one codebase work on both servers without modification.
|
||||
|
||||
```
|
||||
Container names must match exactly:
|
||||
Emby ← HOST1 and HOST2
|
||||
NginxProxyManager ← HOST1 and HOST2
|
||||
Mariadb-Authelia ← HOST1 and HOST2
|
||||
|
||||
Share paths must match exactly:
|
||||
/mnt/user/Movies ← HOST1 and HOST2
|
||||
/mnt/user/Tv_Shows ← HOST1 and HOST2
|
||||
```
|
||||
|
||||
If a container or share has a different name on one server the script skips it gracefully — but it will not do what you expect. Diverge from consistent naming and every script that touches containers or shares needs custom logic for each server. Keep naming consistent and one codebase covers both servers automatically.
|
||||
|
||||
---
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### SSH connection refused
|
||||
- Verify SSH is enabled on the target server (Step 3)
|
||||
- Check the correct key is referenced in `Master.conf`
|
||||
- Confirm the Tailscale IP resolves: `tailscale ip -4 HOSTNAME`
|
||||
- Verify SSH is enabled (Step 2)
|
||||
- Confirm the correct key is referenced in `Master.conf`
|
||||
- Test Tailscale: `tailscale ip -4 HOSTNAME`
|
||||
|
||||
### Pre-flight aborts on rootfs
|
||||
- Remote rootfs is above `ROOTFS_WARN` threshold
|
||||
- Check if the remote array is started and drives are mounted
|
||||
- `df /` on the remote to see current usage
|
||||
- Remote rootfs above `ROOTFS_WARN` threshold
|
||||
- Check remote array is started and drives are mounted
|
||||
- Run `df /` on the remote to see current usage
|
||||
|
||||
### Pre-flight aborts on empty share
|
||||
- Share exists but has no content — drives may not be mounted
|
||||
- Run `recreate_shares.sh` if setting up fresh
|
||||
- Check array status on the remote server
|
||||
|
||||
### Pre-flight aborts on disk check
|
||||
- One or more disks backing the share are not mounted
|
||||
- Check **Main → Array Devices** on the remote for offline disks
|
||||
- Verify disk assignments are correct after any hardware changes
|
||||
|
||||
### Script not found
|
||||
- Verify the repo was cloned to `/mnt/user/appdata/unraid_scripts/`
|
||||
- Check scripts are executable: `chmod +x /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh`
|
||||
|
||||
### Containers not stopping/starting
|
||||
- Verify container names in `Master.conf` match exactly what Docker shows
|
||||
- Check SSH key has access to run docker commands on the remote
|
||||
- Verify container names in `Master.conf` match Docker exactly — case sensitive
|
||||
- Test manually: `ssh -i /root/.ssh/KEY root@REMOTE_IP "docker ps"`
|
||||
|
||||
### Script not found
|
||||
- Verify repo was cloned to `/mnt/user/appdata/unraid_scripts/`
|
||||
- Make scripts executable: `find /mnt/user/appdata/unraid_scripts -name "*.sh" -exec chmod +x {} \;`
|
||||
|
||||
### Profile not matching
|
||||
- Profile key is matched by directory basename lowercased
|
||||
- `/mnt/user/appdata-Failover/Arrs_Stack` → basename `Arrs_Stack` → key `arrs_stack`
|
||||
- Override with `--profile=name` if basename doesn't match
|
||||
|
||||
---
|
||||
|
||||
## Available Flags
|
||||
|
||||
All scripts support the following flags:
|
||||
All scripts support:
|
||||
|
||||
| Flag | Description |
|
||||
|---|---|
|
||||
| `--dry-run` or `-n` | Run without making any changes |
|
||||
| `--dry-run` | Run without making any changes |
|
||||
| `--log` | Enable verbose logging output |
|
||||
| `--no-log` | Disable logging (overrides Master.conf) |
|
||||
| `--no-log` | Disable logging |
|
||||
| `--status` | Print resolved configuration and exit |
|
||||
|
||||
Example:
|
||||
|
||||
```bash
|
||||
# Preview what would be synced without transferring anything
|
||||
# Preview what would be synced
|
||||
bash rsync.sh /mnt/user/Movies --dry-run --log
|
||||
|
||||
# Check what profile and settings resolved for a share
|
||||
# Check resolved profile settings
|
||||
bash rsync.sh /mnt/user/appdata-Failover/Arrs_Stack --status
|
||||
|
||||
# Test daily orchestrator without changes
|
||||
bash daily_sync_maintenance.sh --dry-run
|
||||
```
|
||||
|
||||
---
|
||||
@@ -438,14 +562,67 @@ bash rsync.sh /mnt/user/appdata-Failover/Arrs_Stack --status
|
||||
Unraid_Scripts/
|
||||
├── Master.conf # All user configuration — edit this file only
|
||||
├── common.sh # Shared library — functions used by all scripts
|
||||
│
|
||||
├── Orchestrators/
|
||||
│ ├── array_start.sh # Single array-start entry point
|
||||
│ ├── daily_sync_maintenance.sh # Daily maintenance window orchestrator
|
||||
│ ├── weekly_sync_maintenance.sh # Weekly maintenance window orchestrator
|
||||
│ ├── media_management.sh # Permissions + cleaners + arr cleanup
|
||||
│ └── transcode_management.sh # Transcode cleanup + manager
|
||||
│
|
||||
├── Rsync/
|
||||
│ └── rsync.sh # Core rsync script — called per share
|
||||
├── Orchestrators/
|
||||
│ └── daily_sync.sh # Daily media sync orchestrator
|
||||
│
|
||||
├── Failover/
|
||||
│ ├── failover.sh # Mutual container failover — continuous loop
|
||||
│ ├── failover_test.sh # Controlled failover simulation
|
||||
│ └── failover_state_reset.sh # Reset failover state manually
|
||||
│
|
||||
├── Docker_Essentials/
|
||||
│ ├── docker_watchdog.sh # Two-tier container monitor — continuous loop
|
||||
│ ├── docker_daily_restart.sh # Daily container restarts
|
||||
│ ├── docker_weekly_restart.sh # Weekly container restarts
|
||||
│ └── docker_network_connect.sh # Connect containers to extra networks
|
||||
│
|
||||
├── unRAID_Essentials/
|
||||
│ ├── system_watchdog.sh # System health monitor — continuous loop
|
||||
│ ├── ramdisk_setup.sh # Creates ramdisk + symlink at array start
|
||||
│ ├── docker_syslog_filter.sh # Suppress veth log noise
|
||||
│ ├── php_fpm_max_children.sh # WebGUI performance tuning
|
||||
│ ├── server_reboot.sh # Graceful scheduled reboot
|
||||
│ ├── mover_stop.sh # Stop mover cleanly
|
||||
│ ├── clear_logs.sh # Weekly log cleanup
|
||||
│ ├── webgui_restart.sh # nginx + emhttp restart escalation
|
||||
│ └── git_pull_execute.sh # Pull latest scripts from Gitea
|
||||
│
|
||||
├── Media/
|
||||
│ ├── media_shares_permissions.sh # Apply permissions to media shares
|
||||
│ ├── media_cleaner.sh # Remove junk files from media shares
|
||||
│ ├── lidarr_cleanup.sh # Remove orphaned music files
|
||||
│ ├── sonarr_cleanup.sh # Remove orphaned TV files
|
||||
│ ├── radarr_cleanup.sh # Remove orphaned movie files
|
||||
│ └── arrs_failed_stalled_recovery.sh # Blocklist + re-search failed imports
|
||||
│
|
||||
├── Transcodes/
|
||||
│ ├── transcode_manager.sh # Symlink direction management
|
||||
│ ├── transcode_cleanup.sh # Remove old inactive transcode files
|
||||
│ └── ramdisk_setup.sh # (also in unRAID_Essentials — symlinked)
|
||||
│
|
||||
├── Monitors/
|
||||
│ ├── cert_monitor.sh # SSL certificate expiry monitoring
|
||||
│ ├── backup_verify.sh # Checksum verification against remote
|
||||
│ ├── smart_health.sh # Drive SMART attribute monitoring
|
||||
│ ├── zfs_memory_snapshot.sh # Weekly ZFS health + memory report
|
||||
│ ├── bandwidth_monitor.sh # Rsync transfer logging + weekly summary
|
||||
│ ├── weekly_health_digest.sh # Aggregated health digest email
|
||||
│ ├── emby_session_report.sh # Weekly Emby usage statistics
|
||||
│ └── emby_database_repair.sh # Emby SQLite database repair
|
||||
│
|
||||
└── Tools/
|
||||
└── recreate_shares.sh # Share directory recreation from cfg files
|
||||
├── recreate_shares.sh # Recreate share directories from cfg files
|
||||
├── bulk_permissions_repair.sh # One-shot permission repair
|
||||
├── watchdog_skip_list_manager.sh # Manage docker watchdog skip list
|
||||
├── rsync_stop.sh # Stop active rsync jobs cleanly
|
||||
├── user_scripts_stop.sh # Stop running user scripts
|
||||
└── container_data_export.sh # Export container configuration
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
*Guide version aligned with common.sh v1.6*
|
||||
@@ -3,6 +3,7 @@
|
||||
# --------------------------------- System Watchdog --------------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Last line of defense — reboots the system cleanly if it is about to become unstable.
|
||||
# Runs continuously as a background process — started by array_start.sh at array start.
|
||||
# Works alongside docker_watchdog.sh which handles container-level healing first.
|
||||
#
|
||||
# Checks (all toggleable in Master.conf):
|
||||
@@ -16,6 +17,432 @@
|
||||
# Docker daemon — unresponsive daemon means containers cannot be managed
|
||||
# Required containers — stopped containers that should be running (after watchdog skip list)
|
||||
#
|
||||
# Abort conditions (toggleable):
|
||||
# ZFS pool unhealthy — reboot with bad pool risks data loss
|
||||
# Parity running — aborting parity is better than crashing mid-check
|
||||
# Mover running — aborting move is better than crashing mid-move
|
||||
#
|
||||
# Reboot loop protection:
|
||||
# Tracks reboot timestamps in persistent log on /boot/
|
||||
# Rolling window — old entries purge automatically
|
||||
# If reboot count hits limit in window → shutdown instead of reboot
|
||||
#
|
||||
# Continuous loop:
|
||||
# Checks run every SYSTEM_WATCHDOG_INTERVAL seconds (default 300 = 5min)
|
||||
# Master.conf re-sourced each cycle — config changes picked up without restart
|
||||
# Silent when healthy — only verbose when trigger or reboot
|
||||
# Clean shutdown on SIGTERM/SIGINT — sent by array stop
|
||||
#
|
||||
# All configuration in Master.conf under System Watchdog section.
|
||||
# Supports --dry-run to show triggered conditions without rebooting.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup — runs once at start ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Running as root"
|
||||
|
||||
acquire_lock
|
||||
|
||||
TOTAL_CORES=$(nproc)
|
||||
|
||||
# Ensure state and persistent files exist
|
||||
touch "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null || {
|
||||
error "Cannot create state file: $SYS_WATCHDOG_STATE_FILE"
|
||||
exit 1
|
||||
}
|
||||
|
||||
touch "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null || {
|
||||
error "Cannot create reboot log: $SYS_WATCHDOG_REBOOT_LOG"
|
||||
exit 1
|
||||
}
|
||||
|
||||
touch "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null || {
|
||||
error "Cannot create failed container list: $SYS_WATCHDOG_FAILED_FILE"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Status ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_HEALTH rootfs threshold: ${SYS_WATCHDOG_ROOTFS_PCT}% toggle: $SYS_WATCHDOG_CHECK_ROOTFS"
|
||||
echo "$ICON_HEALTH log threshold: ${SYS_WATCHDOG_LOG_PCT}% toggle: $SYS_WATCHDOG_CHECK_LOG"
|
||||
echo "$ICON_MEM RAM threshold: ${SYS_WATCHDOG_MEM_GB}GB free toggle: $SYS_WATCHDOG_CHECK_RAM"
|
||||
echo "$ICON_ZFS ARC pinned: ${SYS_WATCHDOG_ARC_PINNED_PCT}% toggle: $SYS_WATCHDOG_CHECK_ARC"
|
||||
echo "$ICON_ZFS ARC release: ${SYS_WATCHDOG_ARC_RELEASE_PCT}%"
|
||||
echo "$ICON_GEAR CPU temp max: ${SYS_WATCHDOG_CPU_TEMP_MAX}°C toggle: $SYS_WATCHDOG_CHECK_CPU_TEMP"
|
||||
echo "$ICON_GEAR Load multiplier: ${SYS_WATCHDOG_LOAD_MULTIPLIER}x cores toggle: $SYS_WATCHDOG_CHECK_LOAD"
|
||||
echo "$ICON_GEAR Zombie limit: ${SYS_WATCHDOG_ZOMBIE_LIMIT} toggle: $SYS_WATCHDOG_CHECK_ZOMBIES"
|
||||
echo "$ICON_CONTAINERS Docker daemon: toggle: $SYS_WATCHDOG_CHECK_DOCKER_DAEMON"
|
||||
echo "$ICON_CONTAINERS Containers: toggle: $SYS_WATCHDOG_CHECK_CONTAINERS"
|
||||
echo "$ICON_SHIELD Strike limit: $SYS_WATCHDOG_STRIKE_LIMIT"
|
||||
echo "$ICON_TIME Interval: ${SYSTEM_WATCHDOG_INTERVAL}s"
|
||||
echo "$ICON_REBOOT_SMART Reboot limit: $SYS_WATCHDOG_REBOOT_LIMIT in ${SYS_WATCHDOG_REBOOT_WINDOW_HRS}hrs"
|
||||
echo "$ICON_ZFS Abort ZFS unhealthy: $SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY"
|
||||
echo "$ICON_GEAR Abort on parity: $SYS_WATCHDOG_ABORT_ON_PARITY"
|
||||
echo "$ICON_MOVER Abort on mover: $SYS_WATCHDOG_ABORT_ON_MOVER"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no reboot will be executed"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# STATE HELPERS — defined once, used every cycle
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
get_strikes() {
|
||||
local key="$1"
|
||||
grep -E "^${key}:" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | cut -d':' -f2
|
||||
}
|
||||
|
||||
set_strikes() {
|
||||
local key="$1" count="$2"
|
||||
grep -vE "^${key}:" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null > "${SYS_WATCHDOG_STATE_FILE}.tmp"
|
||||
echo "${key}:${count}" >> "${SYS_WATCHDOG_STATE_FILE}.tmp"
|
||||
mv "${SYS_WATCHDOG_STATE_FILE}.tmp" "$SYS_WATCHDOG_STATE_FILE"
|
||||
}
|
||||
|
||||
increment_strikes() {
|
||||
local key="$1"
|
||||
local current
|
||||
current=$(get_strikes "$key")
|
||||
[[ -z "$current" ]] && current=0
|
||||
((current++))
|
||||
set_strikes "$key" "$current"
|
||||
echo "$current"
|
||||
}
|
||||
|
||||
reset_strikes() {
|
||||
local key="$1"
|
||||
set_strikes "$key" 0
|
||||
}
|
||||
|
||||
purge_old_reboots() {
|
||||
local now
|
||||
now=$(date +%s)
|
||||
local cutoff=$(( now - SYS_WATCHDOG_REBOOT_WINDOW ))
|
||||
grep -v "^$" "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null | while IFS= read -r ts; do
|
||||
[[ "$ts" -gt "$cutoff" ]] && echo "$ts"
|
||||
done > "${SYS_WATCHDOG_REBOOT_LOG}.tmp"
|
||||
mv "${SYS_WATCHDOG_REBOOT_LOG}.tmp" "$SYS_WATCHDOG_REBOOT_LOG"
|
||||
}
|
||||
|
||||
count_recent_reboots() {
|
||||
purge_old_reboots
|
||||
grep -c "." "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null || echo 0
|
||||
}
|
||||
|
||||
log_reboot() {
|
||||
date +%s >> "$SYS_WATCHDOG_REBOOT_LOG"
|
||||
}
|
||||
|
||||
check_abort_conditions() {
|
||||
local should_abort=false
|
||||
|
||||
if command -v zpool >/dev/null 2>&1; then
|
||||
local unhealthy
|
||||
unhealthy=$(zpool list -H -o health 2>/dev/null | grep -v ONLINE || true)
|
||||
if [[ -n "$unhealthy" ]]; then
|
||||
if [[ "$SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY" == true ]]; then
|
||||
error "$ICON_ZFS ZFS pool unhealthy — aborting reboot"
|
||||
notify "System watchdog aborted reboot on $(hostname) — ZFS pool unhealthy" "System Watchdog" "warning"
|
||||
should_abort=true
|
||||
else
|
||||
warn "$ICON_ZFS ZFS pool unhealthy — continuing reboot"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ -f /var/local/emhttp/parity-date.txt ]]; then
|
||||
if grep -q "progress" /var/local/emhttp/parity-date.txt 2>/dev/null; then
|
||||
if [[ "$SYS_WATCHDOG_ABORT_ON_PARITY" == true ]]; then
|
||||
error "$ICON_DISK Parity check running — aborting reboot"
|
||||
notify "System watchdog aborted reboot on $(hostname) — parity running" "System Watchdog" "warning"
|
||||
should_abort=true
|
||||
else
|
||||
warn "$ICON_DISK Parity check running — continuing reboot"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
if pgrep -f "mover" >/dev/null 2>&1; then
|
||||
if [[ "$SYS_WATCHDOG_ABORT_ON_MOVER" == true ]]; then
|
||||
error "$ICON_MOVER Mover running — aborting reboot"
|
||||
notify "System watchdog aborted reboot on $(hostname) — mover running" "System Watchdog" "warning"
|
||||
should_abort=true
|
||||
else
|
||||
warn "$ICON_MOVER Mover running — continuing reboot"
|
||||
fi
|
||||
fi
|
||||
|
||||
[[ "$should_abort" == true ]] && return 1
|
||||
return 0
|
||||
}
|
||||
|
||||
run_strike_check() {
|
||||
local key="$1" triggered="$2" description="$3"
|
||||
if [[ "$triggered" == true ]]; then
|
||||
local strikes
|
||||
strikes=$(increment_strikes "$key")
|
||||
warn "$description — strike $strikes/$SYS_WATCHDOG_STRIKE_LIMIT"
|
||||
if (( strikes >= SYS_WATCHDOG_STRIKE_LIMIT )); then
|
||||
error "$description hit strike limit — reboot triggered"
|
||||
reset_strikes "$key"
|
||||
return 0
|
||||
fi
|
||||
else
|
||||
local current
|
||||
current=$(get_strikes "$key")
|
||||
if [[ -n "$current" && "$current" -gt 0 ]]; then
|
||||
reset_strikes "$key"
|
||||
fi
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
do_reboot() {
|
||||
local triggers=("$@")
|
||||
|
||||
if ! check_abort_conditions; then
|
||||
return
|
||||
fi
|
||||
|
||||
RECENT_REBOOTS=$(count_recent_reboots)
|
||||
info "Recent reboots in ${SYS_WATCHDOG_REBOOT_WINDOW_HRS}hr window: $RECENT_REBOOTS / $SYS_WATCHDOG_REBOOT_LIMIT"
|
||||
|
||||
if [[ "$RECENT_REBOOTS" -ge "$SYS_WATCHDOG_REBOOT_LIMIT" ]]; then
|
||||
error "Reboot limit hit — shutting down instead"
|
||||
notify "Reboot loop detected on $(hostname) — shutting down after $RECENT_REBOOTS reboots — conditions: ${triggers[*]}" "System Watchdog" "warning"
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would shutdown now"
|
||||
return
|
||||
fi
|
||||
sync
|
||||
/sbin/poweroff
|
||||
return
|
||||
fi
|
||||
|
||||
notify "System watchdog reboot triggered on $(hostname) — conditions: ${triggers[*]}" "System Watchdog" "warning"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — reboot sequence would begin now"
|
||||
return
|
||||
fi
|
||||
|
||||
log_reboot
|
||||
|
||||
info "Shutting down VMs..."
|
||||
if command -v virsh >/dev/null 2>&1; then
|
||||
for VM in $(virsh list --name 2>/dev/null); do
|
||||
[[ -z "$VM" ]] && continue
|
||||
virsh shutdown "$VM" >/dev/null 2>&1
|
||||
done
|
||||
sleep 30
|
||||
fi
|
||||
|
||||
info "Stopping Docker containers..."
|
||||
if command -v docker >/dev/null 2>&1; then
|
||||
docker ps -q | xargs -r docker stop >/dev/null 2>&1
|
||||
fi
|
||||
|
||||
info "Stopping User Scripts..."
|
||||
pkill -f "/tmp/user.scripts" 2>/dev/null || true
|
||||
|
||||
info "Syncing disks..."
|
||||
sync
|
||||
|
||||
echo ""
|
||||
echo "$ICON_REBOOT_SMART Rebooting system NOW..."
|
||||
sleep 5
|
||||
/sbin/reboot
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# CLEAN SHUTDOWN — trap SIGTERM/SIGINT from array stop
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
WATCHDOG_RUNNING=true
|
||||
|
||||
cleanup() {
|
||||
echo ""
|
||||
info "System watchdog received shutdown signal — stopping cleanly"
|
||||
WATCHDOG_RUNNING=false
|
||||
exit 0
|
||||
}
|
||||
|
||||
trap cleanup SIGTERM SIGINT
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ CONTINUOUS MONITORING LOOP ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
info "System watchdog started — checking every ${SYSTEM_WATCHDOG_INTERVAL}s"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
CYCLE=0
|
||||
|
||||
while [[ "$WATCHDOG_RUNNING" == true ]]; do
|
||||
((CYCLE++))
|
||||
|
||||
# Re-source Master.conf each cycle — picks up config changes without restart
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
SYS_WATCHDOG_REBOOT_WINDOW=$(( SYS_WATCHDOG_REBOOT_WINDOW_HRS * 3600 ))
|
||||
TOTAL_CORES=$(nproc)
|
||||
|
||||
# Per-cycle triggers — cleared each iteration
|
||||
TRIGGERS=()
|
||||
|
||||
# ── rootfs usage ─────────────────────────────────────────────────────────────────────────
|
||||
if [[ "$SYS_WATCHDOG_CHECK_ROOTFS" == true ]]; then
|
||||
ROOTFS_USED=$(df / --output=pcent | tail -1 | tr -d ' %')
|
||||
TRIGGERED=false
|
||||
[[ "$ROOTFS_USED" -ge "$SYS_WATCHDOG_ROOTFS_PCT" ]] && TRIGGERED=true
|
||||
run_strike_check "rootfs" "$TRIGGERED" "rootfs ${ROOTFS_USED}%" && \
|
||||
TRIGGERS+=("rootfs=${ROOTFS_USED}%")
|
||||
fi
|
||||
|
||||
# ── /var/log usage ────────────────────────────────────────────────────────────────────────
|
||||
if [[ "$SYS_WATCHDOG_CHECK_LOG" == true ]]; then
|
||||
LOG_USED=$(df -P /var/log | awk 'NR==2 {print $5}' | tr -d '%')
|
||||
TRIGGERED=false
|
||||
[[ "$LOG_USED" -ge "$SYS_WATCHDOG_LOG_PCT" ]] && TRIGGERED=true
|
||||
run_strike_check "log" "$TRIGGERED" "/var/log ${LOG_USED}%" && \
|
||||
TRIGGERS+=("log=${LOG_USED}%")
|
||||
fi
|
||||
|
||||
# ── Free RAM ─────────────────────────────────────────────────────────────────────────────
|
||||
if [[ "$SYS_WATCHDOG_CHECK_RAM" == true ]]; then
|
||||
MEM_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo)
|
||||
MEM_GB=$((MEM_KB / 1024 / 1024))
|
||||
TRIGGERED=false
|
||||
[[ "$MEM_GB" -lt "$SYS_WATCHDOG_MEM_GB" ]] && TRIGGERED=true
|
||||
run_strike_check "ram" "$TRIGGERED" "RAM ${MEM_GB}GB free" && \
|
||||
TRIGGERS+=("low_ram=${MEM_GB}GB")
|
||||
fi
|
||||
|
||||
# ── ZFS ARC ──────────────────────────────────────────────────────────────────────────────
|
||||
if [[ "$SYS_WATCHDOG_CHECK_ARC" == true ]] && [[ -f /proc/spl/kstat/zfs/arcstats ]]; then
|
||||
ARC_SIZE=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats)
|
||||
ARC_MAX=$(awk '/^c_max / {print $3}' /proc/spl/kstat/zfs/arcstats)
|
||||
ARC_PCT=$(( ARC_SIZE * 100 / ARC_MAX ))
|
||||
TRIGGERED=false
|
||||
if [[ "$ARC_PCT" -ge "$SYS_WATCHDOG_ARC_PINNED_PCT" ]]; then
|
||||
sync; echo 3 > /proc/sys/vm/drop_caches; sleep 5
|
||||
ARC_AFTER=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats)
|
||||
ARC_AFTER_PCT=$(( ARC_AFTER * 100 / ARC_MAX ))
|
||||
[[ "$ARC_AFTER_PCT" -ge "$SYS_WATCHDOG_ARC_RELEASE_PCT" ]] && TRIGGERED=true
|
||||
fi
|
||||
run_strike_check "arc" "$TRIGGERED" "ZFS ARC pinned" && \
|
||||
TRIGGERS+=("arc_pinned=${ARC_PCT}%")
|
||||
fi
|
||||
|
||||
# ── CPU temperature ──────────────────────────────────────────────────────────────────────
|
||||
if [[ "$SYS_WATCHDOG_CHECK_CPU_TEMP" == true ]]; then
|
||||
CPU_TEMP=""
|
||||
if command -v sensors >/dev/null 2>&1; then
|
||||
CPU_TEMP=$(sensors 2>/dev/null | grep -i "Package id 0\|Tctl\|CPU Temp" | \
|
||||
awk '{print $NF}' | tr -d '+°C' | head -1)
|
||||
fi
|
||||
if [[ -n "$CPU_TEMP" ]]; then
|
||||
CPU_TEMP_INT=$(printf "%.0f" "$CPU_TEMP")
|
||||
TRIGGERED=false
|
||||
[[ "$CPU_TEMP_INT" -ge "$SYS_WATCHDOG_CPU_TEMP_MAX" ]] && TRIGGERED=true
|
||||
run_strike_check "cpu_temp" "$TRIGGERED" "CPU temp ${CPU_TEMP_INT}°C" && \
|
||||
TRIGGERS+=("cpu_temp=${CPU_TEMP_INT}C")
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Load average ─────────────────────────────────────────────────────────────────────────
|
||||
if [[ "$SYS_WATCHDOG_CHECK_LOAD" == true ]]; then
|
||||
LOAD=$(awk '{print $1}' /proc/loadavg)
|
||||
LOAD_INT=$(printf "%.0f" "$LOAD")
|
||||
LOAD_THRESHOLD=$(( TOTAL_CORES * SYS_WATCHDOG_LOAD_MULTIPLIER ))
|
||||
TRIGGERED=false
|
||||
[[ "$LOAD_INT" -ge "$LOAD_THRESHOLD" ]] && TRIGGERED=true
|
||||
run_strike_check "load" "$TRIGGERED" "load avg ${LOAD}" && \
|
||||
TRIGGERS+=("load=${LOAD}")
|
||||
fi
|
||||
|
||||
# ── Zombie processes ─────────────────────────────────────────────────────────────────────
|
||||
if [[ "$SYS_WATCHDOG_CHECK_ZOMBIES" == true ]]; then
|
||||
ZOMBIE_COUNT=$(ps aux | awk '{print $8}' | grep -c "^Z$" || echo 0)
|
||||
TRIGGERED=false
|
||||
[[ "$ZOMBIE_COUNT" -ge "$SYS_WATCHDOG_ZOMBIE_LIMIT" ]] && TRIGGERED=true
|
||||
run_strike_check "zombies" "$TRIGGERED" "zombies ${ZOMBIE_COUNT}" && \
|
||||
TRIGGERS+=("zombies=${ZOMBIE_COUNT}")
|
||||
fi
|
||||
|
||||
# ── Docker daemon ────────────────────────────────────────────────────────────────────────
|
||||
if [[ "$SYS_WATCHDOG_CHECK_DOCKER_DAEMON" == true ]]; then
|
||||
TRIGGERED=false
|
||||
! timeout 10 docker ps >/dev/null 2>&1 && TRIGGERED=true
|
||||
run_strike_check "docker_daemon" "$TRIGGERED" "Docker daemon unresponsive" && \
|
||||
TRIGGERS+=("docker_daemon")
|
||||
fi
|
||||
|
||||
# ── Required containers from skip list ───────────────────────────────────────────────────
|
||||
if [[ "$SYS_WATCHDOG_CHECK_CONTAINERS" == true ]] && [[ -s "$SYS_WATCHDOG_FAILED_FILE" ]]; then
|
||||
FAILED_CONTAINERS=()
|
||||
while IFS= read -r container; do
|
||||
[[ -z "$container" ]] && continue
|
||||
STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null || echo "unknown")
|
||||
[[ "$STATUS" != "true" ]] && FAILED_CONTAINERS+=("$container")
|
||||
done < "$SYS_WATCHDOG_FAILED_FILE"
|
||||
|
||||
if [[ ${#FAILED_CONTAINERS[@]} -gt 0 ]]; then
|
||||
run_strike_check "failed_containers" "true" "required containers stopped" && \
|
||||
TRIGGERS+=("containers=${FAILED_CONTAINERS[*]}")
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Evaluate triggers ────────────────────────────────────────────────────────────────────
|
||||
if [[ ${#TRIGGERS[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_REBOOT_SMART System Watchdog — Cycle $CYCLE — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
for t in "${TRIGGERS[@]}"; do
|
||||
echo " $ICON_REBOOT_SMART $t"
|
||||
done
|
||||
echo ""
|
||||
do_reboot "${TRIGGERS[@]}"
|
||||
else
|
||||
log "Cycle $CYCLE — system healthy ($(date '+%H:%M:%S'))"
|
||||
fi
|
||||
|
||||
# Sleep until next cycle — interruptible by SIGTERM
|
||||
sleep "$SYSTEM_WATCHDOG_INTERVAL" &
|
||||
wait $!
|
||||
|
||||
done
|
||||
#
|
||||
# Checks (all toggleable in Master.conf):
|
||||
# rootfs usage — high rootfs fills rapidly when array is down, crash imminent
|
||||
# /var/log usage — log spam can fill rootfs, indicates something is broken
|
||||
# free RAM — critically low RAM means OOM or swap imminent
|
||||
# ZFS ARC pinned — ARC not releasing after reclaim means memory is stuck
|
||||
# CPU temperature — sustained tjmax causes throttling or kernel panic
|
||||
# load average — sustained high load means something is stuck or runaway
|
||||
# zombie processes — large zombie count indicates serious process management failure
|
||||
# Docker daemon — unresponsive daemon means containers cannot be managed
|
||||
# Required containers — stopped containers that should be running (after watchdog skip list)
|
||||
#
|
||||
# Abort conditions (toggleable — true = abort, false = reboot anyway):
|
||||
# ZFS pool unhealthy — reboot with bad pool risks data loss
|
||||
# Parity running — aborting parity is better than crashing mid-check
|
||||
|
||||
+34
-25
@@ -65,10 +65,13 @@
|
||||
# │ └── README-Monitors.md
|
||||
# │
|
||||
# ├── Orchestrators/
|
||||
# │ ├── media_shares_sync.sh # Runs all daily media share syncs sequentially
|
||||
# │ ├── critical_shares_maintenance.sh # Maintenance window — stop, update, sync, restart
|
||||
# │ ├── media_management.sh # Runs permissions + cleaners + arr cleanup
|
||||
# │ ├── transcode_management.sh # Runs cleanup then manager every 3min + daily stats
|
||||
# │ ├── array_start.sh # Single entry point — launches all array-start scripts
|
||||
# │ ├── daily_sync_maintenance.sh # Daily — git pull, media sync, media mgmt, docker restart
|
||||
# │ ├── weekly_sync_maintenance.sh # Weekly — critical sync + updates, docker weekly restart
|
||||
# │ ├── daily_sync_maintenance.sh # Media shares sync both directions
|
||||
# │ ├── weekly_sync_maintenance.sh # Clean sync + container updates (Emby + auth stack)
|
||||
# │ ├── media_management.sh # Permissions + cleaners + arr cleanup — run manually
|
||||
# │ ├── transcode_management.sh # Cleanup then manager every 3min + daily stats
|
||||
# │ └── README-Orchestrators.md
|
||||
# │
|
||||
# ├── Rsync/
|
||||
@@ -160,13 +163,16 @@
|
||||
#/mnt/user/appdata/unraid_scripts/Monitors/zfs_memory_snapshot.sh
|
||||
#
|
||||
# ━━━ Orchestrators ━━━
|
||||
# Orchestrators run their child scripts in the correct order.
|
||||
# Individual scripts below are still accessible for manual runs or testing.
|
||||
# array_start.sh is the single User Scripts entry for array startup.
|
||||
# All other orchestrators are scheduled via cron — not started at array start.
|
||||
#
|
||||
#/mnt/user/appdata/unraid_scripts/Orchestrators/media_shares_sync.sh
|
||||
#/mnt/user/appdata/unraid_scripts/Orchestrators/critical_shares_maintenance.sh
|
||||
#/mnt/user/appdata/unraid_scripts/Orchestrators/media_management.sh
|
||||
#/mnt/user/appdata/unraid_scripts/Orchestrators/array_start.sh
|
||||
# ^^ set to: At Startup of Array
|
||||
#/mnt/user/appdata/unraid_scripts/Orchestrators/daily_sync_maintenance.sh
|
||||
#/mnt/user/appdata/unraid_scripts/Orchestrators/weekly_sync_maintenance.sh
|
||||
#/mnt/user/appdata/unraid_scripts/Orchestrators/transcode_management.sh
|
||||
# ^^ media_management.sh is absorbed into daily_sync_maintenance.sh via MEDIA_MANAGEMENT_JOBS
|
||||
# ^^ run manually: bash Orchestrators/media_management.sh --dry-run
|
||||
#
|
||||
# ━━━ Rsync — Appdata Profiles ━━━
|
||||
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-Failover/Arrs_Stack
|
||||
@@ -175,7 +181,7 @@
|
||||
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Media_Server/Emby --profile=emby-failover
|
||||
# ^^ schedule every 30-60min — dirty sync, Emby running, critical data only
|
||||
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Media_Server/Emby
|
||||
# ^^ do NOT schedule — called by critical_shares_maintenance.sh Sunday 2:30am only
|
||||
# ^^ do NOT schedule — called by weekly_sync_maintenance.sh Sunday 2:30am only
|
||||
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-Failover/Gmer4Lfe
|
||||
#
|
||||
# ━━━ Rsync — Individual Media Shares (ad hoc) ━━━
|
||||
@@ -202,8 +208,8 @@
|
||||
#/mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_network_connect.sh
|
||||
#
|
||||
# ━━━ Media ━━━
|
||||
# media_management.sh runs all of these in order — individual entries for manual use.
|
||||
# Always --dry-run --log first on arr cleanup scripts before running live.
|
||||
# media_management.sh is absorbed into daily_sync_maintenance.sh via MEDIA_MANAGEMENT_JOBS.
|
||||
# Scripts below available for manual runs only — always --dry-run first on arr cleanup scripts.
|
||||
#
|
||||
#/mnt/user/appdata/unraid_scripts/Media/media_shares_permissions.sh
|
||||
#/mnt/user/appdata/unraid_scripts/Media/media_cleaner.sh anime --dry-run
|
||||
@@ -286,37 +292,40 @@
|
||||
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
#
|
||||
# ━━━ At Startup of Array ━━━
|
||||
# failover.sh — background task (continuous loop)
|
||||
# ramdisk_setup.sh — creates ramdisk before Emby starts
|
||||
# docker_network_connect.sh — connect containers to extra networks
|
||||
# docker_syslog_filter.sh — suppress veth noise before logs fill
|
||||
# php_fpm_max_children.sh — WebGUI performance tuning
|
||||
# array_start.sh — single entry point, launches everything below
|
||||
# configure what runs in ARRAY_START_SCRIPTS in Master.conf
|
||||
#
|
||||
# Launched by array_start.sh:
|
||||
# ramdisk_setup.sh — creates ramdisk before Emby starts (one-shot)
|
||||
# docker_syslog_filter.sh — suppress veth noise before logs fill (one-shot)
|
||||
# php_fpm_max_children.sh — WebGUI performance tuning (one-shot)
|
||||
# docker_network_connect.sh — connect containers to extra networks (one-shot)
|
||||
# system_watchdog.sh — system health monitor (continuous)
|
||||
# docker_watchdog.sh — container health monitor (continuous)
|
||||
# failover.sh — mutual failover (continuous)
|
||||
#
|
||||
# ━━━ Frequent (cron) ━━━
|
||||
# */3 * * * * transcode_management.sh (cleanup + manager every 3min)
|
||||
# */30 * * * * rsync.sh /mnt/user/Media_Server/Emby (emby-failover dirty sync)
|
||||
# --profile=emby-failover
|
||||
# */10 * * * * webgui_restart.sh
|
||||
# */15 * * * * docker_watchdog.sh
|
||||
# */6 * * * * arrs_failed_stalled_recovery.sh (blocklist + re-search)
|
||||
# */15 * * * * system_watchdog.sh
|
||||
#
|
||||
# ━━━ Daily ━━━
|
||||
# 0 1 * * * media_shares_sync.sh (media shares both directions)
|
||||
# 0 2 * * * media_management.sh (permissions + cleaners + arrs)
|
||||
# 0 3 * * * docker_daily_restart.sh
|
||||
# 0 1 * * * daily_sync_maintenance.sh (git pull + media sync + media_management + docker restart)
|
||||
# 0 5 * * * arrs_failed_stalled_recovery.sh (blocklist + re-search failed imports + stalled)
|
||||
# 0 8 * * * weekly_health_digest.sh (profile controls if it sends)
|
||||
#
|
||||
# ━━━ Rsync profiles — schedule individually ━━━
|
||||
# Arrs_Stack — daily or every few days (arr databases change on every download)
|
||||
# Critical-Data — handled by critical_shares_maintenance.sh — no separate schedule needed
|
||||
# Critical-Data — handled by weekly_sync_maintenance.sh — no separate schedule needed
|
||||
# Important-Data — daily (NextCloud file changes)
|
||||
# Gmer4Lfe — daily or weekly (personal appdata, rarely changes)
|
||||
# Emby — handled by critical_shares_maintenance.sh — no separate schedule needed
|
||||
# Emby — handled by weekly_sync_maintenance.sh — no separate schedule needed
|
||||
# emby-failover — every 30-60min via frequent cron above
|
||||
#
|
||||
# ━━━ Weekly — Sunday morning block ━━━
|
||||
# 30 2 * * 0 critical_shares_maintenance.sh (clean Emby + auth stack — cache resets weekly)
|
||||
# 30 2 * * 0 weekly_sync_maintenance.sh (clean Emby + auth stack — cache resets weekly)
|
||||
# 0 3 * * 0 docker_weekly_restart.sh
|
||||
# 0 5 * * 0 clear_logs.sh
|
||||
# 0 6 * * 0 zfs_memory_snapshot.sh
|
||||
|
||||
Reference in New Issue
Block a user