FailedProxy/Varaverk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

added failover script and other sytem scripts

Browse Source
This commit is contained in:
FailedProxy
2026-04-11 17:46:11 -04:00
parent ee107e5df7
commit 55930a1166
7 changed files with 1388 additions and 386 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Docker_essentials/docker_network_connect.sh
+159
View File
@@ -0,0 +1,159 @@
#!/bin/bash
# -----------------------------------------------------------------------------------------------
# --------------------------------- Docker Network Connect -------------------------------------
# -----------------------------------------------------------------------------------------------
# Connects specified Docker containers to extra networks on array start.
# Useful when containers need to communicate across networks they were not
# originally configured with — e.g. memcached needing access to nextcloud-aio network.
#
# Every container in NETWORK_CONNECT_CONTAINERS is connected to every network
# in NETWORK_CONNECT_NETWORKS. Already-connected containers are skipped cleanly.
#
# Run once at array start via User Scripts plugin.
# All configuration in Master.conf under Docker Network Connect section.
# Supports --dry-run to preview connections without making them.
# -----------------------------------------------------------------------------------------------
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../Master.conf"
source "$SCRIPT_DIR/../common.sh"
parse_args "$@"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Setup ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_GEAR Setup ━━━"
if [[ "$EUID" -ne 0 ]]; then
error "Must be run as root"
exit 1
fi
success "Running as root"
if ! command -v docker &>/dev/null; then
error "Docker not found"
exit 1
fi
success "Docker found"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Status ━━━
# -----------------------------------------------------------------------------------------------
if [[ "$SHOW_STATUS" == true ]]; then
echo ""
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
echo "$ICON_CONTAINERS Containers: ${NETWORK_CONNECT_CONTAINERS[*]}"
echo "$ICON_DOCKER_NET Networks: ${NETWORK_CONNECT_NETWORKS[*]}"
echo "$ICON_GEAR Dry Run: $DRY_RUN"
echo ""
echo "━━━ Current Connections ━━━"
for container in "${NETWORK_CONNECT_CONTAINERS[@]}"; do
[[ -z "$container" ]] && continue
echo "$ICON_CONTAINERS $container:"
docker inspect "$container" \
--format '{{range $k, $v := .NetworkSettings.Networks}} {{$k}}{{"\n"}}{{end}}' \
2>/dev/null || echo " not found"
done
echo "━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
fi
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no network connections will be made"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_DOCKER_NET Network Connect ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_DOCKER_NET Network Connect — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
echo "$ICON_CONTAINERS Containers: ${NETWORK_CONNECT_CONTAINERS[*]}"
echo "$ICON_DOCKER_NET Networks: ${NETWORK_CONNECT_NETWORKS[*]}"
echo ""
START=$(date +%s)
CONNECTED=()
SKIPPED=()
FAILED=()
for container in "${NETWORK_CONNECT_CONTAINERS[@]}"; do
[[ -z "$container" ]] && continue
echo "━━━ $ICON_CONTAINERS $container ━━━"
# Verify container exists
if ! docker inspect "$container" &>/dev/null; then
warn "$container not found — skipping"
FAILED+=("$container")
echo ""
continue
fi
for network in "${NETWORK_CONNECT_NETWORKS[@]}"; do
[[ -z "$network" ]] && continue
# Verify network exists
if ! docker network inspect "$network" &>/dev/null; then
warn "Network $network not found — skipping"
FAILED+=("$container:$network")
continue
fi
# Check if already connected
if docker network inspect "$network" \
--format '{{range .Containers}}{{.Name}} {{end}}' 2>/dev/null \
| grep -qw "$container"; then
info "$ICON_DOCKER_NET $container already connected to $network — skipping"
SKIPPED+=("$container→$network")
continue
fi
# Connect
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would connect $container to $network"
continue
fi
info "$ICON_DOCKER_NET Connecting $container to $network..."
if docker network connect "$network" "$container" 2>/dev/null; then
success "$ICON_DOCKER_NET $container connected to $network"
CONNECTED+=("$container→$network")
else
error "Failed to connect $container to $network"
FAILED+=("$container:$network")
fi
done
echo ""
done
END=$(date +%s)
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Summary ━━━
# -----------------------------------------------------------------------------------------------
echo "━━━━━ $ICON_SUMMARY NETWORK CONNECT SUMMARY ━━━━━"
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
[[ ${#CONNECTED[@]} -gt 0 ]] && echo "$ICON_DOCKER_NET Connected: ${CONNECTED[*]}"
[[ ${#SKIPPED[@]} -gt 0 ]] && echo "$ICON_RUNNING Skipped: ${SKIPPED[*]}"
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}"
echo ""
if [[ "$DRY_RUN" == true ]]; then
echo "$ICON_WARN Status: DRY RUN — no changes made"
elif [[ ${#FAILED[@]} -gt 0 ]]; then
echo "$ICON_ERROR Status: $ICON_ERROR SOME CONNECTIONS FAILED"
notify "Docker network connect failed on $(hostname) — ${FAILED[*]}" "Network Connect" "warning"
else
echo "$ICON_DONE Status: $ICON_SUCCESS DONE"
if [[ ${#CONNECTED[@]} -gt 0 ]]; then
notify "Docker networks connected on $(hostname) — ${CONNECTED[*]}" "Network Connect" "normal"
fi
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
[[ ${#FAILED[@]} -gt 0 ]] && exit 1
exit 0
Failover/failover.sh
+512
View File
@@ -0,0 +1,512 @@
#!/bin/bash
# -----------------------------------------------------------------------------------------------
# --------------------------------- Failover Script --------------------------------------------
# -----------------------------------------------------------------------------------------------
# Mutual container failover between two unRAID servers.
# Runs continuously on BOTH servers — each operates fully autonomously.
# No coordination between servers — decisions based solely on ping results.
#
# States:
# NORMAL — remote up, internet up — own containers only
# FAILOVER — remote down, internet up — own + remote's containers (additive)
# NO_INTERNET — internet down — stop public-facing containers
# DARK — remote down + internet down — same as NO_INTERNET
#
# Handback (remote returns after FAILOVER):
# Strike confirmation → pre-flight checks → rsync → start on remote → stop locally
#
# All configuration in Master.conf under Failover section.
# Run via User Scripts plugin as a background task — runs continuously until stopped.
# Supports --dry-run to walk through logic without taking any action.
# -----------------------------------------------------------------------------------------------
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../Master.conf"
source "$SCRIPT_DIR/../common.sh"
parse_args "$@"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Setup ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_GEAR Setup ━━━"
if [[ "$EUID" -ne 0 ]]; then
error "Must be run as root"
exit 1
fi
success "Running as root"
detect_hosts
resolve_remote_ip
# Select correct arrays based on which server we are
if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then
FAILOVER_START_CONTAINERS=("${FAILOVER_HOST1_STARTS_FOR_HOST2[@]}")
FAILOVER_STOP_ON_NO_NET=("${FAILOVER_HOST1_STOP_ON_NO_NET[@]}")
FAILOVER_RSYNC_JOBS=("${FAILOVER_HOST1_RSYNC_JOBS[@]}")
else
FAILOVER_START_CONTAINERS=("${FAILOVER_HOST2_STARTS_FOR_HOST1[@]}")
FAILOVER_STOP_ON_NO_NET=("${FAILOVER_HOST2_STOP_ON_NO_NET[@]}")
FAILOVER_RSYNC_JOBS=("${FAILOVER_HOST2_RSYNC_JOBS[@]}")
fi
info "$ICON_FAILOVER Failover containers: ${FAILOVER_START_CONTAINERS[*]}"
info "$ICON_STOP Stop on no-net: ${FAILOVER_STOP_ON_NO_NET[*]}"
info "$ICON_SYNC Rsync jobs: ${FAILOVER_RSYNC_JOBS[*]}"
# Ensure state file directory exists
touch "$FAILOVER_STATE_FILE" 2>/dev/null || {
error "Cannot create state file: $FAILOVER_STATE_FILE"
exit 1
}
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Status ━━━
# -----------------------------------------------------------------------------------------------
if [[ "$SHOW_STATUS" == true ]]; then
echo ""
echo "━━━━━ $ICON_SUMMARY FAILOVER STATUS ━━━━━"
echo "$ICON_HOST Local: $LOCAL_SERVER_NAME"
echo "$ICON_NET Remote: $REMOTE_SERVER_NAME ($REMOTE_SERVER)"
echo "$ICON_PING External IP: $EXTERNAL_IP"
echo "$ICON_TIME Check interval: ${FAILOVER_CHECK_INTERVAL}s"
echo "$ICON_RETRY Handback strikes: $FAILOVER_HANDBACK_STRIKES"
echo "$ICON_FAILOVER Failover containers: ${FAILOVER_START_CONTAINERS[*]}"
echo "$ICON_STOP Stop on no-net: ${FAILOVER_STOP_ON_NO_NET[*]}"
echo "$ICON_SYNC Rsync jobs: ${FAILOVER_RSYNC_JOBS[*]}"
echo "$ICON_GEAR State file: $FAILOVER_STATE_FILE"
echo "$ICON_GEAR Dry Run: $DRY_RUN"
if [[ -f "$FAILOVER_STATE_FILE" ]]; then
echo ""
echo "━━━ Persisted State ━━━"
cat "$FAILOVER_STATE_FILE"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
fi
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be started or stopped"
# -----------------------------------------------------------------------------------------------
# STATE FILE HELPERS
# State persists to /boot/ so it survives reboots.
# On restart the script re-evaluates from scratch using live ping results.
# State file is reference only — pings are always the source of truth.
# -----------------------------------------------------------------------------------------------
get_state() {
grep -E "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d'=' -f2
}
set_state() {
local new_state="$1"
local timestamp
timestamp=$(date '+%Y-%m-%d %H:%M:%S')
cat > "$FAILOVER_STATE_FILE" <<EOF
state=$new_state
last_change=$timestamp
host=$LOCAL_SERVER_NAME
EOF
log "State set to $new_state"
}
get_handback_strikes() {
grep -E "^handback_strikes=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d'=' -f2
}
set_handback_strikes() {
local count="$1"
if grep -q "^handback_strikes=" "$FAILOVER_STATE_FILE" 2>/dev/null; then
sed -i "s/^handback_strikes=.*/handback_strikes=$count/" "$FAILOVER_STATE_FILE"
else
echo "handback_strikes=$count" >> "$FAILOVER_STATE_FILE"
fi
}
# -----------------------------------------------------------------------------------------------
# CONTAINER HELPERS — LOCAL operations for failover
# These operate on the LOCAL server unlike stop_containers/start_containers in common.sh
# which operate on the REMOTE server via SSH.
# -----------------------------------------------------------------------------------------------
# Start a container locally — skip if already running
start_local_container() {
local container="$1"
local status
status=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null || echo "unknown")
if [[ "$status" == "true" ]]; then
log "$container already running locally — skipping"
return 0
fi
if [[ "$status" == "unknown" ]]; then
warn "$container not found on this host — skipping"
return 1
fi
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would start $container locally"
return 0
fi
echo "$ICON_START Starting $container locally..."
if docker start "$container" >/dev/null 2>&1; then
echo "$ICON_STARTED $container started"
return 0
else
error "Failed to start $container locally"
return 1
fi
}
# Stop a container locally — skip if already stopped
stop_local_container() {
local container="$1"
local status
status=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null || echo "unknown")
if [[ "$status" == "false" ]]; then
log "$container already stopped locally — skipping"
return 0
fi
if [[ "$status" == "unknown" ]]; then
log "$container not found on this host — skipping"
return 0
fi
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would stop $container locally"
return 0
fi
echo "$ICON_STOP Stopping $container locally..."
if docker stop "$container" >/dev/null 2>&1; then
echo "$ICON_STOPPED $container stopped"
return 0
else
error "Failed to stop $container locally"
return 1
fi
}
# Start a container on the remote server via SSH
start_remote_container() {
local container="$1"
local status
status=$(ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \
"docker inspect -f '{{.State.Running}}' $container 2>/dev/null || echo unknown" 2>/dev/null)
if [[ "$status" == "true" ]]; then
log "$container already running on remote — skipping"
return 0
fi
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would start $container on $REMOTE_SERVER_NAME"
return 0
fi
echo "$ICON_START Starting $container on $REMOTE_SERVER_NAME..."
if ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \
"docker start $container" >/dev/null 2>&1; then
echo "$ICON_STARTED $container started on $REMOTE_SERVER_NAME"
return 0
else
error "Failed to start $container on $REMOTE_SERVER_NAME"
return 1
fi
}
# -----------------------------------------------------------------------------------------------
# STATE ACTIONS
# Each state has a corresponding action function.
# These are idempotent — safe to call on every loop iteration.
# -----------------------------------------------------------------------------------------------
# NORMAL — remote is up, internet is up
# Ensure failover containers are stopped locally (cleanup after returning from FAILOVER)
action_normal() {
local prev_state="$1"
if [[ "$prev_state" == "FAILOVER" ]]; then
info "$ICON_FAILOVER Returning from FAILOVER — cleaning up local failover containers"
for c in "${FAILOVER_START_CONTAINERS[@]}"; do
[[ -z "$c" ]] && continue
stop_local_container "$c"
done
notify "Failover ended on $LOCAL_SERVER_NAME — $REMOTE_SERVER_NAME is back online" "Failover" "normal"
fi
# Ensure no-net containers are running (they may have been stopped)
if [[ "$prev_state" == "NO_INTERNET" || "$prev_state" == "DARK" ]]; then
info "$ICON_START Internet restored — starting previously stopped containers"
for c in "${FAILOVER_STOP_ON_NO_NET[@]}"; do
[[ -z "$c" ]] && continue
start_local_container "$c"
done
notify "Internet restored on $LOCAL_SERVER_NAME — normal containers restarted" "Failover" "normal"
fi
}
# FAILOVER — remote is down, internet is up
# Start remote's containers locally (additive — own containers keep running)
action_failover() {
local prev_state="$1"
if [[ "$prev_state" != "FAILOVER" ]]; then
# Just entered failover — notify and start containers
info "$ICON_FAILOVER Entering FAILOVER — $REMOTE_SERVER_NAME is unreachable"
# Check local array before starting containers
if ! check_local_array; then
error "Local array not ready — cannot start failover containers safely"
notify "Failover triggered but local array not ready on $LOCAL_SERVER_NAME" "Failover" "warning"
return 1
fi
# Check local docker daemon
if ! timeout 10 docker ps >/dev/null 2>&1; then
error "Local Docker daemon not responding — cannot start failover containers"
notify "Failover triggered but Docker not ready on $LOCAL_SERVER_NAME" "Failover" "warning"
return 1
fi
notify "Failover ACTIVE on $LOCAL_SERVER_NAME — starting ${REMOTE_SERVER_NAME} containers: ${FAILOVER_START_CONTAINERS[*]}" "Failover" "warning"
fi
# Start each failover container — idempotent, skips if already running
for c in "${FAILOVER_START_CONTAINERS[@]}"; do
[[ -z "$c" ]] && continue
start_local_container "$c"
done
}
# NO_INTERNET / DARK — this server has no internet
# Stop public-facing containers — no point serving if offline
action_no_internet() {
local prev_state="$1"
if [[ "$prev_state" != "NO_INTERNET" && "$prev_state" != "DARK" ]]; then
warn "$ICON_WARN Internet lost on $LOCAL_SERVER_NAME — stopping public containers"
notify "Internet lost on $LOCAL_SERVER_NAME — stopping public-facing containers" "Failover" "warning"
fi
for c in "${FAILOVER_STOP_ON_NO_NET[@]}"; do
[[ -z "$c" ]] && continue
stop_local_container "$c"
done
# If we were in FAILOVER, also stop the failover containers
if [[ "$prev_state" == "FAILOVER" ]]; then
info "Was in FAILOVER — stopping failover containers too"
for c in "${FAILOVER_START_CONTAINERS[@]}"; do
[[ -z "$c" ]] && continue
stop_local_container "$c"
done
fi
}
# HANDBACK — remote has returned, ready to hand containers back
# Confirmed by FAILOVER_HANDBACK_STRIKES consecutive remote-up checks
action_handback() {
info "$ICON_FAILOVER Initiating handback to $REMOTE_SERVER_NAME..."
notify "Handback starting on $LOCAL_SERVER_NAME — syncing data to $REMOTE_SERVER_NAME" "Failover" "normal"
# Pre-flight checks before rsync
echo ""
echo "━━━ $ICON_SHIELD Handback Pre-flight ━━━"
if ! check_remote_array; then
error "Remote array not ready — deferring handback"
notify "Handback deferred on $LOCAL_SERVER_NAME — remote array not ready" "Failover" "warning"
set_handback_strikes 0
return 1
fi
success "Remote array ready"
if ! check_remote_docker; then
error "Remote Docker not ready — deferring handback"
notify "Handback deferred on $LOCAL_SERVER_NAME — remote Docker not ready" "Failover" "warning"
set_handback_strikes 0
return 1
fi
success "Remote Docker ready"
REMOTE_USAGE=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
"df / --output=pcent | tail -1 | tr -d ' %'" 2>/dev/null)
if [[ -n "$REMOTE_USAGE" ]] && [[ "$REMOTE_USAGE" -ge "${ROOTFS_WARN:-75}" ]]; then
error "Remote rootfs ${REMOTE_USAGE}% full — deferring handback"
notify "Handback deferred — remote rootfs ${REMOTE_USAGE}% full on $REMOTE_SERVER_NAME" "Failover" "warning"
set_handback_strikes 0
return 1
fi
success "Remote rootfs healthy"
# Rsync data back via rsync.sh — uses existing profile system
echo ""
echo "━━━ $ICON_SYNC Handback Rsync ━━━"
local rsync_failed=false
for path in "${FAILOVER_RSYNC_JOBS[@]}"; do
[[ -z "$path" ]] && continue
info "Syncing $path → $REMOTE_SERVER_NAME"
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would rsync $path"
else
if bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$path" --no-log; then
success "Rsync complete: $path"
else
error "Rsync failed: $path"
rsync_failed=true
fi
fi
done
if [[ "$rsync_failed" == true ]]; then
error "One or more rsync jobs failed — deferring container handback"
notify "Handback rsync failed on $LOCAL_SERVER_NAME — containers not handed back yet" "Failover" "warning"
set_handback_strikes 0
return 1
fi
# Start containers on remote
echo ""
echo "━━━ $ICON_START $ICON_CONTAINERS Start on Remote ━━━"
local start_failed=false
for c in "${FAILOVER_START_CONTAINERS[@]}"; do
[[ -z "$c" ]] && continue
if ! start_remote_container "$c"; then
start_failed=true
fi
done
if [[ "$start_failed" == true ]]; then
error "One or more containers failed to start on remote — not stopping local copies"
notify "Handback partial failure on $LOCAL_SERVER_NAME — some containers failed to start on $REMOTE_SERVER_NAME" "Failover" "warning"
set_handback_strikes 0
return 1
fi
# Stop local failover containers — only after remote confirmed started
echo ""
echo "━━━ $ICON_STOP $ICON_CONTAINERS Stop Local Failover ━━━"
for c in "${FAILOVER_START_CONTAINERS[@]}"; do
[[ -z "$c" ]] && continue
stop_local_container "$c"
done
notify "Handback complete on $LOCAL_SERVER_NAME — $REMOTE_SERVER_NAME has containers back" "Failover" "normal"
set_handback_strikes 0
return 0
}
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_FAILOVER Main Loop ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_FAILOVER Failover Starting — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
echo "$ICON_HOST Local: $LOCAL_SERVER_NAME"
echo "$ICON_NET Remote: $REMOTE_SERVER_NAME ($REMOTE_SERVER)"
echo "$ICON_TIME Interval: ${FAILOVER_CHECK_INTERVAL}s"
echo ""
notify "Failover script started on $LOCAL_SERVER_NAME — monitoring $REMOTE_SERVER_NAME" "Failover" "normal"
LOOP_COUNT=0
while true; do
LOOP_COUNT=$((LOOP_COUNT + 1))
CURRENT_STATE=$(get_state)
[[ -z "$CURRENT_STATE" ]] && CURRENT_STATE="UNKNOWN"
echo ""
echo "━━━ $ICON_PING Check #${LOOP_COUNT} — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
log "Previous state: $CURRENT_STATE"
# ── Two ping checks — the only inputs to state decisions ──
REMOTE_UP=false
INTERNET_UP=false
ping_remote && REMOTE_UP=true
ping_internet && INTERNET_UP=true
log "Remote reachable: $REMOTE_UP | Internet reachable: $INTERNET_UP"
# ── Determine new state ──
if [[ "$REMOTE_UP" == true && "$INTERNET_UP" == true ]]; then
NEW_STATE="NORMAL"
elif [[ "$REMOTE_UP" == false && "$INTERNET_UP" == true ]]; then
NEW_STATE="FAILOVER"
elif [[ "$INTERNET_UP" == false ]]; then
NEW_STATE="NO_INTERNET"
else
NEW_STATE="DARK"
fi
# ── Handle handback strike system ──
# When returning from FAILOVER to NORMAL, require consecutive confirmations
if [[ "$CURRENT_STATE" == "FAILOVER" && "$NEW_STATE" == "NORMAL" ]]; then
STRIKES=$(get_handback_strikes)
[[ -z "$STRIKES" ]] && STRIKES=0
STRIKES=$((STRIKES + 1))
set_handback_strikes "$STRIKES"
if [[ "$STRIKES" -lt "$FAILOVER_HANDBACK_STRIKES" ]]; then
info "$ICON_RETRY Remote appears back — strike $STRIKES/$FAILOVER_HANDBACK_STRIKES — waiting for confirmation"
NEW_STATE="FAILOVER" # Stay in FAILOVER until strikes confirmed
else
info "$ICON_RETRY Remote confirmed stable ($STRIKES/$FAILOVER_HANDBACK_STRIKES) — initiating handback"
NEW_STATE="HANDBACK"
fi
else
# Reset handback strikes if we're not in the confirmation window
set_handback_strikes 0
fi
# ── Log state ──
if [[ "$NEW_STATE" != "$CURRENT_STATE" ]]; then
echo "$ICON_FAILOVER State: $CURRENT_STATE → $NEW_STATE"
else
info "State: $NEW_STATE (unchanged)"
fi
# ── Execute state action ──
case "$NEW_STATE" in
NORMAL)
action_normal "$CURRENT_STATE"
set_state "NORMAL"
;;
FAILOVER)
action_failover "$CURRENT_STATE"
set_state "FAILOVER"
;;
NO_INTERNET|DARK)
action_no_internet "$CURRENT_STATE"
set_state "$NEW_STATE"
;;
HANDBACK)
if action_handback; then
set_state "NORMAL"
action_normal "FAILOVER"
else
# Handback failed — stay in FAILOVER, retry next cycle
set_state "FAILOVER"
fi
;;
esac
echo "$ICON_TIME Next check in ${FAILOVER_CHECK_INTERVAL}s"
sleep "$FAILOVER_CHECK_INTERVAL"
done
Master.conf
+116 -122
View File
@@ -20,10 +20,14 @@
# DAILY SYNC SHARES Media shares synced by daily_sync.sh
# RSYNC PROFILE SYSTEM Per-profile overrides (appdata profiles)
#
# ── FAILOVER ───────────────────────────────────────────────────────────────────────────────
# FAILOVER Mutual container failover between two servers
#
# ── DOCKER ESSENTIALS ──────────────────────────────────────────────────────────────────────
# DOCKER DAILY RESTART Containers restarted daily
# DOCKER WEEKLY RESTART Containers restarted weekly
# DOCKER WATCHDOG Container health monitoring — memory, CPU, HTTP
# DOCKER NETWORK CONNECT Connect containers to extra networks on boot
#
# ── UNRAID ESSENTIALS ──────────────────────────────────────────────────────────────────────
# REBOOT User warning delay before scheduled reboot
@@ -31,6 +35,8 @@
# SYSLOG FILTER Docker veth noise filter file path
# PHP-FPM PHP-FPM max children config
# CLEAR LOGS System log file paths
# WEBGUI WATCHDOG WebGUI nginx + emhttp monitoring and restart
# ZFS MEMORY SNAPSHOT Weekly ZFS health and memory diagnostic report
#
# ── MEDIA ──────────────────────────────────────────────────────────────────────────────────
# MEDIA PERMISSIONS Share list, mode and owner for permissions script
@@ -45,22 +51,16 @@
# ==============================================================================================
# ━━━ Host Configuration ━━━
# Hostnames must match Tailscale machine names exactly — case sensitive
HOST1="unRAID-Gmer4Lfe"
HOST2="unRAID-Jayred365"
# SSH keys for server-to-server rsync — each server needs the other's key authorised
HOST1_SSH_KEY="/root/.ssh/Gmer4Lfe-rsync-key"
HOST2_SSH_KEY="/root/.ssh/Jayred365-rsync-key"
# ━━━ Logging ━━━
# true = verbose [LOG] output in scripts / false = user-facing output only
ENABLE_LOGGING=true
# ━━━ Notifications ━━━
# unRAID native — configure Settings → Notification Settings for errors/warnings only
NOTIFY_UNRAID=true
# Discord webhook URL — leave blank to disable
DISCORD_WEBHOOK=""
# ━━━ Git / Repo ━━━
@@ -74,25 +74,19 @@
# ==============================================================================================
# ━━━ Rsync Defaults ━━━
# Global fallback values — used when no profile match is found for a share.
# Shares in DAILY_SYNC_SHARES always use these globals (no profile defined).
BW_LIMIT=12500 # network speed limit KB/s
RETRY_COUNT=3 # number of retry attempts on failure
SLEEP=300 # seconds between retries
CRITICAL_CONTAINER_NAMES=() # containers to stop before rsync
DELAYED_CONTAINERS=() # containers needing delay before start
CONTAINER_DELAY=5 # seconds delay before starting delayed containers
EXCLUDE_DIRS=() # directories to exclude from transfer
BW_LIMIT=12500
RETRY_COUNT=3
SLEEP=300
CRITICAL_CONTAINER_NAMES=()
DELAYED_CONTAINERS=()
CONTAINER_DELAY=5
EXCLUDE_DIRS=()
DEFAULT_RSYNC_OPTS=(-av --info=progress2 --human-readable --bwlimit="$BW_LIMIT" --delete --inplace --no-whole-file)
# ━━━ Remote Health Checks ━━━
# Abort rsync if remote rootfs exceeds this percentage.
# Protects against rsync filling rootfs when remote array is down or drives are missing.
ROOTFS_WARN=75
# ━━━ Daily Sync Shares ━━━
# Shares synced once daily by Orchestrators/daily_sync.sh
# No profile needed — all fall through to DEFAULT_RSYNC_OPTS above.
DAILY_SYNC_SHARES=(
/mnt/user/Anime_Movies-Old
/mnt/user/Anime_Shows-Old
@@ -109,23 +103,6 @@ DAILY_SYNC_SHARES=(
)
# ━━━ Rsync Profile System ━━━
# Profiles are matched by directory basename (lowercased).
# Example: /mnt/user/appdata-Failover/Arrs_Stack → profile key = arrs_stack
#
# How fallthrough works:
# - If a key exists in a profile array that value is used
# - If a key is missing the global default above is used instead
# - Shares in DAILY_SYNC_SHARES have no profile and always use globals
#
# To add a new profile:
# 1. Add a key to each array below with your chosen profile name
# 2. Call rsync.sh with a directory whose basename matches that key
# 3. Any array you omit falls back to its global default
#
# Note: PROFILE_RSYNC_OPTS does NOT inherit from DEFAULT_RSYNC_OPTS —
# list all desired options explicitly if you define a profile entry
# SPACE-SEPARATED STRINGS
declare -A PROFILE_RSYNC_OPTS=(
[arrs_stack]="-av --info=progress2 --human-readable --bwlimit=$BW_LIMIT --delete --inplace"
[critical-data]="-av --human-readable --bwlimit=$BW_LIMIT --delete"
@@ -158,7 +135,6 @@ declare -A PROFILE_SLEEP=(
[emby]=300
)
# SPACE-SEPARATED STRINGS
declare -A PROFILE_CRITICAL_CONTAINER_NAMES=(
[arrs_stack]="Sonarr Lidarr Readarr Radarr Prowlarr Bazarr Pinchflat"
[critical-data]="Mariadb-Authelia Redis-Authelia Lldap-Gmer4Lfe NginxProxyManager Authelia"
@@ -167,7 +143,6 @@ declare -A PROFILE_CRITICAL_CONTAINER_NAMES=(
[emby]=""
)
# SPACE-SEPARATED STRINGS — containers needing delay before starting
declare -A PROFILE_DELAYED_CONTAINERS=(
[arrs_stack]=""
[critical-data]="Authelia"
@@ -184,7 +159,6 @@ declare -A PROFILE_CONTAINER_DELAY=(
[emby]=5
)
# SPACE-SEPARATED STRINGS
declare -A PROFILE_EXCLUDE_DIRS=(
[arrs_stack]="logs *.tmp"
[critical-data]="logs *.tmp"
@@ -193,12 +167,59 @@ declare -A PROFILE_EXCLUDE_DIRS=(
[emby]="logs *.tmp"
)
# ==============================================================================================
# ── FAILOVER ──────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Mutual container failover between two unRAID servers.
# Each server runs failover.sh independently — no coordination between servers.
# Decisions based solely on two ping checks: remote reachable + internet reachable.
# Comment out any container or rsync job to disable without removing the entry.
EXTERNAL_IP="8.8.8.8"
FAILOVER_CHECK_INTERVAL=120
FAILOVER_HANDBACK_STRIKES=2
FAILOVER_STATE_FILE="/boot/config/failover_state.db"
# Containers HOST1 starts locally when HOST2 goes down
FAILOVER_HOST1_STARTS_FOR_HOST2=(
"Vaultwarden-Jayred365"
"Nextcloud-Jayred365"
"Cloudflare-DDNS-Jayred365"
)
# Containers HOST1 stops when it loses internet
FAILOVER_HOST1_STOP_ON_NO_NET=(
"Emby"
"Cloudflare-DDNS-Gmer4Lfe"
)
# Rsync jobs HOST1 runs before handing containers back to HOST2
FAILOVER_HOST1_RSYNC_JOBS=(
"/mnt/user/appdata-Failover/Jayred365"
"/mnt/user/Media_Server/Emby-Jayred"
)
# Containers HOST2 starts locally when HOST1 goes down
FAILOVER_HOST2_STARTS_FOR_HOST1=(
"Emby"
"Cloudflare-DDNS-Gmer4Lfe"
)
# Containers HOST2 stops when it loses internet
FAILOVER_HOST2_STOP_ON_NO_NET=(
"Cloudflare-DDNS-Jayred365"
)
# Rsync jobs HOST2 runs before handing containers back to HOST1
FAILOVER_HOST2_RSYNC_JOBS=(
"/mnt/user/appdata-Failover/Gmer4Lfe"
)
# ==============================================================================================
# ── DOCKER ESSENTIALS ─────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# ━━━ Docker Daily Restart ━━━
# Containers restarted every day — case-sensitive names
DAILY_RESTART_CONTAINERS=(
"NginxProxyManager"
"Authelia"
@@ -209,7 +230,6 @@ DAILY_RESTART_CONTAINERS=(
)
# ━━━ Docker Weekly Restart ━━━
# Containers restarted once per week — case-sensitive names
WEEKLY_RESTART_CONTAINERS=(
"NginxProxyManager"
"Authelia"
@@ -220,11 +240,6 @@ WEEKLY_RESTART_CONTAINERS=(
)
# ━━━ Docker Watchdog ━━━
# First line of defense — monitors and restarts unhealthy containers.
# Runs on a cron schedule (recommended every 15 minutes).
# Strike system prevents restarts on brief spikes.
# Containers to monitor with memory hard limits in MB
# 20GB=20480 16GB=16384 14GB=14336 12GB=12288 10GB=10240
# 8GB=8192 6GB=6144 4GB=4096 2GB=2048 1GB=1024
declare -A WATCHDOG_CONTAINERS=(
@@ -235,44 +250,47 @@ declare -A WATCHDOG_CONTAINERS=(
["Code-Server"]=1024
)
# Containers to check HTTP responsiveness — omit to skip
declare -A WATCHDOG_CONTAINER_URLS=(
["Emby"]="http://localhost:8096"
["Jellyfin-Gmer4Lfe"]="http://localhost:8095"
)
# Containers that should always be running — monitored for unexpected stops
# Strike system used — persistent skip list prevents reboot loops
WATCHDOG_REQUIRED_CONTAINERS=(
"NginxProxyManager"
"Authelia"
"Emby"
)
# Strike state file — /tmp resets on reboot, correct for strike tracking
WATCHDOG_STATE_FILE="/tmp/container_watchdog_state.db"
SOFT_CPU_THRESHOLD=80
HARD_CPU_THRESHOLD=85
CPU_FAIL_LIMIT=2
SOFT_MEM_THRESHOLD=80
RESP_FAIL_LIMIT=2
CURL_TIMEOUT=5
# CPU thresholds — normalised against total core count at runtime
SOFT_CPU_THRESHOLD=80 # warn at this % of total system CPU
HARD_CPU_THRESHOLD=85 # strike at this % of total system CPU
CPU_FAIL_LIMIT=2 # consecutive strikes before restart
# ━━━ Docker Network Connect ━━━
# Containers connected to extra networks on array start — many-to-many
# Every container connects to every network listed
# Comment out entries to disable without removing them
NETWORK_CONNECT_CONTAINERS=(
"memcached"
"Npm-CrowdSec"
)
# Memory threshold
SOFT_MEM_THRESHOLD=80 # warn at this % of per-container hard limit
# Responsiveness check
RESP_FAIL_LIMIT=2 # consecutive failures before restart
CURL_TIMEOUT=5 # seconds before curl gives up
NETWORK_CONNECT_NETWORKS=(
"nextcloud-aio"
)
# ==============================================================================================
# ── UNRAID ESSENTIALS ─────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# ━━━ Reboot ━━━
REBOOT_SLEEP=300 # user warning delay before scheduled reboot (seconds)
REBOOT_SLEEP=300
# ━━━ Mover ━━━
MOVER_STOP_TIMEOUT=300 # timeout before stopping the mover (seconds)
MOVER_STOP_TIMEOUT=300
# ━━━ Syslog Filter ━━━
FILTER_FILE="/etc/rsyslog.d/ignore-docker-veth.conf"
@@ -284,6 +302,21 @@ WATCHDOG_REQUIRED_CONTAINERS=(
# ━━━ Clear Logs ━━━
LOG_FILES=(/var/log/syslog /var/log/messages /var/log/dmesg)
# ━━━ WebGUI Watchdog ━━━
# Monitors unRAID WebGUI — escalates from nginx restart to emhttp restart if needed
WEBGUI_URL="http://localhost" # adjust port if running non-standard e.g. http://localhost:8080
WEBGUI_TIMEOUT=5 # seconds before curl gives up
WEBGUI_NGINX_WAIT=15 # seconds to wait after nginx restart before recheck
WEBGUI_EMHTTP_WAIT=30 # seconds to wait after emhttp restart before recheck
# ━━━ ZFS Memory Snapshot ━━━
# Weekly ZFS health and memory diagnostic report
ZFS_REPORT_LOG="/var/log/zfs-weekly-health.log"
ZFS_REPORT_ARC_WARN_PCT=90 # warn if ARC utilization above this %
ZFS_REPORT_FREE_WARN_GB=10 # warn if free RAM below this GB
ZFS_REPORT_AVAIL_WARN_GB=20 # warn if available RAM below this GB
ZFS_REPORT_DOCKER_TOP=10 # number of top Docker memory users to show
# ==============================================================================================
# ── MEDIA ─────────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
@@ -318,9 +351,6 @@ MEDIA_PERMISSION_SHARES=(
)
# ━━━ Media Cleaner ━━━
# Two profiles: anime and media — passed as argument to media_cleaner.sh
# Usage: media_cleaner.sh anime or media_cleaner.sh media
ANIME_CLEAN_FOLDERS=(
/mnt/user/Anime_Movies
/mnt/user/Anime_Movies-Old
@@ -355,70 +385,38 @@ MEDIA_FILE_PATTERNS=(
# ==============================================================================================
# ── TRANSCODES ────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Session-based storage allocator using filesystem indirection.
# New transcode sessions land wherever TRANSCODE_LINK points.
# Existing sessions are never interrupted — ffmpeg resolves path once at session start.
#
# Flow:
# ramdisk_setup.sh — run once at array start, creates ramdisk and symlink
# transcode_manager.sh — runs every 2-3 min, monitors usage and flips symlink
# transcode_cleanup.sh — runs every 5 min, removes old inactive files
#
# Hysteresis gap between RAMDISK_WARN_GB and RAMDISK_LOW_GB prevents flip-flop
# when usage hovers near the threshold. Gap should be at least 0.5-1GB.
# ━━━ Transcode Manager ━━━
# Paths
RAMDISK_PATH="/mnt/ramdisk_transcodes" # tmpfs mount point
RAMDISK_SIZE="8G" # increase if you have RAM headroom
TRANSCODE_LINK="/mnt/ram-transcode" # symlink Emby points at
TRANSCODE_SSD="/mnt/cache/Temp_Storage/Emby/Transcodes/" # SSD fallback location
# Thresholds in GB
RAMDISK_WARN_GB=6.8 # flip symlink to SSD at or above this usage
RAMDISK_LOW_GB=5.5 # flip symlink back to ramdisk when usage drops here
RAMDISK_SSD_MIN_GB=20 # minimum free GB on SSD before allowing flip — abort if below
# Cleanup settings
TRANSCODE_MAX_AGE=20 # minutes before a file is eligible for cleanup
TRANSCODE_ORPHAN_AGE=30 # minutes before an orphaned file is eligible (slightly longer buffer)
# Flip frequency monitoring
TRANSCODE_FLIP_WARN=3 # notify if symlink flips this many times in one hour
# Permissions — should match your Emby container user
RAMDISK_PATH="/mnt/ramdisk_transcodes"
RAMDISK_SIZE="8G"
TRANSCODE_LINK="/mnt/ram-transcode"
TRANSCODE_SSD="/mnt/cache/Temp_Storage/Emby/Transcodes/"
RAMDISK_WARN_GB=6.8
RAMDISK_LOW_GB=5.5
RAMDISK_SSD_MIN_GB=20
TRANSCODE_MAX_AGE=20
TRANSCODE_ORPHAN_AGE=30
TRANSCODE_FLIP_WARN=3
TRANSCODE_OWNER="nobody:users"
TRANSCODE_MODE="755"
# ==============================================================================================
# ── SYSTEM WATCHDOG ───────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Last line of defense — reboots the system cleanly if it is about to become unstable.
# Works alongside docker_watchdog.sh — containers first, system second.
# Thresholds set at "about to fall over" levels — not just high usage.
# ━━━ System Watchdog State Files ━━━
SYS_WATCHDOG_STATE_FILE="/tmp/system_watchdog_state.db"
SYS_WATCHDOG_FAILED_FILE="/boot/config/system_watchdog_failed.db"
SYS_WATCHDOG_REBOOT_LOG="/boot/config/system_watchdog_reboots.db"
# ━━━ Strike and Reboot Loop Settings ━━━
SYS_WATCHDOG_STRIKE_LIMIT=2 # consecutive hits before reboot trigger
SYS_WATCHDOG_REBOOT_LIMIT=3 # max reboots in window before shutdown instead
SYS_WATCHDOG_REBOOT_WINDOW_HRS=12 # window in hours — controls reboot count AND log purge
# ━━━ Thresholds ━━━
SYS_WATCHDOG_ROOTFS_PCT=95 # rootfs % before strike
SYS_WATCHDOG_LOG_PCT=95 # /var/log % before strike
SYS_WATCHDOG_MEM_GB=4 # free RAM GB below which strikes (128GB system)
SYS_WATCHDOG_ARC_PINNED_PCT=98 # ZFS ARC % of max before reclaim attempt
SYS_WATCHDOG_ARC_RELEASE_PCT=95 # ZFS ARC % after reclaim that still triggers
SYS_WATCHDOG_LOAD_MULTIPLIER=3 # strike if load avg > cores x this value
SYS_WATCHDOG_ZOMBIE_LIMIT=50 # strike if zombie count exceeds this
SYS_WATCHDOG_CPU_TEMP_MAX=95 # degrees C — adjust for your CPU tjmax
# ━━━ Check Toggles ━━━
# true = run this check / false = skip entirely
SYS_WATCHDOG_STRIKE_LIMIT=2
SYS_WATCHDOG_REBOOT_LIMIT=3
SYS_WATCHDOG_REBOOT_WINDOW_HRS=12
SYS_WATCHDOG_ROOTFS_PCT=95
SYS_WATCHDOG_LOG_PCT=95
SYS_WATCHDOG_MEM_GB=4
SYS_WATCHDOG_ARC_PINNED_PCT=98
SYS_WATCHDOG_ARC_RELEASE_PCT=95
SYS_WATCHDOG_LOAD_MULTIPLIER=3
SYS_WATCHDOG_ZOMBIE_LIMIT=50
SYS_WATCHDOG_CPU_TEMP_MAX=95
SYS_WATCHDOG_CHECK_ROOTFS=true
SYS_WATCHDOG_CHECK_LOG=true
SYS_WATCHDOG_CHECK_RAM=true
@@ -428,10 +426,6 @@ MEDIA_FILE_PATTERNS=(
SYS_WATCHDOG_CHECK_ZOMBIES=true
SYS_WATCHDOG_CHECK_CONTAINERS=true
SYS_WATCHDOG_CHECK_DOCKER_DAEMON=true
# ━━━ Abort Toggles ━━━
# true = abort reboot if condition active / false = reboot anyway
# Default true = conservative — set false only when you want "reboot no matter what"
SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY=true
SYS_WATCHDOG_ABORT_ON_PARITY=true
SYS_WATCHDOG_ABORT_ON_MOVER=true
common.sh
+114 -129
View File
@@ -2,7 +2,7 @@
# -----------------------------------------------------------------------------------------------
# ----------------- UNRAID OPS COMMON LIBRARY (STABLE FRAMEWORK v1) ----------------------------
# -----------------------------------------------------------------------------------------------
# Version: 2.4
# Version: 2.7
# -----------------------------------------------------------------------------------------------
# Changelog:
# v1.0 — Initial stable framework
@@ -40,6 +40,17 @@
# v2.4 — ICON_CLEAN, ICON_TRASH added for media cleaner operations
# ICON_PERMS, ICON_UNLOCKED added for media permissions operations
# ICON_REBOOT_SMART added for smart conditional reboot
# v2.5 — ICON_RAM added for ramdisk operations
# ICON_LINK added for symlink state and management
# Transcode scripts group added to ecosystem
# v2.6 — ICON_FAILOVER added for failover operations
# check_local_array added — verifies local /mnt/user is mounted and healthy
# check_remote_array added — verifies remote /mnt/user is mounted and healthy
# check_remote_docker added — verifies remote Docker daemon is responding
# ping_remote added — non-fatal ping returning status for failover use
# ping_internet added — non-fatal external ping for failover use
# v2.7 — ICON_WEBGUI added for WebGUI watchdog operations
# ICON_DOCKER_NET added for Docker network connect operations
# -----------------------------------------------------------------------------------------------
# -----------------------------------------------------------------------------------------------
@@ -59,13 +70,13 @@ ICON_HEALTH="🩺" # rootfs / share health checks
ICON_SHIELD="🛡️" # pre-flight section header
# Containers
ICON_CONTAINERS="📦" # container section anchor — paired with action icon for direction
ICON_CONTAINERS="📦" # container section anchor
ICON_STOP="⛔" # stop command being issued
ICON_STOPPED="🔴" # container confirmed stopped
ICON_START="▶️" # start command being issued
ICON_STARTED="💚" # container confirmed started
ICON_RUNNING="🟢" # container already running when checked
ICON_NOT_RUNNING="⭕" # container already stopped when checked — distinct from ICON_STOPPED
ICON_NOT_RUNNING="⭕" # container already stopped when checked
# Transfer
ICON_SYNC="🔄" # transfer section header
@@ -83,6 +94,7 @@ ICON_REBOOT="⚡" # scheduled server reboot
ICON_REBOOT_SMART="🚨" # smart conditional reboot triggered
ICON_PLUGIN="🧩" # user scripts plugin operations
ICON_PHP="👥" # PHP-FPM operations
ICON_WEBGUI="💻" # WebGUI / nginx / emhttp operations
# Diagnostics
ICON_ZFS="📊" # ZFS ARC statistics
@@ -95,6 +107,16 @@ ICON_TRASH="🗑️" # files being deleted
ICON_PERMS="🔐" # permissions operation / section header
ICON_UNLOCKED="🔓" # permissions successfully applied to a share
# Transcode Operations
ICON_RAM="💨" # ramdisk operations — fast ephemeral storage
ICON_LINK="🔗" # symlink state and management
# Failover Operations
ICON_FAILOVER="🔀" # failover state changes and operations
# Docker Network Operations
ICON_DOCKER_NET="🔌" # Docker network connect operations
# Notifications
ICON_NOTIFY="🔔" # notification operations
@@ -106,8 +128,6 @@ ICON_SUCCESS="✅"
# -----------------------------------------------------------------------------------------------
# OUTPUT HELPERS
# Standardised output functions used across all scripts.
# log() is gated by ENABLE_LOGGING — set in Master.conf or via --log flag.
# -----------------------------------------------------------------------------------------------
info() { echo "$ICON_INFO [INFO] $*"; }
warn() { echo "$ICON_WARN [WARN] $*"; }
@@ -120,14 +140,8 @@ log() {
# -----------------------------------------------------------------------------------------------
# NOTIFICATION
# Sends a notification via unRAID native system and/or Discord webhook.
# Both channels are optional and independently controlled via Master.conf.
# unRAID native: requires NOTIFY_UNRAID=true and the dynamix notify script to be present.
# Discord: requires DISCORD_WEBHOOK to be set to a valid webhook URL.
# Severity levels: normal, warning, alert — maps to unRAID notification severity.
# Usage: notify "message" "subject" "severity"
# notify "Rsync failed: Movies" "Rsync Alert" "warning"
# notify "Daily sync complete" "Daily Sync" "normal"
# Severity: normal, warning, alert
# -----------------------------------------------------------------------------------------------
notify() {
local message="$1"
@@ -136,7 +150,6 @@ notify() {
log "$ICON_NOTIFY Sending notification: $subject — $message"
# unRAID native notification
if [[ "${NOTIFY_UNRAID:-false}" == true ]]; then
local notify_script="/usr/local/emhttp/plugins/dynamix/scripts/notify"
if [[ -x "$notify_script" ]]; then
@@ -147,7 +160,6 @@ notify() {
fi
fi
# Discord webhook notification
if [[ -n "${DISCORD_WEBHOOK:-}" ]]; then
local payload
payload=$(printf '{"content": "%s — **%s**\\n%s"}' \
@@ -163,9 +175,6 @@ notify() {
# -----------------------------------------------------------------------------------------------
# DURATION FORMATTER
# Converts raw seconds into a human readable string — e.g. 10m53s or 47s
# Used by rsync.sh summary and daily_sync.sh summary.
# Sourced from common.sh so both scripts share the same implementation.
# -----------------------------------------------------------------------------------------------
format_duration() {
local secs=$1
@@ -176,24 +185,17 @@ format_duration() {
# -----------------------------------------------------------------------------------------------
# ARG PARSER
# Processes all flags and key=value pairs passed to any script.
# Positional arguments (directory paths) are separated before calling this — see rsync.sh.
# Supported flags: --dry-run, --log, --no-log, --status, --help
# Supported key=value: LOG=true/false, or any declared variable e.g. BW_LIMIT=5000
# Unparsed positional args are returned in PARSED_ARGS array.
# -----------------------------------------------------------------------------------------------
parse_args() {
ENABLE_LOGGING=${ENABLE_LOGGING:-false}
DRY_RUN=${DRY_RUN:-false}
SHOW_STATUS=${SHOW_STATUS:-false}
CLEAN_ARGS=()
for ARG in "$@"; do
if [[ "$ARG" == *=* ]]; then
VAR="${ARG%%=*}"
VAL="${ARG#*=}"
case "$VAR" in
LOG)
[[ "$VAL" == "true" ]] && ENABLE_LOGGING=true
@@ -215,7 +217,7 @@ parse_args() {
--no-log) ENABLE_LOGGING=false ;;
--status|--summary) SHOW_STATUS=true ;;
--help|-h)
echo "Usage: script <dir> [--dry-run] [--log] [--status]"
echo "Usage: script [--dry-run] [--log] [--status]"
exit 0
;;
*) CLEAN_ARGS+=("$ARG") ;;
@@ -228,43 +230,26 @@ parse_args() {
# -----------------------------------------------------------------------------------------------
# VALIDATION
# Checks that a required variable is set and non-empty.
# Usage: require_var VAR_NAME
# Exits with error if the variable is missing.
# -----------------------------------------------------------------------------------------------
require_var() {
[[ -z "${!1:-}" ]] && error "Missing required: $1" && exit 1
}
# -----------------------------------------------------------------------------------------------
# INTEGER VALIDATION
# Checks that a variable contains a valid positive integer.
# Exits with a clear error if the value is missing, empty, or not a number.
# Usage: validate_int VAR_NAME "$VAR_VALUE"
# Example: validate_int MOVER_STOP_TIMEOUT "$MOVER_STOP_TIMEOUT"
# -----------------------------------------------------------------------------------------------
validate_int() {
local name="$1"
local value="$2"
local name="$1" value="$2"
if [[ -z "$value" ]]; then
error "$name is not set — check Master.conf"
exit 1
fi
if ! [[ "$value" =~ ^[0-9]+$ ]]; then
error "$name must be a positive integer — got: '$value'"
exit 1
fi
log "$name validated: $value"
}
# -----------------------------------------------------------------------------------------------
# HOST DETECTION
# Determines local and remote server names by comparing hostname against HOST1/HOST2.
# Sets LOCAL_SERVER_NAME, REMOTE_SERVER_NAME, and SSH_KEY for the current run direction.
# Both HOST1 and HOST2 must be defined in Master.conf.
# -----------------------------------------------------------------------------------------------
detect_hosts() {
LOCAL_HOSTNAME="$(hostname)"
@@ -283,34 +268,25 @@ detect_hosts() {
declare -A SSH_KEYS
SSH_KEYS["$HOST1|$HOST2"]="$HOST1_SSH_KEY"
SSH_KEYS["$HOST2|$HOST1"]="$HOST2_SSH_KEY"
SSH_KEY="${SSH_KEYS[$LOCAL_SERVER_NAME|$REMOTE_SERVER_NAME]}"
[[ -z "$SSH_KEY" ]] && error "Missing SSH key mapping for $LOCAL_SERVER_NAME → $REMOTE_SERVER_NAME" && exit 1
[[ -z "$SSH_KEY" ]] && error "Missing SSH key mapping" && exit 1
info "$ICON_HOST Host: $LOCAL_SERVER_NAME → $REMOTE_SERVER_NAME"
}
# -----------------------------------------------------------------------------------------------
# REMOTE IP RESOLUTION
# Resolves the Tailscale IPv4 address of the remote server.
# Sets REMOTE_SERVER used by all subsequent SSH and rsync calls.
# Exits if resolution fails — likely means Tailscale is down or peer is offline.
# -----------------------------------------------------------------------------------------------
resolve_remote_ip() {
log "Resolving remote IP for $REMOTE_SERVER_NAME..."
REMOTE_SERVER=$(tailscale ip -4 "$REMOTE_SERVER_NAME" 2>/dev/null)
[[ -z "$REMOTE_SERVER" ]] && error "Failed to resolve Tailscale IP for $REMOTE_SERVER_NAME" && exit 1
info "$ICON_NET Remote IP: $REMOTE_SERVER"
}
# -----------------------------------------------------------------------------------------------
# CONNECTIVITY CHECK
# Pings the remote server to confirm it is reachable before starting any transfers.
# Prevents the rsync retry loop from burning all attempts against an unreachable host.
# If ping fails, prints a tailscale status hint to aid diagnosis before exiting.
# CONNECTIVITY CHECK — fatal, used by rsync scripts
# -----------------------------------------------------------------------------------------------
check_connectivity() {
log "Checking connectivity to $REMOTE_SERVER..."
@@ -323,16 +299,78 @@ check_connectivity() {
}
# -----------------------------------------------------------------------------------------------
# REMOTE ROOTFS SPACE CHECK
# Checks the remote server's rootfs usage before any rsync runs.
# If the array is down or drives are missing, rsync writes land on rootfs instead of the array —
# this can fill the remote filesystem rapidly and crash the server.
# Threshold is set by ROOTFS_WARN in Master.conf (recommended: 75).
# Aborts cleanly with a clear error showing current usage vs threshold.
# PING REMOTE — non-fatal, used by failover
# Returns 0 if reachable, 1 if not
# -----------------------------------------------------------------------------------------------
ping_remote() {
ping -c2 -W3 "$REMOTE_SERVER" &>/dev/null
}
# -----------------------------------------------------------------------------------------------
# PING INTERNET — non-fatal, used by failover
# Returns 0 if internet reachable, 1 if not
# -----------------------------------------------------------------------------------------------
ping_internet() {
ping -c2 -W3 "${EXTERNAL_IP:-8.8.8.8}" &>/dev/null
}
# -----------------------------------------------------------------------------------------------
# LOCAL ARRAY CHECK — non-fatal, returns status
# Used by failover before starting remote containers locally
# -----------------------------------------------------------------------------------------------
check_local_array() {
log "Checking local array..."
if ! mountpoint -q /mnt/user 2>/dev/null; then
error "$ICON_DISK Local array is not started — /mnt/user is not mounted"
return 1
fi
local file_count
file_count=$(ls /mnt/user 2>/dev/null | wc -l)
if [[ "$file_count" -eq 0 ]]; then
error "$ICON_DISK Local array appears empty — shares may not be available"
return 1
fi
log "Local array is healthy"
return 0
}
# -----------------------------------------------------------------------------------------------
# REMOTE ARRAY CHECK — non-fatal, returns status
# Used before handback rsync
# -----------------------------------------------------------------------------------------------
check_remote_array() {
log "Checking remote array on $REMOTE_SERVER_NAME..."
local result
result=$(ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \
"mountpoint -q /mnt/user && echo yes || echo no" 2>/dev/null)
if [[ "$result" != "yes" ]]; then
error "$ICON_DISK Remote array not started on $REMOTE_SERVER_NAME"
return 1
fi
log "Remote array is healthy"
return 0
}
# -----------------------------------------------------------------------------------------------
# REMOTE DOCKER CHECK — non-fatal, returns status
# Used before starting containers on remote
# -----------------------------------------------------------------------------------------------
check_remote_docker() {
log "Checking remote Docker daemon on $REMOTE_SERVER_NAME..."
if ! ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \
"timeout 10 docker ps" >/dev/null 2>&1; then
error "$ICON_CONTAINERS Remote Docker daemon not responding on $REMOTE_SERVER_NAME"
return 1
fi
log "Remote Docker daemon is healthy"
return 0
}
# -----------------------------------------------------------------------------------------------
# REMOTE ROOTFS SPACE CHECK — fatal
# -----------------------------------------------------------------------------------------------
check_remote_rootfs() {
log "Checking remote rootfs usage..."
REMOTE_USAGE=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
"df / --output=pcent | tail -1 | tr -d ' %'" 2>/dev/null)
@@ -342,39 +380,27 @@ check_remote_rootfs() {
fi
if [[ "$REMOTE_USAGE" -ge "${ROOTFS_WARN:-75}" ]]; then
echo ""
error "$ICON_HEALTH Remote rootfs is ${REMOTE_USAGE}% full — threshold is ${ROOTFS_WARN:-75}%"
error "$ICON_HEALTH Remote rootfs ${REMOTE_USAGE}% — threshold ${ROOTFS_WARN:-75}%"
warn "Array may be down or drives missing on $REMOTE_SERVER_NAME"
info "Hint: Check array status on $REMOTE_SERVER_NAME before retrying"
echo ""
exit 1
fi
info "$ICON_HEALTH Remote rootfs: ${REMOTE_USAGE}% used (threshold: ${ROOTFS_WARN:-75}%)"
info "$ICON_HEALTH Remote rootfs: ${REMOTE_USAGE}% (threshold: ${ROOTFS_WARN:-75}%)"
}
# -----------------------------------------------------------------------------------------------
# REMOTE SHARE VALIDATION
# Verifies that the target directory exists and is not empty on the remote server.
# Catches the scenario where the array is mounted but drives are not backing the share —
# the path exists as an empty mountpoint, which would cause --delete to wipe the remote.
# Called with the specific directory being synced so each share is checked individually.
# REMOTE SHARE VALIDATION — fatal
# Usage: check_remote_share "/mnt/user/Movies"
# -----------------------------------------------------------------------------------------------
check_remote_share() {
local dir="$1"
log "Checking remote share: $dir..."
SHARE_EXISTS=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
"[[ -d '$dir' ]] && echo yes || echo no" 2>/dev/null)
if [[ "$SHARE_EXISTS" != "yes" ]]; then
echo ""
error "$ICON_HEALTH Remote share does not exist: $dir"
warn "Array may not be started or share is not configured on $REMOTE_SERVER_NAME"
info "Hint: Check shares and array status on $REMOTE_SERVER_NAME before retrying"
echo ""
exit 1
fi
@@ -382,11 +408,7 @@ check_remote_share() {
"[[ -z \"\$(ls -A '$dir' 2>/dev/null)\" ]] && echo yes || echo no" 2>/dev/null)
if [[ "$SHARE_EMPTY" == "yes" ]]; then
echo ""
warn "$ICON_HEALTH Remote share exists but is empty: $dir"
warn "Drives may not be mounted on $REMOTE_SERVER_NAME — aborting to protect data"
info "Hint: Verify array and drive assignments on $REMOTE_SERVER_NAME before retrying"
echo ""
warn "$ICON_HEALTH Remote share exists but is empty: $dir — aborting to protect data"
exit 1
fi
@@ -394,12 +416,7 @@ check_remote_share() {
}
# -----------------------------------------------------------------------------------------------
# REMOTE DISK CHECK
# Verifies that all physical disks backing a share are online and mounted on the remote server.
# Discovers disk layout automatically at runtime by finding all /mnt/diskN/sharename paths —
# no configuration required, works for any share regardless of how many disks it spans.
# Aborts if any single disk backing the share is offline — partial disk failure means
# incomplete data which could result in files being deleted by --delete during sync.
# REMOTE DISK CHECK — fatal
# Usage: check_remote_disks "/mnt/user/Movies"
# -----------------------------------------------------------------------------------------------
check_remote_disks() {
@@ -413,20 +430,14 @@ check_remote_disks() {
"ls -d /mnt/disk*/$share_name 2>/dev/null" 2>/dev/null)
if [[ -z "$DISK_PATHS" ]]; then
echo ""
error "$ICON_DISK No disks found backing share $share_name on $REMOTE_SERVER_NAME"
warn "Share may not exist on any disk or array may not be started"
info "Hint: Check array and share configuration on $REMOTE_SERVER_NAME"
echo ""
error "$ICON_DISK No disks found backing $share_name on $REMOTE_SERVER_NAME"
exit 1
fi
local all_ok=true
while IFS= read -r disk_share_path; do
local disk_mount
local disk_mount disk_name
disk_mount=$(dirname "$disk_share_path")
local disk_name
disk_name=$(basename "$disk_mount")
MOUNTED=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
@@ -435,17 +446,13 @@ check_remote_disks() {
if [[ "$MOUNTED" == "yes" ]]; then
info "$ICON_DISK $disk_name $ICON_RUNNING — $share_name present"
else
error "$ICON_DISK $disk_name $ICON_STOPPED — $share_name missing or incomplete"
error "$ICON_DISK $disk_name $ICON_STOPPED — $share_name missing"
all_ok=false
fi
done <<< "$DISK_PATHS"
if [[ "$all_ok" == false ]]; then
echo ""
error "One or more disks backing $share_name are offline on $REMOTE_SERVER_NAME"
warn "Aborting to prevent partial or destructive sync"
info "Hint: Check disk assignments and array status on $REMOTE_SERVER_NAME before retrying"
echo ""
exit 1
fi
@@ -453,11 +460,7 @@ check_remote_disks() {
}
# -----------------------------------------------------------------------------------------------
# CONTAINER MANAGEMENT — STOP
# Stops all containers listed in CRITICAL_CONTAINER_NAMES on the remote server.
# Only stops containers that are currently running — skips those already stopped.
# Tracks stopped containers in RUNNING_CONTAINERS for restart after rsync completes.
# CRITICAL_CONTAINER_NAMES must be a bash array — rsync.sh handles conversion from profile strings.
# CONTAINER MANAGEMENT — STOP (remote via SSH)
# -----------------------------------------------------------------------------------------------
RUNNING_CONTAINERS=()
@@ -473,16 +476,12 @@ stop_containers() {
for c in "${CRITICAL_CONTAINER_NAMES[@]}"; do
[[ -z "$c" ]] && continue
info "Checking $c..."
STATUS=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
"docker inspect -f '{{.State.Running}}' $c 2>/dev/null" 2>/dev/null || echo "false")
if [[ "$STATUS" == "true" ]]; then
echo "$ICON_STOP Stopping $c..."
RUNNING_CONTAINERS+=("$c")
if ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" "docker stop $c" >/dev/null; then
echo "$ICON_STOPPED $c stopped"
else
@@ -495,11 +494,7 @@ stop_containers() {
}
# -----------------------------------------------------------------------------------------------
# CONTAINER MANAGEMENT — START
# Restarts only the containers that were running before rsync and were stopped by stop_containers.
# Containers listed in DELAYED_CONTAINERS receive a sleep of CONTAINER_DELAY seconds before
# starting — useful for dependencies like Authelia that need upstream services ready first.
# DELAYED_CONTAINERS must be a bash array — rsync.sh handles conversion from profile strings.
# CONTAINER MANAGEMENT — START (remote via SSH)
# -----------------------------------------------------------------------------------------------
start_containers() {
if [[ ${#RUNNING_CONTAINERS[@]} -eq 0 ]]; then
@@ -514,10 +509,7 @@ start_containers() {
local needs_delay=false
for d in "${DELAYED_CONTAINERS[@]}"; do
if [[ "$c" == "$d" ]]; then
needs_delay=true
break
fi
[[ "$c" == "$d" ]] && needs_delay=true && break
done
if [[ "$needs_delay" == true ]]; then
@@ -536,10 +528,6 @@ start_containers() {
# -----------------------------------------------------------------------------------------------
# RSYNC OPTIONS
# Loads rsync options for the current profile from PROFILE_RSYNC_OPTS in Master.conf.
# If no profile match is found, falls back to DEFAULT_RSYNC_OPTS.
# Note: profile opts do NOT inherit from defaults — all desired flags must be listed explicitly.
# Sets RSYNC_OPTS array used directly in the rsync call in rsync.sh.
# -----------------------------------------------------------------------------------------------
get_rsync_opts() {
if [[ -n "${PROFILE_RSYNC_OPTS[$PROFILE_NAME]:-}" ]]; then
@@ -553,20 +541,17 @@ get_rsync_opts() {
# -----------------------------------------------------------------------------------------------
# STATUS DISPLAY
# Prints a summary of the current runtime configuration.
# Triggered by --status or --summary flag passed to any script.
# Useful for verifying profile resolution and variable state before a live run.
# -----------------------------------------------------------------------------------------------
show_status() {
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
echo "Local: $LOCAL_SERVER_NAME"
echo "Remote: $REMOTE_SERVER_NAME"
echo "IP: $REMOTE_SERVER"
echo "Profile: $PROFILE_NAME"
echo "Profile: ${PROFILE_NAME:-n/a}"
echo "DryRun: $DRY_RUN"
echo "Logging: $ENABLE_LOGGING"
echo "Containers: ${CRITICAL_CONTAINER_NAMES[*]}"
echo "Delayed: ${DELAYED_CONTAINERS[*]}"
echo "Excludes: ${EXCLUDE_DIRS[*]}"
echo "Containers: ${CRITICAL_CONTAINER_NAMES[*]:-n/a}"
echo "Delayed: ${DELAYED_CONTAINERS[*]:-n/a}"
echo "Excludes: ${EXCLUDE_DIRS[*]:-n/a}"
echo "━━━━━━━━━━━━━━━━━━━━━━━"
}
unRAID_Essentials/webgui_restart.sh
+198
View File
@@ -0,0 +1,198 @@
#!/bin/bash
# -----------------------------------------------------------------------------------------------
# --------------------------------- WebGUI Watchdog --------------------------------------------
# -----------------------------------------------------------------------------------------------
# Monitors unRAID's WebGUI and restarts it if unresponsive.
# Uses an escalating restart strategy — tries nginx first, then emhttp if needed.
# emhttp is the unRAID management daemon — restarting it is more disruptive than nginx
# but recovers cleanly. Notification sent on any restart so you know what happened.
#
# Escalation path:
# Check WebGUI → unresponsive → restart nginx → recheck
# Still unresponsive → restart emhttp → recheck
# Still unresponsive → notify warning, manual intervention needed
#
# Run every 5-10 minutes via cron/User Scripts plugin.
# All configuration in Master.conf under WebGUI Watchdog section.
# Supports --dry-run to show what would be restarted without acting.
# -----------------------------------------------------------------------------------------------
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../Master.conf"
source "$SCRIPT_DIR/../common.sh"
parse_args "$@"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Setup ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_GEAR Setup ━━━"
if [[ "$EUID" -ne 0 ]]; then
error "Must be run as root"
exit 1
fi
success "Running as root"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Status ━━━
# -----------------------------------------------------------------------------------------------
if [[ "$SHOW_STATUS" == true ]]; then
echo ""
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
echo "$ICON_WEBGUI URL: $WEBGUI_URL"
echo "$ICON_WEBGUI Curl timeout: ${WEBGUI_TIMEOUT}s"
echo "$ICON_WEBGUI Nginx wait: ${WEBGUI_NGINX_WAIT}s"
echo "$ICON_WEBGUI emhttp wait: ${WEBGUI_EMHTTP_WAIT}s"
echo "$ICON_GEAR Dry Run: $DRY_RUN"
# Show current state
if curl -sf --max-time "$WEBGUI_TIMEOUT" "$WEBGUI_URL" >/dev/null 2>&1; then
echo "$ICON_WEBGUI WebGUI: $ICON_RUNNING responding"
else
echo "$ICON_WEBGUI WebGUI: $ICON_NOT_RUNNING not responding"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
fi
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no services will be restarted"
# -----------------------------------------------------------------------------------------------
# FUNCTIONS
# -----------------------------------------------------------------------------------------------
# Check if WebGUI is responding
check_webgui() {
curl -sf --max-time "$WEBGUI_TIMEOUT" "$WEBGUI_URL" >/dev/null 2>&1
}
# Restart nginx — lightweight fix, try first
restart_nginx() {
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would restart nginx"
return 0
fi
info "$ICON_WEBGUI Restarting nginx..."
if /etc/rc.d/rc.nginx restart >/dev/null 2>&1; then
success "nginx restarted"
return 0
else
error "nginx restart failed"
return 1
fi
}
# Restart emhttp — heavier fix, escalate if nginx didn't help
# emhttp drives the array, Docker management, shares — recovers cleanly but takes longer
restart_emhttp() {
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would restart emhttp"
return 0
fi
info "$ICON_WEBGUI Restarting emhttp..."
if /etc/rc.d/rc.emhttp restart >/dev/null 2>&1; then
success "emhttp restarted"
return 0
else
error "emhttp restart failed"
return 1
fi
}
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_WEBGUI WebGUI Watchdog ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_WEBGUI WebGUI Watchdog — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
echo "$ICON_WEBGUI URL: $WEBGUI_URL"
echo ""
START=$(date +%s)
# Initial check
info "Checking WebGUI..."
if check_webgui; then
success "$ICON_WEBGUI WebGUI is responding — nothing to do"
echo ""
echo "━━━━━ $ICON_SUMMARY WEBGUI WATCHDOG SUMMARY ━━━━━"
echo "$ICON_WEBGUI Status: $ICON_RUNNING HEALTHY"
echo "$ICON_TIME Duration: $(format_duration $(($(date +%s) - START)))"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
fi
# WebGUI not responding — begin escalation
warn "$ICON_WEBGUI WebGUI is not responding at $WEBGUI_URL"
# ── Step 1: Restart nginx ──
echo ""
echo "━━━ $ICON_WEBGUI Step 1 — Nginx Restart ━━━"
restart_nginx
if [[ "$DRY_RUN" == false ]]; then
info "Waiting ${WEBGUI_NGINX_WAIT}s for nginx to recover..."
sleep "$WEBGUI_NGINX_WAIT"
if check_webgui; then
success "$ICON_WEBGUI WebGUI recovered after nginx restart"
notify "WebGUI recovered on $(hostname) after nginx restart" "WebGUI Watchdog" "warning"
echo ""
echo "━━━━━ $ICON_SUMMARY WEBGUI WATCHDOG SUMMARY ━━━━━"
echo "$ICON_WEBGUI Status: $ICON_SUCCESS RECOVERED via nginx restart"
echo "$ICON_TIME Duration: $(format_duration $(($(date +%s) - START)))"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
fi
warn "WebGUI still not responding after nginx restart — escalating to emhttp"
fi
# ── Step 2: Restart emhttp ──
echo ""
echo "━━━ $ICON_WEBGUI Step 2 — emhttp Restart ━━━"
warn "Restarting emhttp — this is the unRAID management daemon"
warn "Array, Docker management and shares remain running but WebGUI will be briefly unavailable"
restart_emhttp
if [[ "$DRY_RUN" == false ]]; then
info "Waiting ${WEBGUI_EMHTTP_WAIT}s for emhttp to recover..."
sleep "$WEBGUI_EMHTTP_WAIT"
if check_webgui; then
success "$ICON_WEBGUI WebGUI recovered after emhttp restart"
notify "WebGUI recovered on $(hostname) after emhttp restart — check system health" "WebGUI Watchdog" "warning"
echo ""
echo "━━━━━ $ICON_SUMMARY WEBGUI WATCHDOG SUMMARY ━━━━━"
echo "$ICON_WEBGUI Status: $ICON_SUCCESS RECOVERED via emhttp restart"
echo "$ICON_TIME Duration: $(format_duration $(($(date +%s) - START)))"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
fi
fi
# ── Both restarts failed ──
END=$(date +%s)
echo ""
echo "━━━━━ $ICON_SUMMARY WEBGUI WATCHDOG SUMMARY ━━━━━"
echo "$ICON_WEBGUI Status: $ICON_ERROR UNRECOVERED — manual intervention needed"
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
if [[ "$DRY_RUN" == false ]]; then
notify "WebGUI unrecovered on $(hostname) after nginx and emhttp restart — manual intervention needed" "WebGUI Watchdog" "warning"
exit 1
fi
exit 0
unRAID_Essentials/zfs_memory_snapshot.sh
+188 -61
View File
@@ -2,9 +2,16 @@
# -----------------------------------------------------------------------------------------------
# --------------------------------- ZFS Memory Snapshot ----------------------------------------
# -----------------------------------------------------------------------------------------------
# Captures a point-in-time snapshot of ZFS ARC statistics and system memory usage.
# Read-only diagnostic tool — no changes are made to the system regardless of flags.
# Dry run mode still collects and displays data since no modifications occur.
# Weekly ZFS pool health and memory diagnostic report.
# Combines ZFS pool status, ARC statistics, memory summary, Docker memory usage
# and kernel pressure into a single report. Informational only — no action taken.
# system_watchdog.sh handles threshold-based intervention.
#
# Output goes to both console and ZFS_REPORT_LOG for later review.
# Notifies if any warning thresholds are exceeded.
#
# All configuration in Master.conf under ZFS Memory Snapshot section.
# Supports --dry-run (preview only, no log write) and --status.
# -----------------------------------------------------------------------------------------------
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
@@ -14,14 +21,24 @@ source "$SCRIPT_DIR/../common.sh"
parse_args "$@"
# Tee output to log file unless dry run
if [[ "$DRY_RUN" == false ]]; then
mkdir -p "$(dirname "$ZFS_REPORT_LOG")"
exec > >(tee -a "$ZFS_REPORT_LOG") 2>&1
fi
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Setup ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_GEAR Setup ━━━"
info "$ICON_ZFS ZFS ARC + Memory Snapshot"
info "$ICON_TIME $(date)"
if [[ "$EUID" -ne 0 ]]; then
error "Must be run as root"
exit 1
fi
success "Running as root"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Status ━━━
@@ -29,65 +46,167 @@ info "$ICON_TIME $(date)"
if [[ "$SHOW_STATUS" == true ]]; then
echo ""
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
echo "$ICON_ZFS Mode: Read-only diagnostics"
echo "$ICON_MEM Source: ZFS ARC + system memory"
echo "$ICON_GEAR Dry Run: $DRY_RUN"
echo "$ICON_ZFS Log file: $ZFS_REPORT_LOG"
echo "$ICON_ZFS ARC warn: ${ZFS_REPORT_ARC_WARN_PCT}%"
echo "$ICON_MEM Free RAM warn: ${ZFS_REPORT_FREE_WARN_GB}GB"
echo "$ICON_MEM Avail RAM warn: ${ZFS_REPORT_AVAIL_WARN_GB}GB"
echo "$ICON_CONTAINERS Docker top: $ZFS_REPORT_DOCKER_TOP"
echo "$ICON_GEAR Dry Run: $DRY_RUN"
echo "━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
fi
# -----------------------------------------------------------------------------------------------
# FUNCTIONS
# -----------------------------------------------------------------------------------------------
# Reads ZFS ARC statistics from the kernel stats interface.
# Filters for the most useful ARC metrics — size, hits, misses and metadata.
# Skips gracefully if ZFS is not available on this system.
show_zfs_arc() {
echo ""
echo "━━━ $ICON_ZFS ZFS ARC Stats ━━━"
if [[ ! -r /proc/spl/kstat/zfs/arcstats ]]; then
warn "ZFS arcstats not available on this system — is ZFS loaded?"
return
fi
grep -iE '^(c|size|hits|misses|arc_meta_used|demand_metadata_misses|mru_ghost_metadata|mfu_ghost_metadata)' \
/proc/spl/kstat/zfs/arcstats 2>/dev/null || warn "Unable to read ARC stats"
}
# Reads current system memory and swap usage using free.
# Skips gracefully if free is not available.
show_memory_status() {
echo ""
echo "━━━ $ICON_MEM Memory Status ━━━"
if ! command -v free >/dev/null 2>&1; then
warn "free command not available on this system"
return
fi
free -h | awk '
NR==1 { print $0 }
/Mem:/ { print $0 }
/Swap:/ { print $0 }'
}
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — output will not be written to log"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_ZFS Snapshot ━━━
# Tracking
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_ZFS Snapshot ━━━"
echo "$ICON_ZFS Source: ZFS ARC + system memory"
echo "$ICON_GEAR Dry Run: $DRY_RUN"
echo ""
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — this script is read-only, data will still be collected"
WARNINGS=()
START=$(date +%s)
DATE=$(date +"%Y-%m-%d %H:%M:%S")
show_zfs_arc
show_memory_status
echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo " $ICON_ZFS ZFS WEEKLY HEALTH REPORT — $DATE"
echo " $ICON_HOST Host: $(hostname)"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_ZFS ZFS Pool Health ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_ZFS ZFS Pool Health ━━━"
if ! command -v zpool >/dev/null 2>&1; then
warn "ZFS not available on this system — skipping pool checks"
else
info "Pool status:"
zpool status 2>/dev/null | grep -E "pool:|state:|status:|errors:|scan:" | while IFS= read -r line; do
echo " $line"
done
echo ""
info "Pool overview:"
zpool list 2>/dev/null | while IFS= read -r line; do
echo " $line"
done
UNHEALTHY=$(zpool list -H -o health 2>/dev/null | grep -v ONLINE || true)
if [[ -n "$UNHEALTHY" ]]; then
error "One or more ZFS pools are NOT ONLINE"
WARNINGS+=("ZFS pool unhealthy")
else
success "All ZFS pools are ONLINE"
fi
fi
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_ZFS ARC Statistics ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_ZFS ARC Statistics ━━━"
if [[ ! -f /proc/spl/kstat/zfs/arcstats ]]; then
warn "ZFS arcstats not available — skipping ARC section"
else
ARC_MAX=$(cat /sys/module/zfs/parameters/zfs_arc_max 2>/dev/null || \
awk '/^c_max / {print $3}' /proc/spl/kstat/zfs/arcstats)
ARC_SIZE=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats)
ARC_META_USED=$(awk '/^arc_meta_used / {print $3}' /proc/spl/kstat/zfs/arcstats)
ARC_MAX_GB=$(awk "BEGIN {printf \"%.1f\", $ARC_MAX / 1073741824}")
ARC_CUR_GB=$(awk "BEGIN {printf \"%.1f\", $ARC_SIZE / 1073741824}")
ARC_META_GB=$(awk "BEGIN {printf \"%.1f\", $ARC_META_USED / 1073741824}")
ARC_PCT=$(awk "BEGIN {printf \"%.1f\", $ARC_SIZE * 100 / $ARC_MAX}")
ARC_PCT_INT=$(printf "%.0f" "$ARC_PCT")
echo " $ICON_ZFS ARC Max: ${ARC_MAX_GB}GB"
echo " $ICON_ZFS ARC Current: ${ARC_CUR_GB}GB"
echo " $ICON_ZFS ARC Meta Used: ${ARC_META_GB}GB"
echo " $ICON_ZFS ARC Utilization: ${ARC_PCT}%"
if [[ "$ARC_PCT_INT" -ge "$ZFS_REPORT_ARC_WARN_PCT" ]]; then
warn "ARC utilization ${ARC_PCT}% — above ${ZFS_REPORT_ARC_WARN_PCT}% threshold"
WARNINGS+=("ARC high: ${ARC_PCT}%")
else
success "ARC utilization ${ARC_PCT}% — within threshold"
fi
echo ""
info "Metadata pressure:"
META_MRU_GHOST=$(awk '/^mru_ghost_metadata / {print $3}' /proc/spl/kstat/zfs/arcstats 2>/dev/null || echo 0)
META_MFU_GHOST=$(awk '/^mfu_ghost_metadata / {print $3}' /proc/spl/kstat/zfs/arcstats 2>/dev/null || echo 0)
META_MISSES=$(awk '/^demand_metadata_misses / {print $3}' /proc/spl/kstat/zfs/arcstats 2>/dev/null || echo 0)
MRU_GB=$(awk "BEGIN {printf \"%.2f\", $META_MRU_GHOST / 1073741824}")
MFU_GB=$(awk "BEGIN {printf \"%.2f\", $META_MFU_GHOST / 1073741824}")
echo " $ICON_ZFS MRU Ghost: ${MRU_GB}GB"
echo " $ICON_ZFS MFU Ghost: ${MFU_GB}GB"
echo " $ICON_ZFS Metadata Misses: ${META_MISSES}"
fi
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_MEM Memory Status ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_MEM Memory Status ━━━"
FREE_HUMAN=$(free -h | awk '/Mem:/ {print $4}')
AVAIL_HUMAN=$(free -h | awk '/Mem:/ {print $7}')
TOTAL_HUMAN=$(free -h | awk '/Mem:/ {print $2}')
FREE_GB=$(free -g | awk '/Mem:/ {print $4}')
AVAIL_GB=$(free -g | awk '/Mem:/ {print $7}')
echo " $ICON_MEM Total RAM: $TOTAL_HUMAN"
echo " $ICON_MEM Free RAM: $FREE_HUMAN"
echo " $ICON_MEM Available RAM: $AVAIL_HUMAN"
if [[ "$FREE_GB" -lt "$ZFS_REPORT_FREE_WARN_GB" ]]; then
warn "Free RAM ${FREE_HUMAN} — below ${ZFS_REPORT_FREE_WARN_GB}GB threshold"
WARNINGS+=("Low free RAM: ${FREE_HUMAN}")
else
success "Free RAM ${FREE_HUMAN} — within threshold"
fi
if [[ "$AVAIL_GB" -lt "$ZFS_REPORT_AVAIL_WARN_GB" ]]; then
warn "Available RAM ${AVAIL_HUMAN} — below ${ZFS_REPORT_AVAIL_WARN_GB}GB threshold"
WARNINGS+=("Low available RAM: ${AVAIL_HUMAN}")
else
success "Available RAM ${AVAIL_HUMAN} — within threshold"
fi
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_CONTAINERS Docker Memory ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_CONTAINERS Top $ZFS_REPORT_DOCKER_TOP Docker Memory Users ━━━"
if ! command -v docker >/dev/null 2>&1; then
warn "Docker not available — skipping container memory section"
else
docker stats --no-stream \
--format "table {{.Name}}\t{{.MemUsage}}\t{{.MemPerc}}" \
2>/dev/null | head -n $(( ZFS_REPORT_DOCKER_TOP + 1 )) | while IFS= read -r line; do
echo " $line"
done
fi
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Kernel Pressure ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_GEAR Kernel Pressure ━━━"
if ! command -v vmstat >/dev/null 2>&1; then
warn "vmstat not available — skipping kernel pressure section"
else
info "vmstat snapshot (3 samples):"
vmstat 1 3 2>/dev/null | while IFS= read -r line; do
echo " $line"
done
fi
END=$(date +%s)
@@ -95,9 +214,17 @@ END=$(date +%s)
# ━━━ $ICON_SUMMARY Summary ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━━━ $ICON_SUMMARY SNAPSHOT SUMMARY ━━━━━"
echo "$ICON_ZFS Source: ZFS ARC + system memory"
echo "$ICON_GEAR Mode: READ-ONLY — no changes made"
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
echo "$ICON_DONE Status: $ICON_SUCCESS DONE"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "━━━━━ $ICON_SUMMARY ZFS REPORT SUMMARY ━━━━━"
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
echo "$ICON_ZFS Log: $ZFS_REPORT_LOG"
echo ""
if [[ ${#WARNINGS[@]} -eq 0 ]]; then
success "Report complete — no warnings"
else
echo "$ICON_WARN Warnings: ${#WARNINGS[@]}"
for w in "${WARNINGS[@]}"; do
echo " $ICON_WARN $w"
done
notify "ZFS weekly report on $(hostname) — ${#WARNINGS[@]} warning(s): ${WARNINGS[*]}" "ZFS Report" "warning"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
user_script_pluin.sh
+101 -74
View File
@@ -16,18 +16,18 @@
# Shared runtime functions used by all scripts:
# /mnt/user/appdata/unraid_scripts/common.sh
#
# For full setup instructions see:
# /mnt/user/appdata/unraid_scripts/README.md
#
# ==============================================================================================
# Changelog:
# v1.0 — Initial template
# v1.1 — MAX_RSYNC_PROCS removed — bandwidth limiting handles concurrency
# Typo fixes Master.cong → Master.conf
# Added --status flag to arguments section
# Added full directory tree
# All script calls pre-written and grouped by type
# Changelog added
# Typo fixes, --status flag added, full directory tree, changelog added
# v1.2 — Docker Essentials, Media, Transcodes, System Watchdog added
# Directory tree updated to reflect full ecosystem
# v1.3 — Failover script added, directory tree updated with Failover folder
# v1.4 — WebGUI watchdog added
# ZFS memory snapshot added
# Docker network connect added
# Directory tree updated to reflect full ecosystem
# ==============================================================================================
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
@@ -39,42 +39,60 @@
# ├── common.sh # Shared library — functions used by all scripts
# ├── README.md # Project overview and quick start
# ├── User_Script_Template.sh # This file — copy into User Scripts plugin
# ├── git_pull_execute.sh # Pulls latest scripts from Gitea repo
# │
# ├── Failover/
# │ └── failover.sh # Mutual container failover — runs continuously
# │
# ├── Orchestrators/
# │ └── daily_sync.sh # Runs all daily media share syncs sequentially
# │ # add shares to master.conf, for seqential syncs
# │
# ├── Rsync/
# │ ├── rsync.sh # Core rsync script — called per share or profile
# │ └── README_Rsync_Setup.md # Rsync-specific setup guide
# │
# ├── Docker_Essentials/
# │ ├── docker_watchdog.sh # Container health monitor — memory, CPU, HTTP, stops
# │ ├── docker_daily_restart.sh # Restarts configured containers daily
# │ ├── docker_weekly_restart.sh # Restarts configured containers weekly
# │ └── docker_network_connect.sh # Connects containers to extra networks on boot
# │
# ├── Media/
# │ ├── media_permissions.sh # Applies permissions to all media shares
# │ └── media_cleaner.sh # Removes junk files — profiles: anime, media
# │
# ├── Transcodes/
# │ ├── ramdisk_setup.sh # Creates ramdisk and transcode symlink — run at array start
# │ ├── transcode_manager.sh # Monitors ramdisk usage, manages symlink direction
# │ └── transcode_cleanup.sh # Removes old inactive transcode files
# │
# ├── Tools/
# │ └── recreate_shares.sh # Recreates share dirs from .cfg files after incident
# │
# └── unRAID_Essentials/
# ├── clear_logs.sh # Clears unRAID log files
# ├── docker_syslog_filter.sh # Filters docker veth noise from syslog
# ├── clear_logs.sh # Clears unRAID system and Docker log files
# ├── docker_syslog_filter.sh # Filters Docker veth noise from syslog
# ├── mover_stop.sh # Safely stops the unRAID mover
# ├── php_fpm_max_children.sh # Sets php-fpm max children value
# ├── rsync_stop.sh # Cleanly stops all running rsync processes
# ├── php_fpm_max_children.sh # Sets PHP-FPM max children value
# ├── rsync_stop.sh # Stops all rsync processes on both servers
# ├── server_reboot.sh # Graceful server reboot with user warning
# ├── user_script_stop.sh # Stops a running User Scripts job
# └── zfs_memory_snapshot.sh # Creates a ZFS memory snapshot
# ├── system_watchdog.sh # System health monitor — last line of defense
# ├── user_script_stop.sh # Stops running User Scripts plugin jobs
# ├── webgui_restart.sh # WebGUI watchdog — nginx + emhttp escalating restart
# └── zfs_memory_snapshot.sh # Weekly ZFS health and memory diagnostic report
#
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
# ━━━ 🚀 Script Commands — Uncomment the one you want to run ━━━
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
#
# ━━━ Orchestrators (README) ━━━
# The daily orchestrator runs all bulk media shares sequentially in a single scheduled job.
# Instead of creating a User Script entry per share, add the share path to DAILY_SYNC_SHARES
# in Master.conf and the orchestrator handles it automatically at 1am.
# ━━━ Failover (README) ━━━
# Runs continuously as a background task — do NOT schedule with a time interval.
# Set schedule to "At Startup of Array" — background task.
# Both servers must have this script running for mutual failover to work.
# Use --status to check current state without restarting the loop.
#
# Media shares are nothing special — they carry no profile and fall through to global defaults
# in Master.conf. Sequential execution means one share finishes before the next starts,
# no concurrency needed, bandwidth limiting keeps things sane if appdata jobs overlap.
#
# This is where 80% of your rsync shares should live. Only create individual scheduled
# entries for shares that need their own timing — like appdata profiles below.
# ━━━ Failover ━━━
#/mnt/user/appdata/unraid_scripts/Failover/failover.sh
#
# ━━━ Orchestrators ━━━
#/mnt/user/appdata/unraid_scripts/Orchestrators/daily_sync.sh
@@ -86,9 +104,36 @@
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-Failover/Emby
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-Failover/Gmer4Lfe
#
# ━━━ Rsync — Individual Media Share (ad hoc use) ━━━
# ━━━ Rsync — Individual Media Shares (ad hoc use) ━━━
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Movies
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Tv_Shows
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Anime_Shows
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Anime_Shows-Old
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Anime_Movies
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Anime_Movies-Old
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Books
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Intros
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Kids_Movies
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Kids_Tv_Shows
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Music_Videos
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Nextcloud
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/stand-up_comedy
#
# ━━━ Docker Essentials ━━━
#/mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_watchdog.sh
#/mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_daily_restart.sh
#/mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_weekly_restart.sh
#/mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_network_connect.sh
#
# ━━━ Media ━━━
#/mnt/user/appdata/unraid_scripts/Media/media_permissions.sh
#/mnt/user/appdata/unraid_scripts/Media/media_cleaner.sh anime
#/mnt/user/appdata/unraid_scripts/Media/media_cleaner.sh media
#
# ━━━ Transcodes ━━━
#/mnt/user/appdata/unraid_scripts/Transcodes/ramdisk_setup.sh
#/mnt/user/appdata/unraid_scripts/Transcodes/transcode_manager.sh
#/mnt/user/appdata/unraid_scripts/Transcodes/transcode_cleanup.sh
#
# ━━━ Tools ━━━
#/mnt/user/appdata/unraid_scripts/Tools/recreate_shares.sh
@@ -100,7 +145,9 @@
#/mnt/user/appdata/unraid_scripts/unRAID_Essentials/php_fpm_max_children.sh
#/mnt/user/appdata/unraid_scripts/unRAID_Essentials/rsync_stop.sh
#/mnt/user/appdata/unraid_scripts/unRAID_Essentials/server_reboot.sh
#/mnt/user/appdata/unraid_scripts/unRAID_Essentials/system_watchdog.sh
#/mnt/user/appdata/unraid_scripts/unRAID_Essentials/user_script_stop.sh
#/mnt/user/appdata/unraid_scripts/unRAID_Essentials/webgui_restart.sh
#/mnt/user/appdata/unraid_scripts/unRAID_Essentials/zfs_memory_snapshot.sh
#
@@ -108,66 +155,46 @@
# ━━━ ⚙️ Arguments — Add after the script path ━━━
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
#
# ━━━ Rsync / Orchestrators ━━━
#
# --dry-run Preview what would be transferred — no changes made
# --dry-run Preview what would happen — no changes made
# --log Enable verbose logging output
# --no-log Disable logging (overrides Master.conf setting)
# --status Print resolved profile and configuration then exit
# --status Print resolved configuration and current state then exit
# --help Show usage information
#
# KEY=VALUE Override any Master.conf variable for this run only
# Better to add a profile in Master.conf for permanent changes
#
# ━━━ unRAID Essentials ━━━
# ━━━ Media Cleaner profile (required first argument) ━━━
# media_cleaner.sh anime --dry-run
# media_cleaner.sh media --dry-run
#
# Arguments vary per script — see comments inside each script for details
# ━━━ Failover specific ━━━
# failover.sh --status — check current state
# failover.sh --dry-run --log — test logic without touching containers
#
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
# ━━━ 📋 Profile System ━━━
# ━━━ 📋 Recommended Schedules ━━━
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
#
# Profiles are defined in Master.conf and matched automatically by the
# basename of the directory path passed to rsync.sh (lowercased).
# failover.sh — At Startup of Array (background task)
# ramdisk_setup.sh — At Startup of Array
# docker_network_connect.sh — At Startup of Array
# docker_syslog_filter.sh — At Startup of Array
# php_fpm_max_children.sh — At Startup of Array
#
# Example:
# /mnt/user/appdata-Failover/Arrs_Stack → matches profile key [arrs_stack]
# /mnt/user/Movies → no match, uses global defaults
# transcode_manager.sh — */3 * * * * (every 3 minutes)
# transcode_cleanup.sh — */5 * * * * (every 5 minutes)
# docker_watchdog.sh — */15 * * * * (every 15 minutes)
# webgui_restart.sh — */10 * * * * (every 10 minutes)
# system_watchdog.sh — */15 * * * * (every 15 minutes)
#
# If a profile match is found — profile settings are used for that run
# If no profile match is found — global defaults from Master.conf are used
# daily_sync.sh — 0 1 * * * (daily at 1am)
# docker_daily_restart.sh — 0 3 * * * (daily at 3am)
# media_cleaner.sh anime — 0 2 * * * (daily at 2am)
# media_cleaner.sh media — 0 2 * * * (daily at 2am)
# media_permissions.sh — 0 4 * * 0 (weekly Sunday 4am)
# clear_logs.sh — 0 5 * * 0 (weekly Sunday 5am)
# zfs_memory_snapshot.sh — 0 6 * * 0 (weekly Sunday 6am)
# docker_weekly_restart.sh — 0 3 * * 0 (weekly Sunday 3am)
#
# Profile settings control:
# - rsync options PROFILE_RSYNC_OPTS
# - bandwidth limit PROFILE_BW_LIMIT
# - retry count PROFILE_RETRY_COUNT
# - sleep between retry PROFILE_SLEEP
# - containers to stop PROFILE_CRITICAL_CONTAINER_NAMES
# - delayed containers PROFILE_DELAYED_CONTAINERS
# - container delay PROFILE_CONTAINER_DELAY
# - excluded dirs PROFILE_EXCLUDE_DIRS
#
# To add a new profile — add a key to each array in Master.conf
# For permanent changes — always use Master.conf
# For one-off overrides — use KEY=VALUE arguments (see above)
#
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
# ━━━ 💡 Examples ━━━
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
#
# Run arrs_stack profile sync:
# /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-Failover/Arrs_Stack
#
# Dry run with logging enabled:
# /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-Failover/Arrs_Stack --dry-run --log
#
# Check what profile and settings resolved before running:
# /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-Failover/Arrs_Stack --status
#
# Override bandwidth limit for this run only:
# /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Movies BW_LIMIT=5000
#
# Run daily orchestrator:
# /mnt/user/appdata/unraid_scripts/Orchestrators/daily_sync.sh
# rsync.sh profiles — schedule individually as needed
#
# ==============================================================================================