diff --git a/Docker_essentials/docker_network_connect.sh b/Docker_essentials/docker_network_connect.sh new file mode 100644 index 0000000..d40b522 --- /dev/null +++ b/Docker_essentials/docker_network_connect.sh @@ -0,0 +1,159 @@ +#!/bin/bash +# ----------------------------------------------------------------------------------------------- +# --------------------------------- Docker Network Connect ------------------------------------- +# ----------------------------------------------------------------------------------------------- +# Connects specified Docker containers to extra networks on array start. +# Useful when containers need to communicate across networks they were not +# originally configured with — e.g. memcached needing access to nextcloud-aio network. +# +# Every container in NETWORK_CONNECT_CONTAINERS is connected to every network +# in NETWORK_CONNECT_NETWORKS. Already-connected containers are skipped cleanly. +# +# Run once at array start via User Scripts plugin. +# All configuration in Master.conf under Docker Network Connect section. +# Supports --dry-run to preview connections without making them. +# ----------------------------------------------------------------------------------------------- + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "$SCRIPT_DIR/../Master.conf" +source "$SCRIPT_DIR/../common.sh" + +parse_args "$@" + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_GEAR Setup ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_GEAR Setup ━━━" + +if [[ "$EUID" -ne 0 ]]; then + error "Must be run as root" + exit 1 +fi + +success "Running as root" + +if ! command -v docker &>/dev/null; then + error "Docker not found" + exit 1 +fi + +success "Docker found" + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_SUMMARY Status ━━━ +# ----------------------------------------------------------------------------------------------- +if [[ "$SHOW_STATUS" == true ]]; then + echo "" + echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" + echo "$ICON_CONTAINERS Containers: ${NETWORK_CONNECT_CONTAINERS[*]}" + echo "$ICON_DOCKER_NET Networks: ${NETWORK_CONNECT_NETWORKS[*]}" + echo "$ICON_GEAR Dry Run: $DRY_RUN" + echo "" + echo "━━━ Current Connections ━━━" + for container in "${NETWORK_CONNECT_CONTAINERS[@]}"; do + [[ -z "$container" ]] && continue + echo "$ICON_CONTAINERS $container:" + docker inspect "$container" \ + --format '{{range $k, $v := .NetworkSettings.Networks}} {{$k}}{{"\n"}}{{end}}' \ + 2>/dev/null || echo " not found" + done + echo "━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no network connections will be made" + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_DOCKER_NET Network Connect ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_DOCKER_NET Network Connect — $(date '+%Y-%m-%d %H:%M:%S') ━━━" +echo "$ICON_CONTAINERS Containers: ${NETWORK_CONNECT_CONTAINERS[*]}" +echo "$ICON_DOCKER_NET Networks: ${NETWORK_CONNECT_NETWORKS[*]}" +echo "" + +START=$(date +%s) +CONNECTED=() +SKIPPED=() +FAILED=() + +for container in "${NETWORK_CONNECT_CONTAINERS[@]}"; do + [[ -z "$container" ]] && continue + + echo "━━━ $ICON_CONTAINERS $container ━━━" + + # Verify container exists + if ! docker inspect "$container" &>/dev/null; then + warn "$container not found — skipping" + FAILED+=("$container") + echo "" + continue + fi + + for network in "${NETWORK_CONNECT_NETWORKS[@]}"; do + [[ -z "$network" ]] && continue + + # Verify network exists + if ! docker network inspect "$network" &>/dev/null; then + warn "Network $network not found — skipping" + FAILED+=("$container:$network") + continue + fi + + # Check if already connected + if docker network inspect "$network" \ + --format '{{range .Containers}}{{.Name}} {{end}}' 2>/dev/null \ + | grep -qw "$container"; then + info "$ICON_DOCKER_NET $container already connected to $network — skipping" + SKIPPED+=("$container→$network") + continue + fi + + # Connect + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would connect $container to $network" + continue + fi + + info "$ICON_DOCKER_NET Connecting $container to $network..." + if docker network connect "$network" "$container" 2>/dev/null; then + success "$ICON_DOCKER_NET $container connected to $network" + CONNECTED+=("$container→$network") + else + error "Failed to connect $container to $network" + FAILED+=("$container:$network") + fi + done + + echo "" +done + +END=$(date +%s) + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_SUMMARY Summary ━━━ +# ----------------------------------------------------------------------------------------------- +echo "━━━━━ $ICON_SUMMARY NETWORK CONNECT SUMMARY ━━━━━" +echo "$ICON_TIME Duration: $(format_duration $((END - START)))" +[[ ${#CONNECTED[@]} -gt 0 ]] && echo "$ICON_DOCKER_NET Connected: ${CONNECTED[*]}" +[[ ${#SKIPPED[@]} -gt 0 ]] && echo "$ICON_RUNNING Skipped: ${SKIPPED[*]}" +[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}" +echo "" + +if [[ "$DRY_RUN" == true ]]; then + echo "$ICON_WARN Status: DRY RUN — no changes made" +elif [[ ${#FAILED[@]} -gt 0 ]]; then + echo "$ICON_ERROR Status: $ICON_ERROR SOME CONNECTIONS FAILED" + notify "Docker network connect failed on $(hostname) — ${FAILED[*]}" "Network Connect" "warning" +else + echo "$ICON_DONE Status: $ICON_SUCCESS DONE" + if [[ ${#CONNECTED[@]} -gt 0 ]]; then + notify "Docker networks connected on $(hostname) — ${CONNECTED[*]}" "Network Connect" "normal" + fi +fi +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + +[[ ${#FAILED[@]} -gt 0 ]] && exit 1 +exit 0 \ No newline at end of file diff --git a/Failover/failover.sh b/Failover/failover.sh new file mode 100644 index 0000000..d284829 --- /dev/null +++ b/Failover/failover.sh @@ -0,0 +1,512 @@ +#!/bin/bash +# ----------------------------------------------------------------------------------------------- +# --------------------------------- Failover Script -------------------------------------------- +# ----------------------------------------------------------------------------------------------- +# Mutual container failover between two unRAID servers. +# Runs continuously on BOTH servers — each operates fully autonomously. +# No coordination between servers — decisions based solely on ping results. +# +# States: +# NORMAL — remote up, internet up — own containers only +# FAILOVER — remote down, internet up — own + remote's containers (additive) +# NO_INTERNET — internet down — stop public-facing containers +# DARK — remote down + internet down — same as NO_INTERNET +# +# Handback (remote returns after FAILOVER): +# Strike confirmation → pre-flight checks → rsync → start on remote → stop locally +# +# All configuration in Master.conf under Failover section. +# Run via User Scripts plugin as a background task — runs continuously until stopped. +# Supports --dry-run to walk through logic without taking any action. +# ----------------------------------------------------------------------------------------------- + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "$SCRIPT_DIR/../Master.conf" +source "$SCRIPT_DIR/../common.sh" + +parse_args "$@" + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_GEAR Setup ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_GEAR Setup ━━━" + +if [[ "$EUID" -ne 0 ]]; then + error "Must be run as root" + exit 1 +fi + +success "Running as root" + +detect_hosts +resolve_remote_ip + +# Select correct arrays based on which server we are +if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then + FAILOVER_START_CONTAINERS=("${FAILOVER_HOST1_STARTS_FOR_HOST2[@]}") + FAILOVER_STOP_ON_NO_NET=("${FAILOVER_HOST1_STOP_ON_NO_NET[@]}") + FAILOVER_RSYNC_JOBS=("${FAILOVER_HOST1_RSYNC_JOBS[@]}") +else + FAILOVER_START_CONTAINERS=("${FAILOVER_HOST2_STARTS_FOR_HOST1[@]}") + FAILOVER_STOP_ON_NO_NET=("${FAILOVER_HOST2_STOP_ON_NO_NET[@]}") + FAILOVER_RSYNC_JOBS=("${FAILOVER_HOST2_RSYNC_JOBS[@]}") +fi + +info "$ICON_FAILOVER Failover containers: ${FAILOVER_START_CONTAINERS[*]}" +info "$ICON_STOP Stop on no-net: ${FAILOVER_STOP_ON_NO_NET[*]}" +info "$ICON_SYNC Rsync jobs: ${FAILOVER_RSYNC_JOBS[*]}" + +# Ensure state file directory exists +touch "$FAILOVER_STATE_FILE" 2>/dev/null || { + error "Cannot create state file: $FAILOVER_STATE_FILE" + exit 1 +} + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_SUMMARY Status ━━━ +# ----------------------------------------------------------------------------------------------- +if [[ "$SHOW_STATUS" == true ]]; then + echo "" + echo "━━━━━ $ICON_SUMMARY FAILOVER STATUS ━━━━━" + echo "$ICON_HOST Local: $LOCAL_SERVER_NAME" + echo "$ICON_NET Remote: $REMOTE_SERVER_NAME ($REMOTE_SERVER)" + echo "$ICON_PING External IP: $EXTERNAL_IP" + echo "$ICON_TIME Check interval: ${FAILOVER_CHECK_INTERVAL}s" + echo "$ICON_RETRY Handback strikes: $FAILOVER_HANDBACK_STRIKES" + echo "$ICON_FAILOVER Failover containers: ${FAILOVER_START_CONTAINERS[*]}" + echo "$ICON_STOP Stop on no-net: ${FAILOVER_STOP_ON_NO_NET[*]}" + echo "$ICON_SYNC Rsync jobs: ${FAILOVER_RSYNC_JOBS[*]}" + echo "$ICON_GEAR State file: $FAILOVER_STATE_FILE" + echo "$ICON_GEAR Dry Run: $DRY_RUN" + + if [[ -f "$FAILOVER_STATE_FILE" ]]; then + echo "" + echo "━━━ Persisted State ━━━" + cat "$FAILOVER_STATE_FILE" + fi + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be started or stopped" + +# ----------------------------------------------------------------------------------------------- +# STATE FILE HELPERS +# State persists to /boot/ so it survives reboots. +# On restart the script re-evaluates from scratch using live ping results. +# State file is reference only — pings are always the source of truth. +# ----------------------------------------------------------------------------------------------- + +get_state() { + grep -E "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d'=' -f2 +} + +set_state() { + local new_state="$1" + local timestamp + timestamp=$(date '+%Y-%m-%d %H:%M:%S') + cat > "$FAILOVER_STATE_FILE" </dev/null | cut -d'=' -f2 +} + +set_handback_strikes() { + local count="$1" + if grep -q "^handback_strikes=" "$FAILOVER_STATE_FILE" 2>/dev/null; then + sed -i "s/^handback_strikes=.*/handback_strikes=$count/" "$FAILOVER_STATE_FILE" + else + echo "handback_strikes=$count" >> "$FAILOVER_STATE_FILE" + fi +} + +# ----------------------------------------------------------------------------------------------- +# CONTAINER HELPERS — LOCAL operations for failover +# These operate on the LOCAL server unlike stop_containers/start_containers in common.sh +# which operate on the REMOTE server via SSH. +# ----------------------------------------------------------------------------------------------- + +# Start a container locally — skip if already running +start_local_container() { + local container="$1" + + local status + status=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null || echo "unknown") + + if [[ "$status" == "true" ]]; then + log "$container already running locally — skipping" + return 0 + fi + + if [[ "$status" == "unknown" ]]; then + warn "$container not found on this host — skipping" + return 1 + fi + + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would start $container locally" + return 0 + fi + + echo "$ICON_START Starting $container locally..." + if docker start "$container" >/dev/null 2>&1; then + echo "$ICON_STARTED $container started" + return 0 + else + error "Failed to start $container locally" + return 1 + fi +} + +# Stop a container locally — skip if already stopped +stop_local_container() { + local container="$1" + + local status + status=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null || echo "unknown") + + if [[ "$status" == "false" ]]; then + log "$container already stopped locally — skipping" + return 0 + fi + + if [[ "$status" == "unknown" ]]; then + log "$container not found on this host — skipping" + return 0 + fi + + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would stop $container locally" + return 0 + fi + + echo "$ICON_STOP Stopping $container locally..." + if docker stop "$container" >/dev/null 2>&1; then + echo "$ICON_STOPPED $container stopped" + return 0 + else + error "Failed to stop $container locally" + return 1 + fi +} + +# Start a container on the remote server via SSH +start_remote_container() { + local container="$1" + + local status + status=$(ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \ + "docker inspect -f '{{.State.Running}}' $container 2>/dev/null || echo unknown" 2>/dev/null) + + if [[ "$status" == "true" ]]; then + log "$container already running on remote — skipping" + return 0 + fi + + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would start $container on $REMOTE_SERVER_NAME" + return 0 + fi + + echo "$ICON_START Starting $container on $REMOTE_SERVER_NAME..." + if ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \ + "docker start $container" >/dev/null 2>&1; then + echo "$ICON_STARTED $container started on $REMOTE_SERVER_NAME" + return 0 + else + error "Failed to start $container on $REMOTE_SERVER_NAME" + return 1 + fi +} + +# ----------------------------------------------------------------------------------------------- +# STATE ACTIONS +# Each state has a corresponding action function. +# These are idempotent — safe to call on every loop iteration. +# ----------------------------------------------------------------------------------------------- + +# NORMAL — remote is up, internet is up +# Ensure failover containers are stopped locally (cleanup after returning from FAILOVER) +action_normal() { + local prev_state="$1" + + if [[ "$prev_state" == "FAILOVER" ]]; then + info "$ICON_FAILOVER Returning from FAILOVER — cleaning up local failover containers" + for c in "${FAILOVER_START_CONTAINERS[@]}"; do + [[ -z "$c" ]] && continue + stop_local_container "$c" + done + notify "Failover ended on $LOCAL_SERVER_NAME — $REMOTE_SERVER_NAME is back online" "Failover" "normal" + fi + + # Ensure no-net containers are running (they may have been stopped) + if [[ "$prev_state" == "NO_INTERNET" || "$prev_state" == "DARK" ]]; then + info "$ICON_START Internet restored — starting previously stopped containers" + for c in "${FAILOVER_STOP_ON_NO_NET[@]}"; do + [[ -z "$c" ]] && continue + start_local_container "$c" + done + notify "Internet restored on $LOCAL_SERVER_NAME — normal containers restarted" "Failover" "normal" + fi +} + +# FAILOVER — remote is down, internet is up +# Start remote's containers locally (additive — own containers keep running) +action_failover() { + local prev_state="$1" + + if [[ "$prev_state" != "FAILOVER" ]]; then + # Just entered failover — notify and start containers + info "$ICON_FAILOVER Entering FAILOVER — $REMOTE_SERVER_NAME is unreachable" + + # Check local array before starting containers + if ! check_local_array; then + error "Local array not ready — cannot start failover containers safely" + notify "Failover triggered but local array not ready on $LOCAL_SERVER_NAME" "Failover" "warning" + return 1 + fi + + # Check local docker daemon + if ! timeout 10 docker ps >/dev/null 2>&1; then + error "Local Docker daemon not responding — cannot start failover containers" + notify "Failover triggered but Docker not ready on $LOCAL_SERVER_NAME" "Failover" "warning" + return 1 + fi + + notify "Failover ACTIVE on $LOCAL_SERVER_NAME — starting ${REMOTE_SERVER_NAME} containers: ${FAILOVER_START_CONTAINERS[*]}" "Failover" "warning" + fi + + # Start each failover container — idempotent, skips if already running + for c in "${FAILOVER_START_CONTAINERS[@]}"; do + [[ -z "$c" ]] && continue + start_local_container "$c" + done +} + +# NO_INTERNET / DARK — this server has no internet +# Stop public-facing containers — no point serving if offline +action_no_internet() { + local prev_state="$1" + + if [[ "$prev_state" != "NO_INTERNET" && "$prev_state" != "DARK" ]]; then + warn "$ICON_WARN Internet lost on $LOCAL_SERVER_NAME — stopping public containers" + notify "Internet lost on $LOCAL_SERVER_NAME — stopping public-facing containers" "Failover" "warning" + fi + + for c in "${FAILOVER_STOP_ON_NO_NET[@]}"; do + [[ -z "$c" ]] && continue + stop_local_container "$c" + done + + # If we were in FAILOVER, also stop the failover containers + if [[ "$prev_state" == "FAILOVER" ]]; then + info "Was in FAILOVER — stopping failover containers too" + for c in "${FAILOVER_START_CONTAINERS[@]}"; do + [[ -z "$c" ]] && continue + stop_local_container "$c" + done + fi +} + +# HANDBACK — remote has returned, ready to hand containers back +# Confirmed by FAILOVER_HANDBACK_STRIKES consecutive remote-up checks +action_handback() { + info "$ICON_FAILOVER Initiating handback to $REMOTE_SERVER_NAME..." + notify "Handback starting on $LOCAL_SERVER_NAME — syncing data to $REMOTE_SERVER_NAME" "Failover" "normal" + + # Pre-flight checks before rsync + echo "" + echo "━━━ $ICON_SHIELD Handback Pre-flight ━━━" + + if ! check_remote_array; then + error "Remote array not ready — deferring handback" + notify "Handback deferred on $LOCAL_SERVER_NAME — remote array not ready" "Failover" "warning" + set_handback_strikes 0 + return 1 + fi + success "Remote array ready" + + if ! check_remote_docker; then + error "Remote Docker not ready — deferring handback" + notify "Handback deferred on $LOCAL_SERVER_NAME — remote Docker not ready" "Failover" "warning" + set_handback_strikes 0 + return 1 + fi + success "Remote Docker ready" + + REMOTE_USAGE=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ + "df / --output=pcent | tail -1 | tr -d ' %'" 2>/dev/null) + if [[ -n "$REMOTE_USAGE" ]] && [[ "$REMOTE_USAGE" -ge "${ROOTFS_WARN:-75}" ]]; then + error "Remote rootfs ${REMOTE_USAGE}% full — deferring handback" + notify "Handback deferred — remote rootfs ${REMOTE_USAGE}% full on $REMOTE_SERVER_NAME" "Failover" "warning" + set_handback_strikes 0 + return 1 + fi + success "Remote rootfs healthy" + + # Rsync data back via rsync.sh — uses existing profile system + echo "" + echo "━━━ $ICON_SYNC Handback Rsync ━━━" + local rsync_failed=false + + for path in "${FAILOVER_RSYNC_JOBS[@]}"; do + [[ -z "$path" ]] && continue + info "Syncing $path → $REMOTE_SERVER_NAME" + + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would rsync $path" + else + if bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$path" --no-log; then + success "Rsync complete: $path" + else + error "Rsync failed: $path" + rsync_failed=true + fi + fi + done + + if [[ "$rsync_failed" == true ]]; then + error "One or more rsync jobs failed — deferring container handback" + notify "Handback rsync failed on $LOCAL_SERVER_NAME — containers not handed back yet" "Failover" "warning" + set_handback_strikes 0 + return 1 + fi + + # Start containers on remote + echo "" + echo "━━━ $ICON_START $ICON_CONTAINERS Start on Remote ━━━" + local start_failed=false + + for c in "${FAILOVER_START_CONTAINERS[@]}"; do + [[ -z "$c" ]] && continue + if ! start_remote_container "$c"; then + start_failed=true + fi + done + + if [[ "$start_failed" == true ]]; then + error "One or more containers failed to start on remote — not stopping local copies" + notify "Handback partial failure on $LOCAL_SERVER_NAME — some containers failed to start on $REMOTE_SERVER_NAME" "Failover" "warning" + set_handback_strikes 0 + return 1 + fi + + # Stop local failover containers — only after remote confirmed started + echo "" + echo "━━━ $ICON_STOP $ICON_CONTAINERS Stop Local Failover ━━━" + for c in "${FAILOVER_START_CONTAINERS[@]}"; do + [[ -z "$c" ]] && continue + stop_local_container "$c" + done + + notify "Handback complete on $LOCAL_SERVER_NAME — $REMOTE_SERVER_NAME has containers back" "Failover" "normal" + set_handback_strikes 0 + return 0 +} + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_FAILOVER Main Loop ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_FAILOVER Failover Starting — $(date '+%Y-%m-%d %H:%M:%S') ━━━" +echo "$ICON_HOST Local: $LOCAL_SERVER_NAME" +echo "$ICON_NET Remote: $REMOTE_SERVER_NAME ($REMOTE_SERVER)" +echo "$ICON_TIME Interval: ${FAILOVER_CHECK_INTERVAL}s" +echo "" + +notify "Failover script started on $LOCAL_SERVER_NAME — monitoring $REMOTE_SERVER_NAME" "Failover" "normal" + +LOOP_COUNT=0 + +while true; do + LOOP_COUNT=$((LOOP_COUNT + 1)) + CURRENT_STATE=$(get_state) + [[ -z "$CURRENT_STATE" ]] && CURRENT_STATE="UNKNOWN" + + echo "" + echo "━━━ $ICON_PING Check #${LOOP_COUNT} — $(date '+%Y-%m-%d %H:%M:%S') ━━━" + log "Previous state: $CURRENT_STATE" + + # ── Two ping checks — the only inputs to state decisions ── + REMOTE_UP=false + INTERNET_UP=false + + ping_remote && REMOTE_UP=true + ping_internet && INTERNET_UP=true + + log "Remote reachable: $REMOTE_UP | Internet reachable: $INTERNET_UP" + + # ── Determine new state ── + if [[ "$REMOTE_UP" == true && "$INTERNET_UP" == true ]]; then + NEW_STATE="NORMAL" + elif [[ "$REMOTE_UP" == false && "$INTERNET_UP" == true ]]; then + NEW_STATE="FAILOVER" + elif [[ "$INTERNET_UP" == false ]]; then + NEW_STATE="NO_INTERNET" + else + NEW_STATE="DARK" + fi + + # ── Handle handback strike system ── + # When returning from FAILOVER to NORMAL, require consecutive confirmations + if [[ "$CURRENT_STATE" == "FAILOVER" && "$NEW_STATE" == "NORMAL" ]]; then + STRIKES=$(get_handback_strikes) + [[ -z "$STRIKES" ]] && STRIKES=0 + STRIKES=$((STRIKES + 1)) + set_handback_strikes "$STRIKES" + + if [[ "$STRIKES" -lt "$FAILOVER_HANDBACK_STRIKES" ]]; then + info "$ICON_RETRY Remote appears back — strike $STRIKES/$FAILOVER_HANDBACK_STRIKES — waiting for confirmation" + NEW_STATE="FAILOVER" # Stay in FAILOVER until strikes confirmed + else + info "$ICON_RETRY Remote confirmed stable ($STRIKES/$FAILOVER_HANDBACK_STRIKES) — initiating handback" + NEW_STATE="HANDBACK" + fi + else + # Reset handback strikes if we're not in the confirmation window + set_handback_strikes 0 + fi + + # ── Log state ── + if [[ "$NEW_STATE" != "$CURRENT_STATE" ]]; then + echo "$ICON_FAILOVER State: $CURRENT_STATE → $NEW_STATE" + else + info "State: $NEW_STATE (unchanged)" + fi + + # ── Execute state action ── + case "$NEW_STATE" in + NORMAL) + action_normal "$CURRENT_STATE" + set_state "NORMAL" + ;; + FAILOVER) + action_failover "$CURRENT_STATE" + set_state "FAILOVER" + ;; + NO_INTERNET|DARK) + action_no_internet "$CURRENT_STATE" + set_state "$NEW_STATE" + ;; + HANDBACK) + if action_handback; then + set_state "NORMAL" + action_normal "FAILOVER" + else + # Handback failed — stay in FAILOVER, retry next cycle + set_state "FAILOVER" + fi + ;; + esac + + echo "$ICON_TIME Next check in ${FAILOVER_CHECK_INTERVAL}s" + sleep "$FAILOVER_CHECK_INTERVAL" +done \ No newline at end of file diff --git a/Master.conf b/Master.conf index 733f988..a0995cd 100644 --- a/Master.conf +++ b/Master.conf @@ -20,10 +20,14 @@ # DAILY SYNC SHARES Media shares synced by daily_sync.sh # RSYNC PROFILE SYSTEM Per-profile overrides (appdata profiles) # +# ── FAILOVER ─────────────────────────────────────────────────────────────────────────────── +# FAILOVER Mutual container failover between two servers +# # ── DOCKER ESSENTIALS ────────────────────────────────────────────────────────────────────── # DOCKER DAILY RESTART Containers restarted daily # DOCKER WEEKLY RESTART Containers restarted weekly # DOCKER WATCHDOG Container health monitoring — memory, CPU, HTTP +# DOCKER NETWORK CONNECT Connect containers to extra networks on boot # # ── UNRAID ESSENTIALS ────────────────────────────────────────────────────────────────────── # REBOOT User warning delay before scheduled reboot @@ -31,6 +35,8 @@ # SYSLOG FILTER Docker veth noise filter file path # PHP-FPM PHP-FPM max children config # CLEAR LOGS System log file paths +# WEBGUI WATCHDOG WebGUI nginx + emhttp monitoring and restart +# ZFS MEMORY SNAPSHOT Weekly ZFS health and memory diagnostic report # # ── MEDIA ────────────────────────────────────────────────────────────────────────────────── # MEDIA PERMISSIONS Share list, mode and owner for permissions script @@ -45,22 +51,16 @@ # ============================================================================================== # ━━━ Host Configuration ━━━ -# Hostnames must match Tailscale machine names exactly — case sensitive HOST1="unRAID-Gmer4Lfe" HOST2="unRAID-Jayred365" - -# SSH keys for server-to-server rsync — each server needs the other's key authorised HOST1_SSH_KEY="/root/.ssh/Gmer4Lfe-rsync-key" HOST2_SSH_KEY="/root/.ssh/Jayred365-rsync-key" # ━━━ Logging ━━━ -# true = verbose [LOG] output in scripts / false = user-facing output only ENABLE_LOGGING=true # ━━━ Notifications ━━━ -# unRAID native — configure Settings → Notification Settings for errors/warnings only NOTIFY_UNRAID=true -# Discord webhook URL — leave blank to disable DISCORD_WEBHOOK="" # ━━━ Git / Repo ━━━ @@ -74,25 +74,19 @@ # ============================================================================================== # ━━━ Rsync Defaults ━━━ -# Global fallback values — used when no profile match is found for a share. -# Shares in DAILY_SYNC_SHARES always use these globals (no profile defined). - BW_LIMIT=12500 # network speed limit KB/s - RETRY_COUNT=3 # number of retry attempts on failure - SLEEP=300 # seconds between retries - CRITICAL_CONTAINER_NAMES=() # containers to stop before rsync - DELAYED_CONTAINERS=() # containers needing delay before start - CONTAINER_DELAY=5 # seconds delay before starting delayed containers - EXCLUDE_DIRS=() # directories to exclude from transfer + BW_LIMIT=12500 + RETRY_COUNT=3 + SLEEP=300 + CRITICAL_CONTAINER_NAMES=() + DELAYED_CONTAINERS=() + CONTAINER_DELAY=5 + EXCLUDE_DIRS=() DEFAULT_RSYNC_OPTS=(-av --info=progress2 --human-readable --bwlimit="$BW_LIMIT" --delete --inplace --no-whole-file) # ━━━ Remote Health Checks ━━━ -# Abort rsync if remote rootfs exceeds this percentage. -# Protects against rsync filling rootfs when remote array is down or drives are missing. ROOTFS_WARN=75 # ━━━ Daily Sync Shares ━━━ -# Shares synced once daily by Orchestrators/daily_sync.sh -# No profile needed — all fall through to DEFAULT_RSYNC_OPTS above. DAILY_SYNC_SHARES=( /mnt/user/Anime_Movies-Old /mnt/user/Anime_Shows-Old @@ -109,23 +103,6 @@ DAILY_SYNC_SHARES=( ) # ━━━ Rsync Profile System ━━━ -# Profiles are matched by directory basename (lowercased). -# Example: /mnt/user/appdata-Failover/Arrs_Stack → profile key = arrs_stack -# -# How fallthrough works: -# - If a key exists in a profile array that value is used -# - If a key is missing the global default above is used instead -# - Shares in DAILY_SYNC_SHARES have no profile and always use globals -# -# To add a new profile: -# 1. Add a key to each array below with your chosen profile name -# 2. Call rsync.sh with a directory whose basename matches that key -# 3. Any array you omit falls back to its global default -# -# Note: PROFILE_RSYNC_OPTS does NOT inherit from DEFAULT_RSYNC_OPTS — -# list all desired options explicitly if you define a profile entry - -# SPACE-SEPARATED STRINGS declare -A PROFILE_RSYNC_OPTS=( [arrs_stack]="-av --info=progress2 --human-readable --bwlimit=$BW_LIMIT --delete --inplace" [critical-data]="-av --human-readable --bwlimit=$BW_LIMIT --delete" @@ -158,7 +135,6 @@ declare -A PROFILE_SLEEP=( [emby]=300 ) -# SPACE-SEPARATED STRINGS declare -A PROFILE_CRITICAL_CONTAINER_NAMES=( [arrs_stack]="Sonarr Lidarr Readarr Radarr Prowlarr Bazarr Pinchflat" [critical-data]="Mariadb-Authelia Redis-Authelia Lldap-Gmer4Lfe NginxProxyManager Authelia" @@ -167,7 +143,6 @@ declare -A PROFILE_CRITICAL_CONTAINER_NAMES=( [emby]="" ) -# SPACE-SEPARATED STRINGS — containers needing delay before starting declare -A PROFILE_DELAYED_CONTAINERS=( [arrs_stack]="" [critical-data]="Authelia" @@ -184,7 +159,6 @@ declare -A PROFILE_CONTAINER_DELAY=( [emby]=5 ) -# SPACE-SEPARATED STRINGS declare -A PROFILE_EXCLUDE_DIRS=( [arrs_stack]="logs *.tmp" [critical-data]="logs *.tmp" @@ -193,12 +167,59 @@ declare -A PROFILE_EXCLUDE_DIRS=( [emby]="logs *.tmp" ) +# ============================================================================================== +# ── FAILOVER ────────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# Mutual container failover between two unRAID servers. +# Each server runs failover.sh independently — no coordination between servers. +# Decisions based solely on two ping checks: remote reachable + internet reachable. +# Comment out any container or rsync job to disable without removing the entry. + + EXTERNAL_IP="8.8.8.8" + FAILOVER_CHECK_INTERVAL=120 + FAILOVER_HANDBACK_STRIKES=2 + FAILOVER_STATE_FILE="/boot/config/failover_state.db" + +# Containers HOST1 starts locally when HOST2 goes down +FAILOVER_HOST1_STARTS_FOR_HOST2=( + "Vaultwarden-Jayred365" + "Nextcloud-Jayred365" + "Cloudflare-DDNS-Jayred365" +) + +# Containers HOST1 stops when it loses internet +FAILOVER_HOST1_STOP_ON_NO_NET=( + "Emby" + "Cloudflare-DDNS-Gmer4Lfe" +) + +# Rsync jobs HOST1 runs before handing containers back to HOST2 +FAILOVER_HOST1_RSYNC_JOBS=( + "/mnt/user/appdata-Failover/Jayred365" + "/mnt/user/Media_Server/Emby-Jayred" +) + +# Containers HOST2 starts locally when HOST1 goes down +FAILOVER_HOST2_STARTS_FOR_HOST1=( + "Emby" + "Cloudflare-DDNS-Gmer4Lfe" +) + +# Containers HOST2 stops when it loses internet +FAILOVER_HOST2_STOP_ON_NO_NET=( + "Cloudflare-DDNS-Jayred365" +) + +# Rsync jobs HOST2 runs before handing containers back to HOST1 +FAILOVER_HOST2_RSYNC_JOBS=( + "/mnt/user/appdata-Failover/Gmer4Lfe" +) + # ============================================================================================== # ── DOCKER ESSENTIALS ───────────────────────────────────────────────────────────────────────── # ============================================================================================== # ━━━ Docker Daily Restart ━━━ -# Containers restarted every day — case-sensitive names DAILY_RESTART_CONTAINERS=( "NginxProxyManager" "Authelia" @@ -209,7 +230,6 @@ DAILY_RESTART_CONTAINERS=( ) # ━━━ Docker Weekly Restart ━━━ -# Containers restarted once per week — case-sensitive names WEEKLY_RESTART_CONTAINERS=( "NginxProxyManager" "Authelia" @@ -220,11 +240,6 @@ WEEKLY_RESTART_CONTAINERS=( ) # ━━━ Docker Watchdog ━━━ -# First line of defense — monitors and restarts unhealthy containers. -# Runs on a cron schedule (recommended every 15 minutes). -# Strike system prevents restarts on brief spikes. - -# Containers to monitor with memory hard limits in MB # 20GB=20480 16GB=16384 14GB=14336 12GB=12288 10GB=10240 # 8GB=8192 6GB=6144 4GB=4096 2GB=2048 1GB=1024 declare -A WATCHDOG_CONTAINERS=( @@ -235,44 +250,47 @@ declare -A WATCHDOG_CONTAINERS=( ["Code-Server"]=1024 ) -# Containers to check HTTP responsiveness — omit to skip declare -A WATCHDOG_CONTAINER_URLS=( ["Emby"]="http://localhost:8096" ["Jellyfin-Gmer4Lfe"]="http://localhost:8095" ) -# Containers that should always be running — monitored for unexpected stops -# Strike system used — persistent skip list prevents reboot loops WATCHDOG_REQUIRED_CONTAINERS=( "NginxProxyManager" "Authelia" "Emby" ) -# Strike state file — /tmp resets on reboot, correct for strike tracking WATCHDOG_STATE_FILE="/tmp/container_watchdog_state.db" + SOFT_CPU_THRESHOLD=80 + HARD_CPU_THRESHOLD=85 + CPU_FAIL_LIMIT=2 + SOFT_MEM_THRESHOLD=80 + RESP_FAIL_LIMIT=2 + CURL_TIMEOUT=5 -# CPU thresholds — normalised against total core count at runtime - SOFT_CPU_THRESHOLD=80 # warn at this % of total system CPU - HARD_CPU_THRESHOLD=85 # strike at this % of total system CPU - CPU_FAIL_LIMIT=2 # consecutive strikes before restart +# ━━━ Docker Network Connect ━━━ +# Containers connected to extra networks on array start — many-to-many +# Every container connects to every network listed +# Comment out entries to disable without removing them +NETWORK_CONNECT_CONTAINERS=( + "memcached" + "Npm-CrowdSec" +) -# Memory threshold - SOFT_MEM_THRESHOLD=80 # warn at this % of per-container hard limit - -# Responsiveness check - RESP_FAIL_LIMIT=2 # consecutive failures before restart - CURL_TIMEOUT=5 # seconds before curl gives up +NETWORK_CONNECT_NETWORKS=( + "nextcloud-aio" +) # ============================================================================================== # ── UNRAID ESSENTIALS ───────────────────────────────────────────────────────────────────────── # ============================================================================================== # ━━━ Reboot ━━━ - REBOOT_SLEEP=300 # user warning delay before scheduled reboot (seconds) + REBOOT_SLEEP=300 # ━━━ Mover ━━━ - MOVER_STOP_TIMEOUT=300 # timeout before stopping the mover (seconds) + MOVER_STOP_TIMEOUT=300 # ━━━ Syslog Filter ━━━ FILTER_FILE="/etc/rsyslog.d/ignore-docker-veth.conf" @@ -284,6 +302,21 @@ WATCHDOG_REQUIRED_CONTAINERS=( # ━━━ Clear Logs ━━━ LOG_FILES=(/var/log/syslog /var/log/messages /var/log/dmesg) +# ━━━ WebGUI Watchdog ━━━ +# Monitors unRAID WebGUI — escalates from nginx restart to emhttp restart if needed + WEBGUI_URL="http://localhost" # adjust port if running non-standard e.g. http://localhost:8080 + WEBGUI_TIMEOUT=5 # seconds before curl gives up + WEBGUI_NGINX_WAIT=15 # seconds to wait after nginx restart before recheck + WEBGUI_EMHTTP_WAIT=30 # seconds to wait after emhttp restart before recheck + +# ━━━ ZFS Memory Snapshot ━━━ +# Weekly ZFS health and memory diagnostic report + ZFS_REPORT_LOG="/var/log/zfs-weekly-health.log" + ZFS_REPORT_ARC_WARN_PCT=90 # warn if ARC utilization above this % + ZFS_REPORT_FREE_WARN_GB=10 # warn if free RAM below this GB + ZFS_REPORT_AVAIL_WARN_GB=20 # warn if available RAM below this GB + ZFS_REPORT_DOCKER_TOP=10 # number of top Docker memory users to show + # ============================================================================================== # ── MEDIA ───────────────────────────────────────────────────────────────────────────────────── # ============================================================================================== @@ -318,9 +351,6 @@ MEDIA_PERMISSION_SHARES=( ) # ━━━ Media Cleaner ━━━ -# Two profiles: anime and media — passed as argument to media_cleaner.sh -# Usage: media_cleaner.sh anime or media_cleaner.sh media - ANIME_CLEAN_FOLDERS=( /mnt/user/Anime_Movies /mnt/user/Anime_Movies-Old @@ -355,70 +385,38 @@ MEDIA_FILE_PATTERNS=( # ============================================================================================== # ── TRANSCODES ──────────────────────────────────────────────────────────────────────────────── # ============================================================================================== -# Session-based storage allocator using filesystem indirection. -# New transcode sessions land wherever TRANSCODE_LINK points. -# Existing sessions are never interrupted — ffmpeg resolves path once at session start. -# -# Flow: -# ramdisk_setup.sh — run once at array start, creates ramdisk and symlink -# transcode_manager.sh — runs every 2-3 min, monitors usage and flips symlink -# transcode_cleanup.sh — runs every 5 min, removes old inactive files -# -# Hysteresis gap between RAMDISK_WARN_GB and RAMDISK_LOW_GB prevents flip-flop -# when usage hovers near the threshold. Gap should be at least 0.5-1GB. -# ━━━ Transcode Manager ━━━ -# Paths - RAMDISK_PATH="/mnt/ramdisk_transcodes" # tmpfs mount point - RAMDISK_SIZE="8G" # increase if you have RAM headroom - TRANSCODE_LINK="/mnt/ram-transcode" # symlink Emby points at - TRANSCODE_SSD="/mnt/cache/Temp_Storage/Emby/Transcodes/" # SSD fallback location - -# Thresholds in GB - RAMDISK_WARN_GB=6.8 # flip symlink to SSD at or above this usage - RAMDISK_LOW_GB=5.5 # flip symlink back to ramdisk when usage drops here - RAMDISK_SSD_MIN_GB=20 # minimum free GB on SSD before allowing flip — abort if below - -# Cleanup settings - TRANSCODE_MAX_AGE=20 # minutes before a file is eligible for cleanup - TRANSCODE_ORPHAN_AGE=30 # minutes before an orphaned file is eligible (slightly longer buffer) - -# Flip frequency monitoring - TRANSCODE_FLIP_WARN=3 # notify if symlink flips this many times in one hour - -# Permissions — should match your Emby container user + RAMDISK_PATH="/mnt/ramdisk_transcodes" + RAMDISK_SIZE="8G" + TRANSCODE_LINK="/mnt/ram-transcode" + TRANSCODE_SSD="/mnt/cache/Temp_Storage/Emby/Transcodes/" + RAMDISK_WARN_GB=6.8 + RAMDISK_LOW_GB=5.5 + RAMDISK_SSD_MIN_GB=20 + TRANSCODE_MAX_AGE=20 + TRANSCODE_ORPHAN_AGE=30 + TRANSCODE_FLIP_WARN=3 TRANSCODE_OWNER="nobody:users" TRANSCODE_MODE="755" # ============================================================================================== # ── SYSTEM WATCHDOG ─────────────────────────────────────────────────────────────────────────── # ============================================================================================== -# Last line of defense — reboots the system cleanly if it is about to become unstable. -# Works alongside docker_watchdog.sh — containers first, system second. -# Thresholds set at "about to fall over" levels — not just high usage. -# ━━━ System Watchdog State Files ━━━ SYS_WATCHDOG_STATE_FILE="/tmp/system_watchdog_state.db" SYS_WATCHDOG_FAILED_FILE="/boot/config/system_watchdog_failed.db" SYS_WATCHDOG_REBOOT_LOG="/boot/config/system_watchdog_reboots.db" - -# ━━━ Strike and Reboot Loop Settings ━━━ - SYS_WATCHDOG_STRIKE_LIMIT=2 # consecutive hits before reboot trigger - SYS_WATCHDOG_REBOOT_LIMIT=3 # max reboots in window before shutdown instead - SYS_WATCHDOG_REBOOT_WINDOW_HRS=12 # window in hours — controls reboot count AND log purge - -# ━━━ Thresholds ━━━ - SYS_WATCHDOG_ROOTFS_PCT=95 # rootfs % before strike - SYS_WATCHDOG_LOG_PCT=95 # /var/log % before strike - SYS_WATCHDOG_MEM_GB=4 # free RAM GB below which strikes (128GB system) - SYS_WATCHDOG_ARC_PINNED_PCT=98 # ZFS ARC % of max before reclaim attempt - SYS_WATCHDOG_ARC_RELEASE_PCT=95 # ZFS ARC % after reclaim that still triggers - SYS_WATCHDOG_LOAD_MULTIPLIER=3 # strike if load avg > cores x this value - SYS_WATCHDOG_ZOMBIE_LIMIT=50 # strike if zombie count exceeds this - SYS_WATCHDOG_CPU_TEMP_MAX=95 # degrees C — adjust for your CPU tjmax - -# ━━━ Check Toggles ━━━ -# true = run this check / false = skip entirely + SYS_WATCHDOG_STRIKE_LIMIT=2 + SYS_WATCHDOG_REBOOT_LIMIT=3 + SYS_WATCHDOG_REBOOT_WINDOW_HRS=12 + SYS_WATCHDOG_ROOTFS_PCT=95 + SYS_WATCHDOG_LOG_PCT=95 + SYS_WATCHDOG_MEM_GB=4 + SYS_WATCHDOG_ARC_PINNED_PCT=98 + SYS_WATCHDOG_ARC_RELEASE_PCT=95 + SYS_WATCHDOG_LOAD_MULTIPLIER=3 + SYS_WATCHDOG_ZOMBIE_LIMIT=50 + SYS_WATCHDOG_CPU_TEMP_MAX=95 SYS_WATCHDOG_CHECK_ROOTFS=true SYS_WATCHDOG_CHECK_LOG=true SYS_WATCHDOG_CHECK_RAM=true @@ -428,10 +426,6 @@ MEDIA_FILE_PATTERNS=( SYS_WATCHDOG_CHECK_ZOMBIES=true SYS_WATCHDOG_CHECK_CONTAINERS=true SYS_WATCHDOG_CHECK_DOCKER_DAEMON=true - -# ━━━ Abort Toggles ━━━ -# true = abort reboot if condition active / false = reboot anyway -# Default true = conservative — set false only when you want "reboot no matter what" SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY=true SYS_WATCHDOG_ABORT_ON_PARITY=true SYS_WATCHDOG_ABORT_ON_MOVER=true diff --git a/common.sh b/common.sh index 9962780..f358b7c 100644 --- a/common.sh +++ b/common.sh @@ -2,7 +2,7 @@ # ----------------------------------------------------------------------------------------------- # ----------------- UNRAID OPS COMMON LIBRARY (STABLE FRAMEWORK v1) ---------------------------- # ----------------------------------------------------------------------------------------------- -# Version: 2.4 +# Version: 2.7 # ----------------------------------------------------------------------------------------------- # Changelog: # v1.0 — Initial stable framework @@ -40,6 +40,17 @@ # v2.4 — ICON_CLEAN, ICON_TRASH added for media cleaner operations # ICON_PERMS, ICON_UNLOCKED added for media permissions operations # ICON_REBOOT_SMART added for smart conditional reboot +# v2.5 — ICON_RAM added for ramdisk operations +# ICON_LINK added for symlink state and management +# Transcode scripts group added to ecosystem +# v2.6 — ICON_FAILOVER added for failover operations +# check_local_array added — verifies local /mnt/user is mounted and healthy +# check_remote_array added — verifies remote /mnt/user is mounted and healthy +# check_remote_docker added — verifies remote Docker daemon is responding +# ping_remote added — non-fatal ping returning status for failover use +# ping_internet added — non-fatal external ping for failover use +# v2.7 — ICON_WEBGUI added for WebGUI watchdog operations +# ICON_DOCKER_NET added for Docker network connect operations # ----------------------------------------------------------------------------------------------- # ----------------------------------------------------------------------------------------------- @@ -59,13 +70,13 @@ ICON_HEALTH="🩺" # rootfs / share health checks ICON_SHIELD="🛡️" # pre-flight section header # Containers -ICON_CONTAINERS="📦" # container section anchor — paired with action icon for direction +ICON_CONTAINERS="📦" # container section anchor ICON_STOP="⛔" # stop command being issued ICON_STOPPED="🔴" # container confirmed stopped ICON_START="▶️" # start command being issued ICON_STARTED="💚" # container confirmed started ICON_RUNNING="🟢" # container already running when checked -ICON_NOT_RUNNING="⭕" # container already stopped when checked — distinct from ICON_STOPPED +ICON_NOT_RUNNING="⭕" # container already stopped when checked # Transfer ICON_SYNC="🔄" # transfer section header @@ -83,6 +94,7 @@ ICON_REBOOT="⚡" # scheduled server reboot ICON_REBOOT_SMART="🚨" # smart conditional reboot triggered ICON_PLUGIN="🧩" # user scripts plugin operations ICON_PHP="👥" # PHP-FPM operations +ICON_WEBGUI="💻" # WebGUI / nginx / emhttp operations # Diagnostics ICON_ZFS="📊" # ZFS ARC statistics @@ -95,6 +107,16 @@ ICON_TRASH="🗑️" # files being deleted ICON_PERMS="🔐" # permissions operation / section header ICON_UNLOCKED="🔓" # permissions successfully applied to a share +# Transcode Operations +ICON_RAM="💨" # ramdisk operations — fast ephemeral storage +ICON_LINK="🔗" # symlink state and management + +# Failover Operations +ICON_FAILOVER="🔀" # failover state changes and operations + +# Docker Network Operations +ICON_DOCKER_NET="🔌" # Docker network connect operations + # Notifications ICON_NOTIFY="🔔" # notification operations @@ -106,8 +128,6 @@ ICON_SUCCESS="✅" # ----------------------------------------------------------------------------------------------- # OUTPUT HELPERS -# Standardised output functions used across all scripts. -# log() is gated by ENABLE_LOGGING — set in Master.conf or via --log flag. # ----------------------------------------------------------------------------------------------- info() { echo "$ICON_INFO [INFO] $*"; } warn() { echo "$ICON_WARN [WARN] $*"; } @@ -120,14 +140,8 @@ log() { # ----------------------------------------------------------------------------------------------- # NOTIFICATION -# Sends a notification via unRAID native system and/or Discord webhook. -# Both channels are optional and independently controlled via Master.conf. -# unRAID native: requires NOTIFY_UNRAID=true and the dynamix notify script to be present. -# Discord: requires DISCORD_WEBHOOK to be set to a valid webhook URL. -# Severity levels: normal, warning, alert — maps to unRAID notification severity. # Usage: notify "message" "subject" "severity" -# notify "Rsync failed: Movies" "Rsync Alert" "warning" -# notify "Daily sync complete" "Daily Sync" "normal" +# Severity: normal, warning, alert # ----------------------------------------------------------------------------------------------- notify() { local message="$1" @@ -136,7 +150,6 @@ notify() { log "$ICON_NOTIFY Sending notification: $subject — $message" - # unRAID native notification if [[ "${NOTIFY_UNRAID:-false}" == true ]]; then local notify_script="/usr/local/emhttp/plugins/dynamix/scripts/notify" if [[ -x "$notify_script" ]]; then @@ -147,7 +160,6 @@ notify() { fi fi - # Discord webhook notification if [[ -n "${DISCORD_WEBHOOK:-}" ]]; then local payload payload=$(printf '{"content": "%s — **%s**\\n%s"}' \ @@ -163,9 +175,6 @@ notify() { # ----------------------------------------------------------------------------------------------- # DURATION FORMATTER -# Converts raw seconds into a human readable string — e.g. 10m53s or 47s -# Used by rsync.sh summary and daily_sync.sh summary. -# Sourced from common.sh so both scripts share the same implementation. # ----------------------------------------------------------------------------------------------- format_duration() { local secs=$1 @@ -176,24 +185,17 @@ format_duration() { # ----------------------------------------------------------------------------------------------- # ARG PARSER -# Processes all flags and key=value pairs passed to any script. -# Positional arguments (directory paths) are separated before calling this — see rsync.sh. -# Supported flags: --dry-run, --log, --no-log, --status, --help -# Supported key=value: LOG=true/false, or any declared variable e.g. BW_LIMIT=5000 -# Unparsed positional args are returned in PARSED_ARGS array. # ----------------------------------------------------------------------------------------------- parse_args() { ENABLE_LOGGING=${ENABLE_LOGGING:-false} DRY_RUN=${DRY_RUN:-false} SHOW_STATUS=${SHOW_STATUS:-false} - CLEAN_ARGS=() for ARG in "$@"; do if [[ "$ARG" == *=* ]]; then VAR="${ARG%%=*}" VAL="${ARG#*=}" - case "$VAR" in LOG) [[ "$VAL" == "true" ]] && ENABLE_LOGGING=true @@ -215,7 +217,7 @@ parse_args() { --no-log) ENABLE_LOGGING=false ;; --status|--summary) SHOW_STATUS=true ;; --help|-h) - echo "Usage: script [--dry-run] [--log] [--status]" + echo "Usage: script [--dry-run] [--log] [--status]" exit 0 ;; *) CLEAN_ARGS+=("$ARG") ;; @@ -228,43 +230,26 @@ parse_args() { # ----------------------------------------------------------------------------------------------- # VALIDATION -# Checks that a required variable is set and non-empty. -# Usage: require_var VAR_NAME -# Exits with error if the variable is missing. # ----------------------------------------------------------------------------------------------- require_var() { [[ -z "${!1:-}" ]] && error "Missing required: $1" && exit 1 } -# ----------------------------------------------------------------------------------------------- -# INTEGER VALIDATION -# Checks that a variable contains a valid positive integer. -# Exits with a clear error if the value is missing, empty, or not a number. -# Usage: validate_int VAR_NAME "$VAR_VALUE" -# Example: validate_int MOVER_STOP_TIMEOUT "$MOVER_STOP_TIMEOUT" -# ----------------------------------------------------------------------------------------------- validate_int() { - local name="$1" - local value="$2" - + local name="$1" value="$2" if [[ -z "$value" ]]; then error "$name is not set — check Master.conf" exit 1 fi - if ! [[ "$value" =~ ^[0-9]+$ ]]; then error "$name must be a positive integer — got: '$value'" exit 1 fi - log "$name validated: $value" } # ----------------------------------------------------------------------------------------------- # HOST DETECTION -# Determines local and remote server names by comparing hostname against HOST1/HOST2. -# Sets LOCAL_SERVER_NAME, REMOTE_SERVER_NAME, and SSH_KEY for the current run direction. -# Both HOST1 and HOST2 must be defined in Master.conf. # ----------------------------------------------------------------------------------------------- detect_hosts() { LOCAL_HOSTNAME="$(hostname)" @@ -283,34 +268,25 @@ detect_hosts() { declare -A SSH_KEYS SSH_KEYS["$HOST1|$HOST2"]="$HOST1_SSH_KEY" SSH_KEYS["$HOST2|$HOST1"]="$HOST2_SSH_KEY" - SSH_KEY="${SSH_KEYS[$LOCAL_SERVER_NAME|$REMOTE_SERVER_NAME]}" - [[ -z "$SSH_KEY" ]] && error "Missing SSH key mapping for $LOCAL_SERVER_NAME → $REMOTE_SERVER_NAME" && exit 1 + [[ -z "$SSH_KEY" ]] && error "Missing SSH key mapping" && exit 1 info "$ICON_HOST Host: $LOCAL_SERVER_NAME → $REMOTE_SERVER_NAME" } # ----------------------------------------------------------------------------------------------- # REMOTE IP RESOLUTION -# Resolves the Tailscale IPv4 address of the remote server. -# Sets REMOTE_SERVER used by all subsequent SSH and rsync calls. -# Exits if resolution fails — likely means Tailscale is down or peer is offline. # ----------------------------------------------------------------------------------------------- resolve_remote_ip() { log "Resolving remote IP for $REMOTE_SERVER_NAME..." REMOTE_SERVER=$(tailscale ip -4 "$REMOTE_SERVER_NAME" 2>/dev/null) - [[ -z "$REMOTE_SERVER" ]] && error "Failed to resolve Tailscale IP for $REMOTE_SERVER_NAME" && exit 1 - info "$ICON_NET Remote IP: $REMOTE_SERVER" } # ----------------------------------------------------------------------------------------------- -# CONNECTIVITY CHECK -# Pings the remote server to confirm it is reachable before starting any transfers. -# Prevents the rsync retry loop from burning all attempts against an unreachable host. -# If ping fails, prints a tailscale status hint to aid diagnosis before exiting. +# CONNECTIVITY CHECK — fatal, used by rsync scripts # ----------------------------------------------------------------------------------------------- check_connectivity() { log "Checking connectivity to $REMOTE_SERVER..." @@ -323,16 +299,78 @@ check_connectivity() { } # ----------------------------------------------------------------------------------------------- -# REMOTE ROOTFS SPACE CHECK -# Checks the remote server's rootfs usage before any rsync runs. -# If the array is down or drives are missing, rsync writes land on rootfs instead of the array — -# this can fill the remote filesystem rapidly and crash the server. -# Threshold is set by ROOTFS_WARN in Master.conf (recommended: 75). -# Aborts cleanly with a clear error showing current usage vs threshold. +# PING REMOTE — non-fatal, used by failover +# Returns 0 if reachable, 1 if not +# ----------------------------------------------------------------------------------------------- +ping_remote() { + ping -c2 -W3 "$REMOTE_SERVER" &>/dev/null +} + +# ----------------------------------------------------------------------------------------------- +# PING INTERNET — non-fatal, used by failover +# Returns 0 if internet reachable, 1 if not +# ----------------------------------------------------------------------------------------------- +ping_internet() { + ping -c2 -W3 "${EXTERNAL_IP:-8.8.8.8}" &>/dev/null +} + +# ----------------------------------------------------------------------------------------------- +# LOCAL ARRAY CHECK — non-fatal, returns status +# Used by failover before starting remote containers locally +# ----------------------------------------------------------------------------------------------- +check_local_array() { + log "Checking local array..." + if ! mountpoint -q /mnt/user 2>/dev/null; then + error "$ICON_DISK Local array is not started — /mnt/user is not mounted" + return 1 + fi + local file_count + file_count=$(ls /mnt/user 2>/dev/null | wc -l) + if [[ "$file_count" -eq 0 ]]; then + error "$ICON_DISK Local array appears empty — shares may not be available" + return 1 + fi + log "Local array is healthy" + return 0 +} + +# ----------------------------------------------------------------------------------------------- +# REMOTE ARRAY CHECK — non-fatal, returns status +# Used before handback rsync +# ----------------------------------------------------------------------------------------------- +check_remote_array() { + log "Checking remote array on $REMOTE_SERVER_NAME..." + local result + result=$(ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \ + "mountpoint -q /mnt/user && echo yes || echo no" 2>/dev/null) + if [[ "$result" != "yes" ]]; then + error "$ICON_DISK Remote array not started on $REMOTE_SERVER_NAME" + return 1 + fi + log "Remote array is healthy" + return 0 +} + +# ----------------------------------------------------------------------------------------------- +# REMOTE DOCKER CHECK — non-fatal, returns status +# Used before starting containers on remote +# ----------------------------------------------------------------------------------------------- +check_remote_docker() { + log "Checking remote Docker daemon on $REMOTE_SERVER_NAME..." + if ! ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \ + "timeout 10 docker ps" >/dev/null 2>&1; then + error "$ICON_CONTAINERS Remote Docker daemon not responding on $REMOTE_SERVER_NAME" + return 1 + fi + log "Remote Docker daemon is healthy" + return 0 +} + +# ----------------------------------------------------------------------------------------------- +# REMOTE ROOTFS SPACE CHECK — fatal # ----------------------------------------------------------------------------------------------- check_remote_rootfs() { log "Checking remote rootfs usage..." - REMOTE_USAGE=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ "df / --output=pcent | tail -1 | tr -d ' %'" 2>/dev/null) @@ -342,39 +380,27 @@ check_remote_rootfs() { fi if [[ "$REMOTE_USAGE" -ge "${ROOTFS_WARN:-75}" ]]; then - echo "" - error "$ICON_HEALTH Remote rootfs is ${REMOTE_USAGE}% full — threshold is ${ROOTFS_WARN:-75}%" + error "$ICON_HEALTH Remote rootfs ${REMOTE_USAGE}% — threshold ${ROOTFS_WARN:-75}%" warn "Array may be down or drives missing on $REMOTE_SERVER_NAME" - info "Hint: Check array status on $REMOTE_SERVER_NAME before retrying" - echo "" exit 1 fi - info "$ICON_HEALTH Remote rootfs: ${REMOTE_USAGE}% used (threshold: ${ROOTFS_WARN:-75}%)" + info "$ICON_HEALTH Remote rootfs: ${REMOTE_USAGE}% (threshold: ${ROOTFS_WARN:-75}%)" } # ----------------------------------------------------------------------------------------------- -# REMOTE SHARE VALIDATION -# Verifies that the target directory exists and is not empty on the remote server. -# Catches the scenario where the array is mounted but drives are not backing the share — -# the path exists as an empty mountpoint, which would cause --delete to wipe the remote. -# Called with the specific directory being synced so each share is checked individually. +# REMOTE SHARE VALIDATION — fatal # Usage: check_remote_share "/mnt/user/Movies" # ----------------------------------------------------------------------------------------------- check_remote_share() { local dir="$1" - log "Checking remote share: $dir..." SHARE_EXISTS=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ "[[ -d '$dir' ]] && echo yes || echo no" 2>/dev/null) if [[ "$SHARE_EXISTS" != "yes" ]]; then - echo "" error "$ICON_HEALTH Remote share does not exist: $dir" - warn "Array may not be started or share is not configured on $REMOTE_SERVER_NAME" - info "Hint: Check shares and array status on $REMOTE_SERVER_NAME before retrying" - echo "" exit 1 fi @@ -382,11 +408,7 @@ check_remote_share() { "[[ -z \"\$(ls -A '$dir' 2>/dev/null)\" ]] && echo yes || echo no" 2>/dev/null) if [[ "$SHARE_EMPTY" == "yes" ]]; then - echo "" - warn "$ICON_HEALTH Remote share exists but is empty: $dir" - warn "Drives may not be mounted on $REMOTE_SERVER_NAME — aborting to protect data" - info "Hint: Verify array and drive assignments on $REMOTE_SERVER_NAME before retrying" - echo "" + warn "$ICON_HEALTH Remote share exists but is empty: $dir — aborting to protect data" exit 1 fi @@ -394,12 +416,7 @@ check_remote_share() { } # ----------------------------------------------------------------------------------------------- -# REMOTE DISK CHECK -# Verifies that all physical disks backing a share are online and mounted on the remote server. -# Discovers disk layout automatically at runtime by finding all /mnt/diskN/sharename paths — -# no configuration required, works for any share regardless of how many disks it spans. -# Aborts if any single disk backing the share is offline — partial disk failure means -# incomplete data which could result in files being deleted by --delete during sync. +# REMOTE DISK CHECK — fatal # Usage: check_remote_disks "/mnt/user/Movies" # ----------------------------------------------------------------------------------------------- check_remote_disks() { @@ -413,20 +430,14 @@ check_remote_disks() { "ls -d /mnt/disk*/$share_name 2>/dev/null" 2>/dev/null) if [[ -z "$DISK_PATHS" ]]; then - echo "" - error "$ICON_DISK No disks found backing share $share_name on $REMOTE_SERVER_NAME" - warn "Share may not exist on any disk or array may not be started" - info "Hint: Check array and share configuration on $REMOTE_SERVER_NAME" - echo "" + error "$ICON_DISK No disks found backing $share_name on $REMOTE_SERVER_NAME" exit 1 fi local all_ok=true - while IFS= read -r disk_share_path; do - local disk_mount + local disk_mount disk_name disk_mount=$(dirname "$disk_share_path") - local disk_name disk_name=$(basename "$disk_mount") MOUNTED=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ @@ -435,17 +446,13 @@ check_remote_disks() { if [[ "$MOUNTED" == "yes" ]]; then info "$ICON_DISK $disk_name $ICON_RUNNING — $share_name present" else - error "$ICON_DISK $disk_name $ICON_STOPPED — $share_name missing or incomplete" + error "$ICON_DISK $disk_name $ICON_STOPPED — $share_name missing" all_ok=false fi done <<< "$DISK_PATHS" if [[ "$all_ok" == false ]]; then - echo "" error "One or more disks backing $share_name are offline on $REMOTE_SERVER_NAME" - warn "Aborting to prevent partial or destructive sync" - info "Hint: Check disk assignments and array status on $REMOTE_SERVER_NAME before retrying" - echo "" exit 1 fi @@ -453,11 +460,7 @@ check_remote_disks() { } # ----------------------------------------------------------------------------------------------- -# CONTAINER MANAGEMENT — STOP -# Stops all containers listed in CRITICAL_CONTAINER_NAMES on the remote server. -# Only stops containers that are currently running — skips those already stopped. -# Tracks stopped containers in RUNNING_CONTAINERS for restart after rsync completes. -# CRITICAL_CONTAINER_NAMES must be a bash array — rsync.sh handles conversion from profile strings. +# CONTAINER MANAGEMENT — STOP (remote via SSH) # ----------------------------------------------------------------------------------------------- RUNNING_CONTAINERS=() @@ -473,16 +476,12 @@ stop_containers() { for c in "${CRITICAL_CONTAINER_NAMES[@]}"; do [[ -z "$c" ]] && continue - - info "Checking $c..." - STATUS=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ "docker inspect -f '{{.State.Running}}' $c 2>/dev/null" 2>/dev/null || echo "false") if [[ "$STATUS" == "true" ]]; then echo "$ICON_STOP Stopping $c..." RUNNING_CONTAINERS+=("$c") - if ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" "docker stop $c" >/dev/null; then echo "$ICON_STOPPED $c stopped" else @@ -495,11 +494,7 @@ stop_containers() { } # ----------------------------------------------------------------------------------------------- -# CONTAINER MANAGEMENT — START -# Restarts only the containers that were running before rsync and were stopped by stop_containers. -# Containers listed in DELAYED_CONTAINERS receive a sleep of CONTAINER_DELAY seconds before -# starting — useful for dependencies like Authelia that need upstream services ready first. -# DELAYED_CONTAINERS must be a bash array — rsync.sh handles conversion from profile strings. +# CONTAINER MANAGEMENT — START (remote via SSH) # ----------------------------------------------------------------------------------------------- start_containers() { if [[ ${#RUNNING_CONTAINERS[@]} -eq 0 ]]; then @@ -514,10 +509,7 @@ start_containers() { local needs_delay=false for d in "${DELAYED_CONTAINERS[@]}"; do - if [[ "$c" == "$d" ]]; then - needs_delay=true - break - fi + [[ "$c" == "$d" ]] && needs_delay=true && break done if [[ "$needs_delay" == true ]]; then @@ -536,10 +528,6 @@ start_containers() { # ----------------------------------------------------------------------------------------------- # RSYNC OPTIONS -# Loads rsync options for the current profile from PROFILE_RSYNC_OPTS in Master.conf. -# If no profile match is found, falls back to DEFAULT_RSYNC_OPTS. -# Note: profile opts do NOT inherit from defaults — all desired flags must be listed explicitly. -# Sets RSYNC_OPTS array used directly in the rsync call in rsync.sh. # ----------------------------------------------------------------------------------------------- get_rsync_opts() { if [[ -n "${PROFILE_RSYNC_OPTS[$PROFILE_NAME]:-}" ]]; then @@ -553,20 +541,17 @@ get_rsync_opts() { # ----------------------------------------------------------------------------------------------- # STATUS DISPLAY -# Prints a summary of the current runtime configuration. -# Triggered by --status or --summary flag passed to any script. -# Useful for verifying profile resolution and variable state before a live run. # ----------------------------------------------------------------------------------------------- show_status() { echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" echo "Local: $LOCAL_SERVER_NAME" echo "Remote: $REMOTE_SERVER_NAME" echo "IP: $REMOTE_SERVER" - echo "Profile: $PROFILE_NAME" + echo "Profile: ${PROFILE_NAME:-n/a}" echo "DryRun: $DRY_RUN" echo "Logging: $ENABLE_LOGGING" - echo "Containers: ${CRITICAL_CONTAINER_NAMES[*]}" - echo "Delayed: ${DELAYED_CONTAINERS[*]}" - echo "Excludes: ${EXCLUDE_DIRS[*]}" + echo "Containers: ${CRITICAL_CONTAINER_NAMES[*]:-n/a}" + echo "Delayed: ${DELAYED_CONTAINERS[*]:-n/a}" + echo "Excludes: ${EXCLUDE_DIRS[*]:-n/a}" echo "━━━━━━━━━━━━━━━━━━━━━━━" } \ No newline at end of file diff --git a/unRAID_Essentials/webgui_restart.sh b/unRAID_Essentials/webgui_restart.sh new file mode 100644 index 0000000..617a823 --- /dev/null +++ b/unRAID_Essentials/webgui_restart.sh @@ -0,0 +1,198 @@ +#!/bin/bash +# ----------------------------------------------------------------------------------------------- +# --------------------------------- WebGUI Watchdog -------------------------------------------- +# ----------------------------------------------------------------------------------------------- +# Monitors unRAID's WebGUI and restarts it if unresponsive. +# Uses an escalating restart strategy — tries nginx first, then emhttp if needed. +# emhttp is the unRAID management daemon — restarting it is more disruptive than nginx +# but recovers cleanly. Notification sent on any restart so you know what happened. +# +# Escalation path: +# Check WebGUI → unresponsive → restart nginx → recheck +# Still unresponsive → restart emhttp → recheck +# Still unresponsive → notify warning, manual intervention needed +# +# Run every 5-10 minutes via cron/User Scripts plugin. +# All configuration in Master.conf under WebGUI Watchdog section. +# Supports --dry-run to show what would be restarted without acting. +# ----------------------------------------------------------------------------------------------- + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "$SCRIPT_DIR/../Master.conf" +source "$SCRIPT_DIR/../common.sh" + +parse_args "$@" + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_GEAR Setup ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_GEAR Setup ━━━" + +if [[ "$EUID" -ne 0 ]]; then + error "Must be run as root" + exit 1 +fi + +success "Running as root" + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_SUMMARY Status ━━━ +# ----------------------------------------------------------------------------------------------- +if [[ "$SHOW_STATUS" == true ]]; then + echo "" + echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" + echo "$ICON_WEBGUI URL: $WEBGUI_URL" + echo "$ICON_WEBGUI Curl timeout: ${WEBGUI_TIMEOUT}s" + echo "$ICON_WEBGUI Nginx wait: ${WEBGUI_NGINX_WAIT}s" + echo "$ICON_WEBGUI emhttp wait: ${WEBGUI_EMHTTP_WAIT}s" + echo "$ICON_GEAR Dry Run: $DRY_RUN" + + # Show current state + if curl -sf --max-time "$WEBGUI_TIMEOUT" "$WEBGUI_URL" >/dev/null 2>&1; then + echo "$ICON_WEBGUI WebGUI: $ICON_RUNNING responding" + else + echo "$ICON_WEBGUI WebGUI: $ICON_NOT_RUNNING not responding" + fi + echo "━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no services will be restarted" + +# ----------------------------------------------------------------------------------------------- +# FUNCTIONS +# ----------------------------------------------------------------------------------------------- + +# Check if WebGUI is responding +check_webgui() { + curl -sf --max-time "$WEBGUI_TIMEOUT" "$WEBGUI_URL" >/dev/null 2>&1 +} + +# Restart nginx — lightweight fix, try first +restart_nginx() { + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would restart nginx" + return 0 + fi + + info "$ICON_WEBGUI Restarting nginx..." + if /etc/rc.d/rc.nginx restart >/dev/null 2>&1; then + success "nginx restarted" + return 0 + else + error "nginx restart failed" + return 1 + fi +} + +# Restart emhttp — heavier fix, escalate if nginx didn't help +# emhttp drives the array, Docker management, shares — recovers cleanly but takes longer +restart_emhttp() { + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would restart emhttp" + return 0 + fi + + info "$ICON_WEBGUI Restarting emhttp..." + if /etc/rc.d/rc.emhttp restart >/dev/null 2>&1; then + success "emhttp restarted" + return 0 + else + error "emhttp restart failed" + return 1 + fi +} + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_WEBGUI WebGUI Watchdog ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_WEBGUI WebGUI Watchdog — $(date '+%Y-%m-%d %H:%M:%S') ━━━" +echo "$ICON_WEBGUI URL: $WEBGUI_URL" +echo "" + +START=$(date +%s) + +# Initial check +info "Checking WebGUI..." + +if check_webgui; then + success "$ICON_WEBGUI WebGUI is responding — nothing to do" + echo "" + echo "━━━━━ $ICON_SUMMARY WEBGUI WATCHDOG SUMMARY ━━━━━" + echo "$ICON_WEBGUI Status: $ICON_RUNNING HEALTHY" + echo "$ICON_TIME Duration: $(format_duration $(($(date +%s) - START)))" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +# WebGUI not responding — begin escalation +warn "$ICON_WEBGUI WebGUI is not responding at $WEBGUI_URL" + +# ── Step 1: Restart nginx ── +echo "" +echo "━━━ $ICON_WEBGUI Step 1 — Nginx Restart ━━━" + +restart_nginx + +if [[ "$DRY_RUN" == false ]]; then + info "Waiting ${WEBGUI_NGINX_WAIT}s for nginx to recover..." + sleep "$WEBGUI_NGINX_WAIT" + + if check_webgui; then + success "$ICON_WEBGUI WebGUI recovered after nginx restart" + notify "WebGUI recovered on $(hostname) after nginx restart" "WebGUI Watchdog" "warning" + + echo "" + echo "━━━━━ $ICON_SUMMARY WEBGUI WATCHDOG SUMMARY ━━━━━" + echo "$ICON_WEBGUI Status: $ICON_SUCCESS RECOVERED via nginx restart" + echo "$ICON_TIME Duration: $(format_duration $(($(date +%s) - START)))" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 + fi + + warn "WebGUI still not responding after nginx restart — escalating to emhttp" +fi + +# ── Step 2: Restart emhttp ── +echo "" +echo "━━━ $ICON_WEBGUI Step 2 — emhttp Restart ━━━" +warn "Restarting emhttp — this is the unRAID management daemon" +warn "Array, Docker management and shares remain running but WebGUI will be briefly unavailable" + +restart_emhttp + +if [[ "$DRY_RUN" == false ]]; then + info "Waiting ${WEBGUI_EMHTTP_WAIT}s for emhttp to recover..." + sleep "$WEBGUI_EMHTTP_WAIT" + + if check_webgui; then + success "$ICON_WEBGUI WebGUI recovered after emhttp restart" + notify "WebGUI recovered on $(hostname) after emhttp restart — check system health" "WebGUI Watchdog" "warning" + + echo "" + echo "━━━━━ $ICON_SUMMARY WEBGUI WATCHDOG SUMMARY ━━━━━" + echo "$ICON_WEBGUI Status: $ICON_SUCCESS RECOVERED via emhttp restart" + echo "$ICON_TIME Duration: $(format_duration $(($(date +%s) - START)))" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 + fi +fi + +# ── Both restarts failed ── +END=$(date +%s) + +echo "" +echo "━━━━━ $ICON_SUMMARY WEBGUI WATCHDOG SUMMARY ━━━━━" +echo "$ICON_WEBGUI Status: $ICON_ERROR UNRECOVERED — manual intervention needed" +echo "$ICON_TIME Duration: $(format_duration $((END - START)))" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + +if [[ "$DRY_RUN" == false ]]; then + notify "WebGUI unrecovered on $(hostname) after nginx and emhttp restart — manual intervention needed" "WebGUI Watchdog" "warning" + exit 1 +fi + +exit 0 \ No newline at end of file diff --git a/unRAID_Essentials/zfs_memory_snapshot.sh b/unRAID_Essentials/zfs_memory_snapshot.sh index 14bd7bb..ad06df9 100644 --- a/unRAID_Essentials/zfs_memory_snapshot.sh +++ b/unRAID_Essentials/zfs_memory_snapshot.sh @@ -2,9 +2,16 @@ # ----------------------------------------------------------------------------------------------- # --------------------------------- ZFS Memory Snapshot ---------------------------------------- # ----------------------------------------------------------------------------------------------- -# Captures a point-in-time snapshot of ZFS ARC statistics and system memory usage. -# Read-only diagnostic tool — no changes are made to the system regardless of flags. -# Dry run mode still collects and displays data since no modifications occur. +# Weekly ZFS pool health and memory diagnostic report. +# Combines ZFS pool status, ARC statistics, memory summary, Docker memory usage +# and kernel pressure into a single report. Informational only — no action taken. +# system_watchdog.sh handles threshold-based intervention. +# +# Output goes to both console and ZFS_REPORT_LOG for later review. +# Notifies if any warning thresholds are exceeded. +# +# All configuration in Master.conf under ZFS Memory Snapshot section. +# Supports --dry-run (preview only, no log write) and --status. # ----------------------------------------------------------------------------------------------- SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -14,14 +21,24 @@ source "$SCRIPT_DIR/../common.sh" parse_args "$@" +# Tee output to log file unless dry run +if [[ "$DRY_RUN" == false ]]; then + mkdir -p "$(dirname "$ZFS_REPORT_LOG")" + exec > >(tee -a "$ZFS_REPORT_LOG") 2>&1 +fi + # ----------------------------------------------------------------------------------------------- # ━━━ $ICON_GEAR Setup ━━━ # ----------------------------------------------------------------------------------------------- echo "" echo "━━━ $ICON_GEAR Setup ━━━" -info "$ICON_ZFS ZFS ARC + Memory Snapshot" -info "$ICON_TIME $(date)" +if [[ "$EUID" -ne 0 ]]; then + error "Must be run as root" + exit 1 +fi + +success "Running as root" # ----------------------------------------------------------------------------------------------- # ━━━ $ICON_SUMMARY Status ━━━ @@ -29,65 +46,167 @@ info "$ICON_TIME $(date)" if [[ "$SHOW_STATUS" == true ]]; then echo "" echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" - echo "$ICON_ZFS Mode: Read-only diagnostics" - echo "$ICON_MEM Source: ZFS ARC + system memory" - echo "$ICON_GEAR Dry Run: $DRY_RUN" + echo "$ICON_ZFS Log file: $ZFS_REPORT_LOG" + echo "$ICON_ZFS ARC warn: ${ZFS_REPORT_ARC_WARN_PCT}%" + echo "$ICON_MEM Free RAM warn: ${ZFS_REPORT_FREE_WARN_GB}GB" + echo "$ICON_MEM Avail RAM warn: ${ZFS_REPORT_AVAIL_WARN_GB}GB" + echo "$ICON_CONTAINERS Docker top: $ZFS_REPORT_DOCKER_TOP" + echo "$ICON_GEAR Dry Run: $DRY_RUN" echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi -# ----------------------------------------------------------------------------------------------- -# FUNCTIONS -# ----------------------------------------------------------------------------------------------- - -# Reads ZFS ARC statistics from the kernel stats interface. -# Filters for the most useful ARC metrics — size, hits, misses and metadata. -# Skips gracefully if ZFS is not available on this system. -show_zfs_arc() { - echo "" - echo "━━━ $ICON_ZFS ZFS ARC Stats ━━━" - - if [[ ! -r /proc/spl/kstat/zfs/arcstats ]]; then - warn "ZFS arcstats not available on this system — is ZFS loaded?" - return - fi - - grep -iE '^(c|size|hits|misses|arc_meta_used|demand_metadata_misses|mru_ghost_metadata|mfu_ghost_metadata)' \ - /proc/spl/kstat/zfs/arcstats 2>/dev/null || warn "Unable to read ARC stats" -} - -# Reads current system memory and swap usage using free. -# Skips gracefully if free is not available. -show_memory_status() { - echo "" - echo "━━━ $ICON_MEM Memory Status ━━━" - - if ! command -v free >/dev/null 2>&1; then - warn "free command not available on this system" - return - fi - - free -h | awk ' - NR==1 { print $0 } - /Mem:/ { print $0 } - /Swap:/ { print $0 }' -} +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — output will not be written to log" # ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_ZFS Snapshot ━━━ +# Tracking # ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_ZFS Snapshot ━━━" -echo "$ICON_ZFS Source: ZFS ARC + system memory" -echo "$ICON_GEAR Dry Run: $DRY_RUN" -echo "" - -[[ "$DRY_RUN" == true ]] && warn "DRY RUN — this script is read-only, data will still be collected" - +WARNINGS=() START=$(date +%s) +DATE=$(date +"%Y-%m-%d %H:%M:%S") -show_zfs_arc -show_memory_status +echo "" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo " $ICON_ZFS ZFS WEEKLY HEALTH REPORT — $DATE" +echo " $ICON_HOST Host: $(hostname)" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_ZFS ZFS Pool Health ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_ZFS ZFS Pool Health ━━━" + +if ! command -v zpool >/dev/null 2>&1; then + warn "ZFS not available on this system — skipping pool checks" +else + info "Pool status:" + zpool status 2>/dev/null | grep -E "pool:|state:|status:|errors:|scan:" | while IFS= read -r line; do + echo " $line" + done + + echo "" + + info "Pool overview:" + zpool list 2>/dev/null | while IFS= read -r line; do + echo " $line" + done + + UNHEALTHY=$(zpool list -H -o health 2>/dev/null | grep -v ONLINE || true) + if [[ -n "$UNHEALTHY" ]]; then + error "One or more ZFS pools are NOT ONLINE" + WARNINGS+=("ZFS pool unhealthy") + else + success "All ZFS pools are ONLINE" + fi +fi + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_ZFS ARC Statistics ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_ZFS ARC Statistics ━━━" + +if [[ ! -f /proc/spl/kstat/zfs/arcstats ]]; then + warn "ZFS arcstats not available — skipping ARC section" +else + ARC_MAX=$(cat /sys/module/zfs/parameters/zfs_arc_max 2>/dev/null || \ + awk '/^c_max / {print $3}' /proc/spl/kstat/zfs/arcstats) + ARC_SIZE=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats) + ARC_META_USED=$(awk '/^arc_meta_used / {print $3}' /proc/spl/kstat/zfs/arcstats) + + ARC_MAX_GB=$(awk "BEGIN {printf \"%.1f\", $ARC_MAX / 1073741824}") + ARC_CUR_GB=$(awk "BEGIN {printf \"%.1f\", $ARC_SIZE / 1073741824}") + ARC_META_GB=$(awk "BEGIN {printf \"%.1f\", $ARC_META_USED / 1073741824}") + ARC_PCT=$(awk "BEGIN {printf \"%.1f\", $ARC_SIZE * 100 / $ARC_MAX}") + ARC_PCT_INT=$(printf "%.0f" "$ARC_PCT") + + echo " $ICON_ZFS ARC Max: ${ARC_MAX_GB}GB" + echo " $ICON_ZFS ARC Current: ${ARC_CUR_GB}GB" + echo " $ICON_ZFS ARC Meta Used: ${ARC_META_GB}GB" + echo " $ICON_ZFS ARC Utilization: ${ARC_PCT}%" + + if [[ "$ARC_PCT_INT" -ge "$ZFS_REPORT_ARC_WARN_PCT" ]]; then + warn "ARC utilization ${ARC_PCT}% — above ${ZFS_REPORT_ARC_WARN_PCT}% threshold" + WARNINGS+=("ARC high: ${ARC_PCT}%") + else + success "ARC utilization ${ARC_PCT}% — within threshold" + fi + + echo "" + info "Metadata pressure:" + META_MRU_GHOST=$(awk '/^mru_ghost_metadata / {print $3}' /proc/spl/kstat/zfs/arcstats 2>/dev/null || echo 0) + META_MFU_GHOST=$(awk '/^mfu_ghost_metadata / {print $3}' /proc/spl/kstat/zfs/arcstats 2>/dev/null || echo 0) + META_MISSES=$(awk '/^demand_metadata_misses / {print $3}' /proc/spl/kstat/zfs/arcstats 2>/dev/null || echo 0) + + MRU_GB=$(awk "BEGIN {printf \"%.2f\", $META_MRU_GHOST / 1073741824}") + MFU_GB=$(awk "BEGIN {printf \"%.2f\", $META_MFU_GHOST / 1073741824}") + + echo " $ICON_ZFS MRU Ghost: ${MRU_GB}GB" + echo " $ICON_ZFS MFU Ghost: ${MFU_GB}GB" + echo " $ICON_ZFS Metadata Misses: ${META_MISSES}" +fi + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_MEM Memory Status ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_MEM Memory Status ━━━" + +FREE_HUMAN=$(free -h | awk '/Mem:/ {print $4}') +AVAIL_HUMAN=$(free -h | awk '/Mem:/ {print $7}') +TOTAL_HUMAN=$(free -h | awk '/Mem:/ {print $2}') +FREE_GB=$(free -g | awk '/Mem:/ {print $4}') +AVAIL_GB=$(free -g | awk '/Mem:/ {print $7}') + +echo " $ICON_MEM Total RAM: $TOTAL_HUMAN" +echo " $ICON_MEM Free RAM: $FREE_HUMAN" +echo " $ICON_MEM Available RAM: $AVAIL_HUMAN" + +if [[ "$FREE_GB" -lt "$ZFS_REPORT_FREE_WARN_GB" ]]; then + warn "Free RAM ${FREE_HUMAN} — below ${ZFS_REPORT_FREE_WARN_GB}GB threshold" + WARNINGS+=("Low free RAM: ${FREE_HUMAN}") +else + success "Free RAM ${FREE_HUMAN} — within threshold" +fi + +if [[ "$AVAIL_GB" -lt "$ZFS_REPORT_AVAIL_WARN_GB" ]]; then + warn "Available RAM ${AVAIL_HUMAN} — below ${ZFS_REPORT_AVAIL_WARN_GB}GB threshold" + WARNINGS+=("Low available RAM: ${AVAIL_HUMAN}") +else + success "Available RAM ${AVAIL_HUMAN} — within threshold" +fi + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_CONTAINERS Docker Memory ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_CONTAINERS Top $ZFS_REPORT_DOCKER_TOP Docker Memory Users ━━━" + +if ! command -v docker >/dev/null 2>&1; then + warn "Docker not available — skipping container memory section" +else + docker stats --no-stream \ + --format "table {{.Name}}\t{{.MemUsage}}\t{{.MemPerc}}" \ + 2>/dev/null | head -n $(( ZFS_REPORT_DOCKER_TOP + 1 )) | while IFS= read -r line; do + echo " $line" + done +fi + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_GEAR Kernel Pressure ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_GEAR Kernel Pressure ━━━" + +if ! command -v vmstat >/dev/null 2>&1; then + warn "vmstat not available — skipping kernel pressure section" +else + info "vmstat snapshot (3 samples):" + vmstat 1 3 2>/dev/null | while IFS= read -r line; do + echo " $line" + done +fi END=$(date +%s) @@ -95,9 +214,17 @@ END=$(date +%s) # ━━━ $ICON_SUMMARY Summary ━━━ # ----------------------------------------------------------------------------------------------- echo "" -echo "━━━━━ $ICON_SUMMARY SNAPSHOT SUMMARY ━━━━━" -echo "$ICON_ZFS Source: ZFS ARC + system memory" -echo "$ICON_GEAR Mode: READ-ONLY — no changes made" -echo "$ICON_TIME Duration: $(format_duration $((END - START)))" -echo "$ICON_DONE Status: $ICON_SUCCESS DONE" -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" \ No newline at end of file +echo "━━━━━ $ICON_SUMMARY ZFS REPORT SUMMARY ━━━━━" +echo "$ICON_TIME Duration: $(format_duration $((END - START)))" +echo "$ICON_ZFS Log: $ZFS_REPORT_LOG" +echo "" +if [[ ${#WARNINGS[@]} -eq 0 ]]; then + success "Report complete — no warnings" +else + echo "$ICON_WARN Warnings: ${#WARNINGS[@]}" + for w in "${WARNINGS[@]}"; do + echo " $ICON_WARN $w" + done + notify "ZFS weekly report on $(hostname) — ${#WARNINGS[@]} warning(s): ${WARNINGS[*]}" "ZFS Report" "warning" +fi +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" \ No newline at end of file diff --git a/user_script_pluin.sh b/user_script_pluin.sh index 67bfc06..03bcaff 100644 --- a/user_script_pluin.sh +++ b/user_script_pluin.sh @@ -16,18 +16,18 @@ # Shared runtime functions used by all scripts: # /mnt/user/appdata/unraid_scripts/common.sh # -# For full setup instructions see: -# /mnt/user/appdata/unraid_scripts/README.md -# # ============================================================================================== # Changelog: # v1.0 — Initial template # v1.1 — MAX_RSYNC_PROCS removed — bandwidth limiting handles concurrency -# Typo fixes Master.cong → Master.conf -# Added --status flag to arguments section -# Added full directory tree -# All script calls pre-written and grouped by type -# Changelog added +# Typo fixes, --status flag added, full directory tree, changelog added +# v1.2 — Docker Essentials, Media, Transcodes, System Watchdog added +# Directory tree updated to reflect full ecosystem +# v1.3 — Failover script added, directory tree updated with Failover folder +# v1.4 — WebGUI watchdog added +# ZFS memory snapshot added +# Docker network connect added +# Directory tree updated to reflect full ecosystem # ============================================================================================== # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ @@ -39,42 +39,60 @@ # ├── common.sh # Shared library — functions used by all scripts # ├── README.md # Project overview and quick start # ├── User_Script_Template.sh # This file — copy into User Scripts plugin +# ├── git_pull_execute.sh # Pulls latest scripts from Gitea repo +# │ +# ├── Failover/ +# │ └── failover.sh # Mutual container failover — runs continuously # │ # ├── Orchestrators/ # │ └── daily_sync.sh # Runs all daily media share syncs sequentially -# │ # add shares to master.conf, for seqential syncs +# │ # ├── Rsync/ # │ ├── rsync.sh # Core rsync script — called per share or profile # │ └── README_Rsync_Setup.md # Rsync-specific setup guide # │ +# ├── Docker_Essentials/ +# │ ├── docker_watchdog.sh # Container health monitor — memory, CPU, HTTP, stops +# │ ├── docker_daily_restart.sh # Restarts configured containers daily +# │ ├── docker_weekly_restart.sh # Restarts configured containers weekly +# │ └── docker_network_connect.sh # Connects containers to extra networks on boot +# │ +# ├── Media/ +# │ ├── media_permissions.sh # Applies permissions to all media shares +# │ └── media_cleaner.sh # Removes junk files — profiles: anime, media +# │ +# ├── Transcodes/ +# │ ├── ramdisk_setup.sh # Creates ramdisk and transcode symlink — run at array start +# │ ├── transcode_manager.sh # Monitors ramdisk usage, manages symlink direction +# │ └── transcode_cleanup.sh # Removes old inactive transcode files +# │ # ├── Tools/ # │ └── recreate_shares.sh # Recreates share dirs from .cfg files after incident # │ # └── unRAID_Essentials/ -# ├── clear_logs.sh # Clears unRAID log files -# ├── docker_syslog_filter.sh # Filters docker veth noise from syslog +# ├── clear_logs.sh # Clears unRAID system and Docker log files +# ├── docker_syslog_filter.sh # Filters Docker veth noise from syslog # ├── mover_stop.sh # Safely stops the unRAID mover -# ├── php_fpm_max_children.sh # Sets php-fpm max children value -# ├── rsync_stop.sh # Cleanly stops all running rsync processes +# ├── php_fpm_max_children.sh # Sets PHP-FPM max children value +# ├── rsync_stop.sh # Stops all rsync processes on both servers # ├── server_reboot.sh # Graceful server reboot with user warning -# ├── user_script_stop.sh # Stops a running User Scripts job -# └── zfs_memory_snapshot.sh # Creates a ZFS memory snapshot +# ├── system_watchdog.sh # System health monitor — last line of defense +# ├── user_script_stop.sh # Stops running User Scripts plugin jobs +# ├── webgui_restart.sh # WebGUI watchdog — nginx + emhttp escalating restart +# └── zfs_memory_snapshot.sh # Weekly ZFS health and memory diagnostic report # # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ # ━━━ 🚀 Script Commands — Uncomment the one you want to run ━━━ # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ # -# ━━━ Orchestrators (README) ━━━ -# The daily orchestrator runs all bulk media shares sequentially in a single scheduled job. -# Instead of creating a User Script entry per share, add the share path to DAILY_SYNC_SHARES -# in Master.conf and the orchestrator handles it automatically at 1am. +# ━━━ Failover (README) ━━━ +# Runs continuously as a background task — do NOT schedule with a time interval. +# Set schedule to "At Startup of Array" — background task. +# Both servers must have this script running for mutual failover to work. +# Use --status to check current state without restarting the loop. # -# Media shares are nothing special — they carry no profile and fall through to global defaults -# in Master.conf. Sequential execution means one share finishes before the next starts, -# no concurrency needed, bandwidth limiting keeps things sane if appdata jobs overlap. -# -# This is where 80% of your rsync shares should live. Only create individual scheduled -# entries for shares that need their own timing — like appdata profiles below. +# ━━━ Failover ━━━ +#/mnt/user/appdata/unraid_scripts/Failover/failover.sh # # ━━━ Orchestrators ━━━ #/mnt/user/appdata/unraid_scripts/Orchestrators/daily_sync.sh @@ -86,9 +104,36 @@ #/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-Failover/Emby #/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-Failover/Gmer4Lfe # -# ━━━ Rsync — Individual Media Share (ad hoc use) ━━━ +# ━━━ Rsync — Individual Media Shares (ad hoc use) ━━━ #/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Movies #/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Tv_Shows +#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Anime_Shows +#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Anime_Shows-Old +#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Anime_Movies +#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Anime_Movies-Old +#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Books +#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Intros +#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Kids_Movies +#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Kids_Tv_Shows +#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Music_Videos +#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Nextcloud +#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/stand-up_comedy +# +# ━━━ Docker Essentials ━━━ +#/mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_watchdog.sh +#/mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_daily_restart.sh +#/mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_weekly_restart.sh +#/mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_network_connect.sh +# +# ━━━ Media ━━━ +#/mnt/user/appdata/unraid_scripts/Media/media_permissions.sh +#/mnt/user/appdata/unraid_scripts/Media/media_cleaner.sh anime +#/mnt/user/appdata/unraid_scripts/Media/media_cleaner.sh media +# +# ━━━ Transcodes ━━━ +#/mnt/user/appdata/unraid_scripts/Transcodes/ramdisk_setup.sh +#/mnt/user/appdata/unraid_scripts/Transcodes/transcode_manager.sh +#/mnt/user/appdata/unraid_scripts/Transcodes/transcode_cleanup.sh # # ━━━ Tools ━━━ #/mnt/user/appdata/unraid_scripts/Tools/recreate_shares.sh @@ -100,7 +145,9 @@ #/mnt/user/appdata/unraid_scripts/unRAID_Essentials/php_fpm_max_children.sh #/mnt/user/appdata/unraid_scripts/unRAID_Essentials/rsync_stop.sh #/mnt/user/appdata/unraid_scripts/unRAID_Essentials/server_reboot.sh +#/mnt/user/appdata/unraid_scripts/unRAID_Essentials/system_watchdog.sh #/mnt/user/appdata/unraid_scripts/unRAID_Essentials/user_script_stop.sh +#/mnt/user/appdata/unraid_scripts/unRAID_Essentials/webgui_restart.sh #/mnt/user/appdata/unraid_scripts/unRAID_Essentials/zfs_memory_snapshot.sh # @@ -108,66 +155,46 @@ # ━━━ ⚙️ Arguments — Add after the script path ━━━ # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ # -# ━━━ Rsync / Orchestrators ━━━ -# -# --dry-run Preview what would be transferred — no changes made +# --dry-run Preview what would happen — no changes made # --log Enable verbose logging output # --no-log Disable logging (overrides Master.conf setting) -# --status Print resolved profile and configuration then exit +# --status Print resolved configuration and current state then exit # --help Show usage information -# # KEY=VALUE Override any Master.conf variable for this run only -# Better to add a profile in Master.conf for permanent changes # -# ━━━ unRAID Essentials ━━━ +# ━━━ Media Cleaner profile (required first argument) ━━━ +# media_cleaner.sh anime --dry-run +# media_cleaner.sh media --dry-run # -# Arguments vary per script — see comments inside each script for details +# ━━━ Failover specific ━━━ +# failover.sh --status — check current state +# failover.sh --dry-run --log — test logic without touching containers # # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -# ━━━ 📋 Profile System ━━━ +# ━━━ 📋 Recommended Schedules ━━━ # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ # -# Profiles are defined in Master.conf and matched automatically by the -# basename of the directory path passed to rsync.sh (lowercased). +# failover.sh — At Startup of Array (background task) +# ramdisk_setup.sh — At Startup of Array +# docker_network_connect.sh — At Startup of Array +# docker_syslog_filter.sh — At Startup of Array +# php_fpm_max_children.sh — At Startup of Array # -# Example: -# /mnt/user/appdata-Failover/Arrs_Stack → matches profile key [arrs_stack] -# /mnt/user/Movies → no match, uses global defaults +# transcode_manager.sh — */3 * * * * (every 3 minutes) +# transcode_cleanup.sh — */5 * * * * (every 5 minutes) +# docker_watchdog.sh — */15 * * * * (every 15 minutes) +# webgui_restart.sh — */10 * * * * (every 10 minutes) +# system_watchdog.sh — */15 * * * * (every 15 minutes) # -# If a profile match is found — profile settings are used for that run -# If no profile match is found — global defaults from Master.conf are used +# daily_sync.sh — 0 1 * * * (daily at 1am) +# docker_daily_restart.sh — 0 3 * * * (daily at 3am) +# media_cleaner.sh anime — 0 2 * * * (daily at 2am) +# media_cleaner.sh media — 0 2 * * * (daily at 2am) +# media_permissions.sh — 0 4 * * 0 (weekly Sunday 4am) +# clear_logs.sh — 0 5 * * 0 (weekly Sunday 5am) +# zfs_memory_snapshot.sh — 0 6 * * 0 (weekly Sunday 6am) +# docker_weekly_restart.sh — 0 3 * * 0 (weekly Sunday 3am) # -# Profile settings control: -# - rsync options PROFILE_RSYNC_OPTS -# - bandwidth limit PROFILE_BW_LIMIT -# - retry count PROFILE_RETRY_COUNT -# - sleep between retry PROFILE_SLEEP -# - containers to stop PROFILE_CRITICAL_CONTAINER_NAMES -# - delayed containers PROFILE_DELAYED_CONTAINERS -# - container delay PROFILE_CONTAINER_DELAY -# - excluded dirs PROFILE_EXCLUDE_DIRS -# -# To add a new profile — add a key to each array in Master.conf -# For permanent changes — always use Master.conf -# For one-off overrides — use KEY=VALUE arguments (see above) -# -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -# ━━━ 💡 Examples ━━━ -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -# -# Run arrs_stack profile sync: -# /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-Failover/Arrs_Stack -# -# Dry run with logging enabled: -# /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-Failover/Arrs_Stack --dry-run --log -# -# Check what profile and settings resolved before running: -# /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-Failover/Arrs_Stack --status -# -# Override bandwidth limit for this run only: -# /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Movies BW_LIMIT=5000 -# -# Run daily orchestrator: -# /mnt/user/appdata/unraid_scripts/Orchestrators/daily_sync.sh +# rsync.sh profiles — schedule individually as needed # # ============================================================================================== \ No newline at end of file