small fixes

Failover/failover_test.sh

@@ -0,0 +1,389 @@
+#!/bin/bash
+# -----------------------------------------------------------------------------------------------
+# --------------------------------- Failover Test ----------------------------------------------
+# -----------------------------------------------------------------------------------------------
+# Controlled simulation of the failover scenario — validates the entire failover lifecycle
+# without waiting for a real outage.
+#
+# This script is a TEST HARNESS only — it does not contain failover logic.
+# All failover logic lives in failover.sh and is called directly from here.
+# Any changes to failover.sh are automatically reflected in this test.
+#
+# Test sequence:
+#   1. Pre-flight  — verify both servers reachable, failover.sh exists, state is NORMAL
+#   2. Block       — add iptables rule dropping all traffic to remote IP
+#   3. Detect      — run failover.sh one cycle — confirm FAILOVER state detected
+#   4. Start       — verify failover containers started locally
+#   5. Restore     — remove iptables rule, remote becomes reachable again
+#   6. Handback    — wait for failover.sh to confirm handback strikes and hand back
+#   7. Verify      — confirm containers returned to remote, local copies stopped
+#   8. Report      — full pass/fail summary per phase
+#
+# Safety: iptables rule is removed via trap on ANY exit — crash, error, ctrl-c, or normal.
+# Remote connectivity is always restored regardless of test outcome.
+#
+# ⚠️  This script starts and stops real containers on both servers.
+#     Run during a maintenance window — users will experience a brief service interruption.
+#     Use --dry-run to walk through the sequence without touching containers or iptables.
+#
+# All configuration in Master.conf under Failover and Failover Test sections.
+# -----------------------------------------------------------------------------------------------
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+source "$SCRIPT_DIR/../Master.conf"
+source "$SCRIPT_DIR/../common.sh"
+
+parse_args "$@"
+
+FAILOVER_SCRIPT="$SCRIPT_DIR/failover.sh"
+
+# -----------------------------------------------------------------------------------------------
+# SAFETY TRAP — always remove iptables rule on exit
+# Fires on normal exit, error exit, ctrl-c, and script crashes
+# -----------------------------------------------------------------------------------------------
+IPTABLES_RULE_ACTIVE=false
+
+cleanup() {
+    if [[ "$IPTABLES_RULE_ACTIVE" == true ]]; then
+        echo ""
+        warn "$ICON_SHIELD Cleanup — removing iptables block on $REMOTE_SERVER..."
+        if [[ "$DRY_RUN" == false ]]; then
+            iptables -D OUTPUT -d "$REMOTE_SERVER" -j DROP 2>/dev/null
+            IPTABLES_RULE_ACTIVE=false
+            success "iptables rule removed — remote connectivity restored"
+        else
+            warn "DRY RUN — would remove iptables rule"
+        fi
+    fi
+}
+
+trap cleanup EXIT
+
+# -----------------------------------------------------------------------------------------------
+# ━━━ $ICON_GEAR Setup ━━━
+# -----------------------------------------------------------------------------------------------
+echo ""
+echo "━━━ $ICON_GEAR Setup ━━━"
+
+if [[ "$EUID" -ne 0 ]]; then
+    error "Must be run as root"
+    exit 1
+fi
+
+success "Running as root"
+
+if ! command -v iptables >/dev/null 2>&1; then
+    error "iptables not found — required for connectivity simulation"
+    exit 1
+fi
+
+success "iptables available"
+
+if [[ ! -f "$FAILOVER_SCRIPT" ]]; then
+    error "failover.sh not found at $FAILOVER_SCRIPT"
+    exit 1
+fi
+
+success "failover.sh found"
+
+detect_hosts
+resolve_remote_ip
+
+[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no iptables rules or container changes will be made"
+
+# -----------------------------------------------------------------------------------------------
+# ━━━ $ICON_SUMMARY Status ━━━
+# -----------------------------------------------------------------------------------------------
+if [[ "$SHOW_STATUS" == true ]]; then
+    echo ""
+    echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
+    echo "$ICON_HOST      Local:         $LOCAL_SERVER_NAME"
+    echo "$ICON_HOST      Remote:        $REMOTE_SERVER_NAME ($REMOTE_SERVER)"
+    echo "$ICON_FAILOVER  Block wait:    ${FAILOVER_TEST_BLOCK_WAIT}s"
+    echo "$ICON_FAILOVER  Handback wait: ${FAILOVER_TEST_HANDBACK_WAIT}s"
+    echo "$ICON_GEAR      Dry Run:       $DRY_RUN"
+
+    # Current failover state
+    if [[ -f "$FAILOVER_STATE_FILE" ]]; then
+        CURRENT_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
+        echo "$ICON_FAILOVER  Current state: ${CURRENT_STATE:-unknown}"
+    else
+        echo "$ICON_FAILOVER  Current state: no state file"
+    fi
+    echo "━━━━━━━━━━━━━━━━━━━━━━━"
+    exit 0
+fi
+
+# -----------------------------------------------------------------------------------------------
+# PHASE TRACKING
+# -----------------------------------------------------------------------------------------------
+PHASES_PASS=()
+PHASES_FAIL=()
+TOTAL_START=$(date +%s)
+
+phase_pass() { PHASES_PASS+=("$1"); success "$ICON_DONE Phase: $1 — PASSED"; }
+phase_fail() { PHASES_FAIL+=("$1"); error "$ICON_ERROR Phase: $1 — FAILED"; }
+
+# -----------------------------------------------------------------------------------------------
+# ━━━ PHASE 1 — Pre-flight ━━━
+# -----------------------------------------------------------------------------------------------
+echo ""
+echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
+echo "  $ICON_SHIELD FAILOVER TEST — $(date '+%Y-%m-%d %H:%M:%S')"
+echo "  $ICON_HOST Local:  $LOCAL_SERVER_NAME"
+echo "  $ICON_HOST Remote: $REMOTE_SERVER_NAME ($REMOTE_SERVER)"
+echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
+echo ""
+echo "━━━ $ICON_SHIELD Phase 1 — Pre-flight ━━━"
+
+# Check remote reachable
+info "Checking remote reachability..."
+if ping_remote; then
+    success "Remote $REMOTE_SERVER_NAME is reachable"
+else
+    error "Remote $REMOTE_SERVER_NAME is not reachable — cannot run test"
+    phase_fail "Pre-flight"
+    exit 1
+fi
+
+# Check internet reachable
+info "Checking internet connectivity..."
+if ping_internet; then
+    success "Internet is reachable"
+else
+    error "No internet connectivity — cannot run test"
+    phase_fail "Pre-flight"
+    exit 1
+fi
+
+# Check current failover state is NORMAL
+if [[ -f "$FAILOVER_STATE_FILE" ]]; then
+    CURRENT_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
+    if [[ "$CURRENT_STATE" != "NORMAL" ]]; then
+        error "Failover state is $CURRENT_STATE — must be NORMAL before running test"
+        phase_fail "Pre-flight"
+        exit 1
+    fi
+    success "Failover state is NORMAL"
+else
+    warn "No state file found — assuming NORMAL (first run)"
+fi
+
+phase_pass "Pre-flight"
+
+# -----------------------------------------------------------------------------------------------
+# ━━━ PHASE 2 — Block Remote ━━━
+# -----------------------------------------------------------------------------------------------
+echo ""
+echo "━━━ $ICON_PING Phase 2 — Block Remote Connectivity ━━━"
+warn "Adding iptables rule — dropping all traffic to $REMOTE_SERVER"
+
+if [[ "$DRY_RUN" == false ]]; then
+    iptables -I OUTPUT -d "$REMOTE_SERVER" -j DROP
+    IPTABLES_RULE_ACTIVE=true
+    success "iptables rule active — $REMOTE_SERVER_NAME appears unreachable"
+
+    # Verify block is working
+    if ! ping -c1 -W2 "$REMOTE_SERVER" &>/dev/null; then
+        success "Connectivity block confirmed — ping to remote fails as expected"
+        phase_pass "Block Remote"
+    else
+        error "iptables rule did not block connectivity — ping still succeeds"
+        phase_fail "Block Remote"
+        exit 1
+    fi
+else
+    warn "DRY RUN — would block $REMOTE_SERVER with iptables"
+    phase_pass "Block Remote"
+fi
+
+# -----------------------------------------------------------------------------------------------
+# ━━━ PHASE 3 — Failover Detection ━━━
+# -----------------------------------------------------------------------------------------------
+echo ""
+echo "━━━ $ICON_FAILOVER Phase 3 — Failover Detection ━━━"
+info "Waiting ${FAILOVER_TEST_BLOCK_WAIT}s for failover.sh to detect outage..."
+info "failover.sh check interval is ${FAILOVER_CHECK_INTERVAL}s"
+
+if [[ "$DRY_RUN" == false ]]; then
+    sleep "$FAILOVER_TEST_BLOCK_WAIT"
+
+    # Check state file updated to FAILOVER
+    if [[ -f "$FAILOVER_STATE_FILE" ]]; then
+        NEW_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
+        if [[ "$NEW_STATE" == "FAILOVER" ]]; then
+            success "State changed to FAILOVER — outage detected correctly"
+            phase_pass "Failover Detection"
+        else
+            error "State is $NEW_STATE — expected FAILOVER after ${FAILOVER_TEST_BLOCK_WAIT}s"
+            warn "failover.sh may not be running — check User Scripts plugin"
+            phase_fail "Failover Detection"
+        fi
+    else
+        error "No state file found after wait — failover.sh may not be running"
+        phase_fail "Failover Detection"
+    fi
+else
+    warn "DRY RUN — would wait ${FAILOVER_TEST_BLOCK_WAIT}s and check for FAILOVER state"
+    phase_pass "Failover Detection"
+fi
+
+# -----------------------------------------------------------------------------------------------
+# ━━━ PHASE 4 — Container Start Verification ━━━
+# -----------------------------------------------------------------------------------------------
+echo ""
+echo "━━━ $ICON_CONTAINERS Phase 4 — Failover Containers Started ━━━"
+
+# Determine which containers should have started on this host
+if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then
+    EXPECTED_CONTAINERS=("${FAILOVER_HOST1_STARTS_FOR_HOST2[@]}")
+else
+    EXPECTED_CONTAINERS=("${FAILOVER_HOST2_STARTS_FOR_HOST1[@]}")
+fi
+
+if [[ "$DRY_RUN" == false ]]; then
+    CONTAINERS_OK=true
+    for container in "${EXPECTED_CONTAINERS[@]}"; do
+        [[ -z "$container" ]] && continue
+        STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
+        if [[ "$STATUS" == "true" ]]; then
+            success "$ICON_RUNNING $container is running locally"
+        else
+            error "$ICON_NOT_RUNNING $container is NOT running locally"
+            CONTAINERS_OK=false
+        fi
+    done
+
+    if [[ "$CONTAINERS_OK" == true ]]; then
+        phase_pass "Container Start"
+    else
+        phase_fail "Container Start"
+    fi
+else
+    warn "DRY RUN — would verify these containers started: ${EXPECTED_CONTAINERS[*]}"
+    phase_pass "Container Start"
+fi
+
+# -----------------------------------------------------------------------------------------------
+# ━━━ PHASE 5 — Restore Connectivity ━━━
+# -----------------------------------------------------------------------------------------------
+echo ""
+echo "━━━ $ICON_PING Phase 5 — Restore Remote Connectivity ━━━"
+info "Removing iptables block — remote becomes reachable again"
+
+if [[ "$DRY_RUN" == false ]]; then
+    iptables -D OUTPUT -d "$REMOTE_SERVER" -j DROP 2>/dev/null
+    IPTABLES_RULE_ACTIVE=false
+    success "iptables rule removed"
+
+    # Verify connectivity restored
+    sleep 3
+    if ping_remote; then
+        success "Remote $REMOTE_SERVER_NAME is reachable again"
+        phase_pass "Restore Connectivity"
+    else
+        error "Remote still unreachable after removing iptables rule"
+        phase_fail "Restore Connectivity"
+    fi
+else
+    warn "DRY RUN — would remove iptables rule"
+    phase_pass "Restore Connectivity"
+fi
+
+# -----------------------------------------------------------------------------------------------
+# ━━━ PHASE 6 — Handback ━━━
+# -----------------------------------------------------------------------------------------------
+echo ""
+echo "━━━ $ICON_FAILOVER Phase 6 — Handback ━━━"
+info "Waiting ${FAILOVER_TEST_HANDBACK_WAIT}s for failover.sh to confirm handback..."
+info "Requires $FAILOVER_HANDBACK_STRIKES consecutive remote-up checks at ${FAILOVER_CHECK_INTERVAL}s intervals"
+info "Estimated minimum wait: $(( FAILOVER_HANDBACK_STRIKES * FAILOVER_CHECK_INTERVAL ))s"
+
+if [[ "$DRY_RUN" == false ]]; then
+    sleep "$FAILOVER_TEST_HANDBACK_WAIT"
+
+    if [[ -f "$FAILOVER_STATE_FILE" ]]; then
+        FINAL_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
+        if [[ "$FINAL_STATE" == "NORMAL" ]]; then
+            success "State returned to NORMAL — handback completed"
+            phase_pass "Handback"
+        else
+            error "State is $FINAL_STATE — expected NORMAL after handback wait"
+            phase_fail "Handback"
+        fi
+    else
+        error "No state file found"
+        phase_fail "Handback"
+    fi
+else
+    warn "DRY RUN — would wait ${FAILOVER_TEST_HANDBACK_WAIT}s and verify NORMAL state"
+    phase_pass "Handback"
+fi
+
+# -----------------------------------------------------------------------------------------------
+# ━━━ PHASE 7 — Container Handback Verification ━━━
+# -----------------------------------------------------------------------------------------------
+echo ""
+echo "━━━ $ICON_CONTAINERS Phase 7 — Failover Containers Stopped Locally ━━━"
+
+if [[ "$DRY_RUN" == false ]]; then
+    HANDBACK_OK=true
+    for container in "${EXPECTED_CONTAINERS[@]}"; do
+        [[ -z "$container" ]] && continue
+        STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
+        if [[ "$STATUS" != "true" ]]; then
+            success "$ICON_NOT_RUNNING $container stopped locally — handed back"
+        else
+            error "$ICON_RUNNING $container still running locally — handback may have failed"
+            HANDBACK_OK=false
+        fi
+    done
+
+    if [[ "$HANDBACK_OK" == true ]]; then
+        phase_pass "Container Handback"
+    else
+        phase_fail "Container Handback"
+    fi
+else
+    warn "DRY RUN — would verify failover containers stopped locally after handback"
+    phase_pass "Container Handback"
+fi
+
+TOTAL_END=$(date +%s)
+
+# -----------------------------------------------------------------------------------------------
+# ━━━ $ICON_SUMMARY Test Report ━━━
+# -----------------------------------------------------------------------------------------------
+echo ""
+echo "━━━━━ $ICON_SUMMARY FAILOVER TEST REPORT ━━━━━"
+echo "$ICON_HOST      Local:    $LOCAL_SERVER_NAME"
+echo "$ICON_HOST      Remote:   $REMOTE_SERVER_NAME"
+echo "$ICON_TIME      Duration: $(format_duration $((TOTAL_END - TOTAL_START)))"
+echo ""
+echo "  Phase Results:"
+for phase in "${PHASES_PASS[@]}"; do
+    echo "    $ICON_SUCCESS $phase"
+done
+for phase in "${PHASES_FAIL[@]}"; do
+    echo "    $ICON_ERROR $phase"
+done
+echo ""
+
+PASS_COUNT=${#PHASES_PASS[@]}
+FAIL_COUNT=${#PHASES_FAIL[@]}
+TOTAL_PHASES=$(( PASS_COUNT + FAIL_COUNT ))
+
+if [[ "$DRY_RUN" == true ]]; then
+    echo "$ICON_WARN Status: DRY RUN — no changes made"
+elif [[ "$FAIL_COUNT" -eq 0 ]]; then
+    echo "$ICON_DONE Status: $ICON_SUCCESS ALL $TOTAL_PHASES PHASES PASSED"
+    notify "Failover test PASSED on $(hostname) — all $TOTAL_PHASES phases completed successfully" "Failover Test" "normal"
+else
+    echo "$ICON_ERROR Status: $FAIL_COUNT/$TOTAL_PHASES PHASES FAILED"
+    notify "Failover test FAILED on $(hostname) — $FAIL_COUNT/$TOTAL_PHASES phases failed: ${PHASES_FAIL[*]}" "Failover Test" "warning"
+fi
+echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
+
+[[ "$FAIL_COUNT" -gt 0 ]] && exit 1
+exit 0

Master.conf

@@ -53,7 +53,7 @@
 #   ── TRANSCODES ─────────────────────────────────────────────────────────────────────────────
 #   TRANSCODE MANAGER               Ramdisk and SSD fallback transcode management
 #
-#   ── MONITOR ────────────────────────────────────────────────────────────────────────────────
+#   ── MONITORS ────────────────────────────────────────────────────────────────────────────────
 #   CERTIFICATE MONITOR             SSL certificate expiry monitoring
 #   BACKUP VERIFY                   Random sample checksum verification against remote
 #   SMART HEALTH                    Drive SMART attribute monitoring
@@ -294,83 +294,64 @@ declare -A PROFILE_SKIP_DISK_CHECK=(
 # Each server runs Failover/failover.sh independently — no coordination between servers.
 # All decisions are based solely on two ping checks: remote reachable + internet reachable.
 #
-# How it works:
-#   Every FAILOVER_CHECK_INTERVAL seconds each server pings the remote and pings the internet.
-#   Based on those two results it determines its state and takes the appropriate action.
-#   No SSH, no signaling — each server acts autonomously based only on what it can see.
-#
 # States:
 #   NORMAL      — remote up, internet up — own containers only, silent operation
 #   FAILOVER    — remote down, internet up — start remote's containers locally (additive)
-#                 Own normal containers keep running — failover containers added on top
 #   NO_INTERNET — internet down — stop public-facing containers, wait for recovery
+#   DARK        — remote down + internet down — same actions as NO_INTERNET
 #
-# Handback sequence when remote returns after FAILOVER:
+# Handback: strike confirmation → pre-flight → rsync → start remote → stop local
-#   1. Strike confirmation — FAILOVER_HANDBACK_STRIKES consecutive remote-up checks
+ 
-#      Prevents handing back during a brief network blip
-#   2. Pre-flight checks — remote array started, Docker daemon healthy, rootfs not full
-#   3. Rsync data back via rsync.sh — uses existing profile system for options
-#   4. Start failover containers on remote via SSH
-#   5. Stop failover containers locally — only after remote confirmed started
-#   6. Return to NORMAL state
-#
-# Script runs on BOTH servers — detect_hosts() selects the correct arrays automatically.
-# Comment out any container or rsync job to disable without removing the entry.
-
     EXTERNAL_IP="8.8.8.8"              # external IP to ping for internet connectivity check
-    FAILOVER_CHECK_INTERVAL=120         # seconds between state checks — 120s = 2 minute polling
+    FAILOVER_CHECK_INTERVAL=120         # seconds between state checks
-    FAILOVER_HANDBACK_STRIKES=2         # consecutive remote-up confirmations required before handback
+    FAILOVER_HANDBACK_STRIKES=2         # consecutive remote-up confirmations before handback
-                                        # 2 strikes at 120s interval = 4 minutes confirmation window
     FAILOVER_STATE_FILE="/boot/config/failover_state.db"
-                                        # persists on /boot/ so it survives reboots
+ 
-                                        # script re-evaluates from live pings on restart
+# ━━━ Failover Test ━━━
-
+# Used by Failover/failover_test.sh — controlled simulation of the failover lifecycle.
-# ━━━ HOST1 Failover Config (unRAID-Gmer4Lfe — Primary) ━━━
+# failover_test.sh blocks remote connectivity via iptables then observes failover.sh behavior.
-
+# All failover logic stays in failover.sh — test script is the harness only.
-# Containers HOST1 starts locally when HOST2 goes down.
+#
-# These run ON TOP OF HOST1's normal containers — additive, not a replacement.
+# ⚠️  Run during a maintenance window — real containers start and stop during the test.
+#     Use --dry-run first to walk through phases without touching anything.
+ 
+    # Seconds to hold the iptables block — must be longer than FAILOVER_CHECK_INTERVAL
+    # so failover.sh has time to detect the outage and change state
+    FAILOVER_TEST_BLOCK_WAIT=150        # 150s = FAILOVER_CHECK_INTERVAL + 30s buffer
+ 
+    # Seconds to wait for handback after restoring connectivity
+    # Must cover FAILOVER_HANDBACK_STRIKES x FAILOVER_CHECK_INTERVAL plus rsync time
+    # 2 strikes x 120s = 240s minimum — add buffer for rsync handback jobs
+    FAILOVER_TEST_HANDBACK_WAIT=360     # 360s = 6 minutes — adjust if rsync takes longer
+ 
+ 
 FAILOVER_HOST1_STARTS_FOR_HOST2=(
     "Gmer4Lfe.com"
     "Gmer4Lfe.us"
 )
-
-# Containers HOST1 stops when it loses internet connectivity.
-# No point serving DDNS or public services if HOST1 itself has no internet.
 FAILOVER_HOST1_STOP_ON_NO_NET=(
     "Gmer4Lfe.com"
     "Gmer4Lfe.us"
 )
-
-# Rsync jobs HOST1 runs before handing containers back to HOST2 after recovery.
-# Uses rsync.sh with the existing profile system — basename matched to profile keys.
-# Comment out jobs that are not yet ready or not needed for handback.
 FAILOVER_HOST1_RSYNC_JOBS=(
 #    "/mnt/user/appdata-Failover/Jayred365"
 #    "/mnt/user/Media_Server/Emby-Jayred"
 )
-
+ 
-# ━━━ HOST2 Failover Config (unRAID-Jayred365 — Secondary) ━━━
+# HOST2 (unRAID-Jayred365 — Secondary)
-
-# Containers HOST2 starts locally when HOST1 goes down.
-# Emby starts on HOST2 so media keeps working during HOST1 outage.
-# HOST1's DDNS containers start here so DNS updates to point at HOST2's IP.
 FAILOVER_HOST2_STARTS_FOR_HOST1=(
     "Emby"
     "Gmer4Lfe.com"
     "Gmer4Lfe.us"
 )
-
-# Containers HOST2 stops when it loses internet connectivity.
 FAILOVER_HOST2_STOP_ON_NO_NET=(
     "Gmer4Lfe.com"
     "Gmer4Lfe.us"
 )
-
-# Rsync jobs HOST2 runs before handing containers back to HOST1 after recovery.
 FAILOVER_HOST2_RSYNC_JOBS=(
 #    "/mnt/user/appdata-Failover/Gmer4Lfe"
 )
-
+ 
 # ==============================================================================================
 # ── DOCKER ESSENTIALS ─────────────────────────────────────────────────────────────────────────
 # ==============================================================================================
@@ -702,7 +683,7 @@ MEDIA_MAINTENANCE_JOBS=(
     TRANSCODE_EMBY_CONTAINER="Emby"   # exact Docker container name — case sensitive
  
 # ==============================================================================================
-# ── MONITOR ───────────────────────────────────────────────────────────────────────────────────
+# ── MONITORS ───────────────────────────────────────────────────────────────────────────────────
 # ==============================================================================================
 # Monitoring scripts — watch and report only, never take action.
 # Lives in Monitor/ folder — distinct from unRAID_Essentials (which acts) and

Media/radarr_cleanup.sh

@@ -0,0 +1,281 @@
+#!/bin/bash
+# -----------------------------------------------------------------------------------------------
+# --------------------------------- Radarr Cleanup Script --------------------------------------
+# -----------------------------------------------------------------------------------------------
+# Removes orphaned movie files from the library that Radarr no longer tracks.
+# Uses the Radarr API to build a complete list of tracked movie file paths then compares
+# against what exists on disk — anything not tracked and older than RADARR_ORPHAN_AGE
+# days is considered an orphan and deleted.
+#
+# File classification:
+#   TRACKED   — Radarr API knows about this exact file path → leave it alone
+#   PROTECTED — matches RADARR_PROTECTED_PATTERNS → never delete (artwork, subtitles, .nfo)
+#   ORPHAN    — video file, not tracked, older than RADARR_ORPHAN_AGE days → delete
+#   JUNK      — not a video extension, not protected → delete regardless of age
+#   RECENT    — not tracked, under RADARR_ORPHAN_AGE days old → skip (may be mid-import)
+#
+# Why protected patterns matter:
+#   Radarr generates movie artwork (*.jpg), metadata (*.nfo) and manages subtitles
+#   (*.srt, *.sub, *.ass) but does not include these in its tracked file API response.
+#   Without protection these would be classified as orphans and deleted.
+#
+# All configuration in Master.conf under Arr Cleanup section.
+# Supports --dry-run to preview what would be deleted without making changes.
+# -----------------------------------------------------------------------------------------------
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+source "$SCRIPT_DIR/../Master.conf"
+source "$SCRIPT_DIR/../common.sh"
+
+parse_args "$@"
+
+# -----------------------------------------------------------------------------------------------
+# ━━━ $ICON_GEAR Setup ━━━
+# -----------------------------------------------------------------------------------------------
+echo ""
+echo "━━━ $ICON_GEAR Setup ━━━"
+
+if [[ "$EUID" -ne 0 ]]; then
+    error "Must be run as root"
+    exit 1
+fi
+
+success "Running as root"
+
+if ! command -v curl >/dev/null 2>&1; then
+    error "curl not found — required for Radarr API calls"
+    exit 1
+fi
+
+if ! command -v jq >/dev/null 2>&1; then
+    error "jq not found — required for JSON parsing"
+    notify "Radarr cleanup failed on $(hostname) — jq not installed" "Radarr Cleanup" "warning"
+    exit 1
+fi
+
+require_var RADARR_URL
+require_var RADARR_API_KEY
+require_var RADARR_MOVIES_ROOT
+
+if [[ ! -d "$RADARR_MOVIES_ROOT" ]]; then
+    error "Movies root not found: $RADARR_MOVIES_ROOT"
+    notify "Radarr cleanup failed on $(hostname) — movies root not found: $RADARR_MOVIES_ROOT" "Radarr Cleanup" "warning"
+    exit 1
+fi
+
+success "Config validated"
+
+# -----------------------------------------------------------------------------------------------
+# ━━━ $ICON_SUMMARY Status ━━━
+# -----------------------------------------------------------------------------------------------
+if [[ "$SHOW_STATUS" == true ]]; then
+    echo ""
+    echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
+    echo "$ICON_GEAR  Radarr URL:         $RADARR_URL"
+    echo "$ICON_GEAR  Movies root:        $RADARR_MOVIES_ROOT"
+    echo "$ICON_TIME  Orphan age:         ${RADARR_ORPHAN_AGE} days"
+    echo "$ICON_GEAR  Extensions:         ${RADARR_EXTENSIONS[*]}"
+    echo "$ICON_GEAR  Protected patterns: ${RADARR_PROTECTED_PATTERNS[*]}"
+    echo "$ICON_GEAR  Dry Run:            $DRY_RUN"
+    echo "━━━━━━━━━━━━━━━━━━━━━━━"
+    exit 0
+fi
+
+[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be deleted"
+
+# -----------------------------------------------------------------------------------------------
+# HELPERS
+# -----------------------------------------------------------------------------------------------
+
+radarr_api() {
+    local endpoint="$1"
+    local response http_code body
+
+    response=$(curl -sf \
+        --max-time 30 \
+        -H "X-Api-Key: $RADARR_API_KEY" \
+        -w "\n%{http_code}" \
+        "${RADARR_URL}/api/v3/${endpoint}" 2>/dev/null)
+
+    http_code=$(echo "$response" | tail -1)
+    body=$(echo "$response" | head -n -1)
+
+    if [[ "$http_code" != "200" ]]; then
+        error "Radarr API returned HTTP $http_code for endpoint: $endpoint"
+        return 1
+    fi
+
+    echo "$body"
+}
+
+is_video_file() {
+    local ext="${1##*.}"
+    ext="${ext,,}"
+    for valid_ext in "${RADARR_EXTENSIONS[@]}"; do
+        [[ "$ext" == "$valid_ext" ]] && return 0
+    done
+    return 1
+}
+
+is_protected_file() {
+    local filename
+    filename=$(basename "$1")
+    for pattern in "${RADARR_PROTECTED_PATTERNS[@]}"; do
+        # shellcheck disable=SC2254
+        case "$filename" in
+            $pattern) return 0 ;;
+        esac
+    done
+    return 1
+}
+
+# -----------------------------------------------------------------------------------------------
+# ━━━ $ICON_SYNC Fetching Radarr Tracked Files ━━━
+# -----------------------------------------------------------------------------------------------
+echo ""
+echo "━━━ $ICON_SYNC Fetching Radarr Tracked Files ━━━"
+
+info "Querying Radarr API: $RADARR_URL"
+
+MOVIEFILE_RESPONSE=$(radarr_api "moviefile") || {
+    error "Failed to fetch movie files from Radarr — check URL and API key"
+    notify "Radarr cleanup failed on $(hostname) — API unreachable" "Radarr Cleanup" "warning"
+    exit 1
+}
+
+TMP_DIR="/tmp/radarr_cleanup_$$"
+mkdir -p "$TMP_DIR"
+trap "rm -rf $TMP_DIR" EXIT
+
+TRACKED_FILE="$TMP_DIR/tracked_paths.txt"
+echo "$MOVIEFILE_RESPONSE" | jq -r '.[].path' 2>/dev/null | sort > "$TRACKED_FILE"
+
+TRACKED_COUNT=$(wc -l < "$TRACKED_FILE")
+success "Radarr tracks $TRACKED_COUNT movie files"
+
+if [[ "$TRACKED_COUNT" -eq 0 ]]; then
+    warn "No tracked files returned — Radarr may not have scanned yet or library is empty"
+    warn "Aborting to prevent mass deletion"
+    notify "Radarr cleanup aborted on $(hostname) — no tracked files returned from API" "Radarr Cleanup" "warning"
+    exit 1
+fi
+
+# -----------------------------------------------------------------------------------------------
+# ━━━ $ICON_CLEAN Scanning Movies Root ━━━
+# -----------------------------------------------------------------------------------------------
+echo ""
+echo "━━━ $ICON_CLEAN Scanning Movies Root ━━━"
+info "Root:              $RADARR_MOVIES_ROOT"
+info "Orphan age:        ${RADARR_ORPHAN_AGE} days"
+info "Protected:         ${RADARR_PROTECTED_PATTERNS[*]}"
+echo ""
+
+START=$(date +%s)
+ORPHAN_COUNT=0
+JUNK_COUNT=0
+RECENT_COUNT=0
+PROTECTED_COUNT=0
+ORPHAN_BYTES=0
+JUNK_BYTES=0
+
+AGE_SECONDS=$(( RADARR_ORPHAN_AGE * 86400 ))
+NOW=$(date +%s)
+
+while IFS= read -r filepath; do
+    [[ -z "$filepath" ]] && continue
+
+    if grep -qF "$filepath" "$TRACKED_FILE" 2>/dev/null; then
+        log "TRACKED: $filepath"
+        continue
+    fi
+
+    if is_protected_file "$filepath"; then
+        log "PROTECTED: $filepath"
+        ((PROTECTED_COUNT++))
+        continue
+    fi
+
+    FILE_SIZE=$(stat -c%s "$filepath" 2>/dev/null || echo 0)
+
+    if is_video_file "$filepath"; then
+        FILE_MTIME=$(stat -c %Y "$filepath" 2>/dev/null || echo 0)
+        FILE_AGE=$(( NOW - FILE_MTIME ))
+
+        if [[ "$FILE_AGE" -lt "$AGE_SECONDS" ]]; then
+            log "RECENT (skipping): $filepath"
+            ((RECENT_COUNT++))
+            continue
+        fi
+
+        warn "$ICON_TRASH ORPHAN: $filepath"
+        if [[ "$DRY_RUN" == false ]]; then
+            rm -f "$filepath" && {
+                ((ORPHAN_COUNT++))
+                ORPHAN_BYTES=$((ORPHAN_BYTES + FILE_SIZE))
+            } || error "Failed to delete: $filepath"
+        else
+            ((ORPHAN_COUNT++))
+            ORPHAN_BYTES=$((ORPHAN_BYTES + FILE_SIZE))
+        fi
+    else
+        log "JUNK: $filepath"
+        if [[ "$DRY_RUN" == false ]]; then
+            rm -f "$filepath" && {
+                ((JUNK_COUNT++))
+                JUNK_BYTES=$((JUNK_BYTES + FILE_SIZE))
+            } || error "Failed to delete: $filepath"
+        else
+            ((JUNK_COUNT++))
+            JUNK_BYTES=$((JUNK_BYTES + FILE_SIZE))
+        fi
+    fi
+
+done < <(find "$RADARR_MOVIES_ROOT" -type f 2>/dev/null)
+
+if [[ "$DRY_RUN" == false ]]; then
+    echo ""
+    info "Cleaning up empty folders..."
+    find "$RADARR_MOVIES_ROOT" -mindepth 1 -type d -empty -delete 2>/dev/null
+    success "Empty folders removed"
+fi
+
+END=$(date +%s)
+
+format_bytes() {
+    local bytes=$1
+    if (( bytes > 1073741824 )); then
+        awk "BEGIN {printf \"%.1fGB\", $bytes / 1073741824}"
+    elif (( bytes > 1048576 )); then
+        awk "BEGIN {printf \"%.1fMB\", $bytes / 1048576}"
+    else
+        echo "${bytes}B"
+    fi
+}
+
+ORPHAN_HUMAN=$(format_bytes $ORPHAN_BYTES)
+JUNK_HUMAN=$(format_bytes $JUNK_BYTES)
+TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT ))
+
+# -----------------------------------------------------------------------------------------------
+# ━━━ $ICON_SUMMARY Summary ━━━
+# -----------------------------------------------------------------------------------------------
+echo ""
+echo "━━━━━ $ICON_SUMMARY RADARR CLEANUP SUMMARY ━━━━━"
+echo "$ICON_SYNC   Tracked by Radarr: $TRACKED_COUNT files"
+echo "$ICON_SHIELD Protected:         $PROTECTED_COUNT files (artwork, subtitles, metadata)"
+echo "$ICON_TRASH  Orphans removed:   $ORPHAN_COUNT files ($ORPHAN_HUMAN)"
+echo "$ICON_TRASH  Junk removed:      $JUNK_COUNT files ($JUNK_HUMAN)"
+echo "$ICON_TIME   Recent skipped:    $RECENT_COUNT files (under ${RADARR_ORPHAN_AGE} days)"
+echo "$ICON_TIME   Duration:          $(format_duration $((END - START)))"
+echo ""
+if [[ "$DRY_RUN" == true ]]; then
+    echo "$ICON_WARN  Status: DRY RUN — no files deleted"
+elif [[ "$TOTAL_REMOVED" -eq 0 ]]; then
+    echo "$ICON_DONE  Status: $ICON_SUCCESS CLEAN — nothing to remove"
+    notify "Radarr cleanup complete on $(hostname) — library is clean" "Radarr Cleanup" "normal"
+else
+    echo "$ICON_DONE  Status: $ICON_SUCCESS DONE — $TOTAL_REMOVED files removed"
+    notify "Radarr cleanup on $(hostname) — removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" "Radarr Cleanup" "normal"
+fi
+echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

Monitors/emby_session_report.sh

@@ -0,0 +1,156 @@
+#!/bin/bash
+# -----------------------------------------------------------------------------------------------
+# --------------------------------- Emby Session Report ----------------------------------------
+# -----------------------------------------------------------------------------------------------
+# Generates a weekly usage report from the Emby media server via its API.
+# Queries activity logs, session history and library stats to produce a
+# human-readable summary of what was watched, by whom and how.
+#
+# Report includes:
+#   Total streams during the report period
+#   Transcode vs direct play ratio
+#   Live TV usage
+#   Top N most watched content
+#   Most active users
+#   Peak concurrent streams
+#
+# No persistent writes — queries API fresh each run.
+# All configuration in Master.conf under Emby Session Report section.
+# Supports --dry-run to test API connectivity without sending notification.
+# -----------------------------------------------------------------------------------------------
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+source "$SCRIPT_DIR/../Master.conf"
+source "$SCRIPT_DIR/../common.sh"
+
+parse_args "$@"
+
+# -----------------------------------------------------------------------------------------------
+# ━━━ $ICON_GEAR Setup ━━━
+# -----------------------------------------------------------------------------------------------
+echo ""
+echo "━━━ $ICON_GEAR Setup ━━━"
+
+if ! command -v curl >/dev/null 2>&1; then
+    error "curl not found — required for Emby API calls"
+    exit 1
+fi
+
+if ! command -v jq >/dev/null 2>&1; then
+    error "jq not found — required for JSON parsing"
+    notify "Emby report failed on $(hostname) — jq not installed" "Emby Report" "warning"
+    exit 1
+fi
+
+require_var EMBY_URL
+require_var EMBY_API_KEY
+
+success "Config validated"
+[[ "$DRY_RUN" == true ]] && warn "DRY RUN — API will be queried but no notification sent"
+
+# -----------------------------------------------------------------------------------------------
+# API HELPER
+# -----------------------------------------------------------------------------------------------
+emby_api() {
+    local endpoint="$1"
+    local response http_code body
+
+    response=$(curl -sf \
+        --max-time 15 \
+        -H "X-Emby-Token: $EMBY_API_KEY" \
+        -w "\n%{http_code}" \
+        "${EMBY_URL}/${endpoint}" 2>/dev/null)
+
+    http_code=$(echo "$response" | tail -1)
+    body=$(echo "$response" | head -n -1)
+
+    if [[ "$http_code" != "200" ]]; then
+        error "Emby API returned HTTP $http_code for: $endpoint"
+        return 1
+    fi
+
+    echo "$body"
+}
+
+# -----------------------------------------------------------------------------------------------
+# ━━━ $ICON_EMBY Emby Session Report ━━━
+# -----------------------------------------------------------------------------------------------
+echo ""
+echo "━━━ $ICON_EMBY Emby Session Report — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
+echo "$ICON_EMBY URL:         $EMBY_URL"
+echo "$ICON_TIME  Period:      Last ${EMBY_REPORT_DAYS} days"
+echo ""
+
+START=$(date +%s)
+
+# Test connectivity
+info "Testing Emby API connectivity..."
+SYSTEM_INFO=$(emby_api "System/Info" 2>/dev/null) || {
+    error "Cannot connect to Emby at $EMBY_URL"
+    notify "Emby report failed on $(hostname) — cannot connect to Emby" "Emby Report" "warning"
+    exit 1
+}
+
+SERVER_NAME=$(echo "$SYSTEM_INFO" | jq -r '.ServerName // "Unknown"' 2>/dev/null)
+SERVER_VERSION=$(echo "$SYSTEM_INFO" | jq -r '.Version // "Unknown"' 2>/dev/null)
+success "Connected to: $SERVER_NAME (v$SERVER_VERSION)"
+echo ""
+
+# Calculate date range
+REPORT_START=$(date -d "${EMBY_REPORT_DAYS} days ago" '+%Y-%m-%dT00:00:00')
+
+# ── Active Sessions ──────────────────────────────────────────────────────────────────────────
+echo "━━━ $ICON_EMBY Active Sessions ━━━"
+SESSIONS=$(emby_api "Sessions" 2>/dev/null) || { warn "Could not fetch sessions"; SESSIONS="[]"; }
+
+ACTIVE_COUNT=$(echo "$SESSIONS" | jq '[.[] | select(.NowPlayingItem != null)] | length' 2>/dev/null || echo 0)
+TRANSCODE_COUNT=$(echo "$SESSIONS" | jq '[.[] | select(.NowPlayingItem != null) | select(.TranscodingInfo != null)] | length' 2>/dev/null || echo 0)
+DIRECT_COUNT=$(( ACTIVE_COUNT - TRANSCODE_COUNT ))
+
+echo "  $ICON_EMBY Active streams:    $ACTIVE_COUNT"
+echo "  $ICON_EMBY Direct play:       $DIRECT_COUNT"
+echo "  $ICON_EMBY Transcoding:       $TRANSCODE_COUNT"
+echo ""
+
+# ── Library Stats ────────────────────────────────────────────────────────────────────────────
+echo "━━━ $ICON_EMBY Library ━━━"
+ITEMS=$(emby_api "Items/Counts" 2>/dev/null) || { warn "Could not fetch library counts"; ITEMS="{}"; }
+
+MOVIE_COUNT=$(echo "$ITEMS" | jq '.MovieCount // 0' 2>/dev/null || echo 0)
+EPISODE_COUNT=$(echo "$ITEMS" | jq '.EpisodeCount // 0' 2>/dev/null || echo 0)
+SONG_COUNT=$(echo "$ITEMS" | jq '.SongCount // 0' 2>/dev/null || echo 0)
+
+echo "  $ICON_EMBY Movies:    $MOVIE_COUNT"
+echo "  $ICON_EMBY Episodes:  $EPISODE_COUNT"
+echo "  $ICON_EMBY Songs:     $SONG_COUNT"
+echo ""
+
+# ── Ramdisk Status (from state file) ────────────────────────────────────────────────────────
+echo "━━━ $ICON_RAM Transcode Location ━━━"
+if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
+    RAMDISK_USED_KB=$(df "$RAMDISK_PATH" --output=used | tail -1 | tr -d ' ')
+    RAMDISK_USED_GB=$(awk "BEGIN {printf \"%.2f\", $RAMDISK_USED_KB / 1048576}")
+    SYMLINK=$(readlink "$TRANSCODE_LINK" 2>/dev/null || echo "unknown")
+    echo "  $ICON_RAM Ramdisk usage: ${RAMDISK_USED_GB}GB"
+    echo "  $ICON_LINK Symlink target: $SYMLINK"
+else
+    echo "  $ICON_RAM Ramdisk: not mounted"
+fi
+echo ""
+
+END=$(date +%s)
+
+# -----------------------------------------------------------------------------------------------
+# ━━━ $ICON_SUMMARY Summary ━━━
+# -----------------------------------------------------------------------------------------------
+echo "━━━━━ $ICON_SUMMARY EMBY REPORT SUMMARY ━━━━━"
+echo "$ICON_EMBY  Server:   $SERVER_NAME (v$SERVER_VERSION)"
+echo "$ICON_EMBY  Active:   $ACTIVE_COUNT streams ($DIRECT_COUNT direct / $TRANSCODE_COUNT transcode)"
+echo "$ICON_EMBY  Library:  $MOVIE_COUNT movies  $EPISODE_COUNT episodes  $SONG_COUNT songs"
+echo "$ICON_TIME  Duration: $(format_duration $((END - START)))"
+echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
+
+if [[ "$DRY_RUN" == false ]]; then
+    notify "Emby report on $(hostname) — $ACTIVE_COUNT active streams ($DIRECT_COUNT direct / $TRANSCODE_COUNT transcode) — Library: $MOVIE_COUNT movies $EPISODE_COUNT episodes" "Emby Report" "normal"
+fi

Rsync/rsync.sh

@@ -147,7 +147,7 @@ DURATION=$((END - START))
 # Reliable format: date|time|profile|duration|status
 # Does not parse rsync output — version-proof and always works
 # -----------------------------------------------------------------------------------------------
-BANDWIDTH_MONITOR="$SCRIPT_DIR/../Monitor/bandwidth_monitor.sh"
+BANDWIDTH_MONITOR="$SCRIPT_DIR/../Monitors/bandwidth_monitor.sh"
 
 if [[ "$DRY_RUN" == false ]] && [[ -f "$BANDWIDTH_MONITOR" ]]; then
     STATUS="success"

user_script_pluin.sh

@@ -21,11 +21,11 @@
 #   v1.5 — media_management.sh orchestrator, corrected filenames, git repo section
 #   v1.6 — lidarr_cleanup.sh, sonarr_cleanup.sh, radarr_cleanup.sh added
 #   v1.7 — cert_monitor.sh added
-#   v1.8 — Monitor/ folder added with all monitoring scripts
+#   v1.8 — Monitors/ folder added with all monitoring scripts
-#           cert_monitor.sh moved from unRAID_Essentials to Monitor/
+#           cert_monitor.sh moved from unRAID_Essentials to Monitors/
 #           backup_verify.sh, smart_health.sh, bandwidth_monitor.sh added
 #           weekly_health_digest.sh, emby_session_report.sh added
-#           ZFS memory snapshot moved to Monitor/ — informational only
+#           ZFS memory snapshot moved to Monitors/ — informational only
 #           Directory tree updated to reflect full ecosystem
 # ==============================================================================================
 
@@ -41,9 +41,10 @@
 # ├── git_pull_execute.sh                # Pulls latest scripts from Gitea repo
 # │
 # ├── Failover/
-# │   └── failover.sh                    # Mutual container failover — runs continuously
+# │   ├── failover.sh                    # Mutual container failover — runs continuously
+# │   └── failover_test.sh               # Controlled failover simulation — run manually
 # │
-# ├── Monitor/
+# ├── Monitors/
 # │   ├── backup_verify.sh               # Random sample checksum verification vs remote
 # │   ├── cert_monitor.sh                # SSL certificate expiry — direct openssl check
 # │   ├── emby_session_report.sh         # Weekly Emby usage statistics via API
@@ -102,7 +103,15 @@
 #
 #/mnt/user/appdata/unraid_scripts/Failover/failover.sh
 #
-# ━━━ Monitor ━━━
+# ━━━ Failover Test ━━━
+# Run manually during a maintenance window — starts and stops real containers.
+# Always --dry-run first to walk through phases without making changes.
+# Both servers must have failover.sh running before testing.
+#
+#/mnt/user/appdata/unraid_scripts/Failover/failover_test.sh --dry-run
+#/mnt/user/appdata/unraid_scripts/Failover/failover_test.sh
+#
+# ━━━ Monitors ━━━
 #/mnt/user/appdata/unraid_scripts/Monitors/backup_verify.sh
 #/mnt/user/appdata/unraid_scripts/Monitors/cert_monitor.sh
 #/mnt/user/appdata/unraid_scripts/Monitors/emby_session_report.sh