diff --git a/Failover/failover_test.sh b/Failover/failover_test.sh new file mode 100644 index 0000000..075075f --- /dev/null +++ b/Failover/failover_test.sh @@ -0,0 +1,389 @@ +#!/bin/bash +# ----------------------------------------------------------------------------------------------- +# --------------------------------- Failover Test ---------------------------------------------- +# ----------------------------------------------------------------------------------------------- +# Controlled simulation of the failover scenario — validates the entire failover lifecycle +# without waiting for a real outage. +# +# This script is a TEST HARNESS only — it does not contain failover logic. +# All failover logic lives in failover.sh and is called directly from here. +# Any changes to failover.sh are automatically reflected in this test. +# +# Test sequence: +# 1. Pre-flight — verify both servers reachable, failover.sh exists, state is NORMAL +# 2. Block — add iptables rule dropping all traffic to remote IP +# 3. Detect — run failover.sh one cycle — confirm FAILOVER state detected +# 4. Start — verify failover containers started locally +# 5. Restore — remove iptables rule, remote becomes reachable again +# 6. Handback — wait for failover.sh to confirm handback strikes and hand back +# 7. Verify — confirm containers returned to remote, local copies stopped +# 8. Report — full pass/fail summary per phase +# +# Safety: iptables rule is removed via trap on ANY exit — crash, error, ctrl-c, or normal. +# Remote connectivity is always restored regardless of test outcome. +# +# ⚠️ This script starts and stops real containers on both servers. +# Run during a maintenance window — users will experience a brief service interruption. +# Use --dry-run to walk through the sequence without touching containers or iptables. +# +# All configuration in Master.conf under Failover and Failover Test sections. +# ----------------------------------------------------------------------------------------------- + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "$SCRIPT_DIR/../Master.conf" +source "$SCRIPT_DIR/../common.sh" + +parse_args "$@" + +FAILOVER_SCRIPT="$SCRIPT_DIR/failover.sh" + +# ----------------------------------------------------------------------------------------------- +# SAFETY TRAP — always remove iptables rule on exit +# Fires on normal exit, error exit, ctrl-c, and script crashes +# ----------------------------------------------------------------------------------------------- +IPTABLES_RULE_ACTIVE=false + +cleanup() { + if [[ "$IPTABLES_RULE_ACTIVE" == true ]]; then + echo "" + warn "$ICON_SHIELD Cleanup — removing iptables block on $REMOTE_SERVER..." + if [[ "$DRY_RUN" == false ]]; then + iptables -D OUTPUT -d "$REMOTE_SERVER" -j DROP 2>/dev/null + IPTABLES_RULE_ACTIVE=false + success "iptables rule removed — remote connectivity restored" + else + warn "DRY RUN — would remove iptables rule" + fi + fi +} + +trap cleanup EXIT + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_GEAR Setup ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_GEAR Setup ━━━" + +if [[ "$EUID" -ne 0 ]]; then + error "Must be run as root" + exit 1 +fi + +success "Running as root" + +if ! command -v iptables >/dev/null 2>&1; then + error "iptables not found — required for connectivity simulation" + exit 1 +fi + +success "iptables available" + +if [[ ! -f "$FAILOVER_SCRIPT" ]]; then + error "failover.sh not found at $FAILOVER_SCRIPT" + exit 1 +fi + +success "failover.sh found" + +detect_hosts +resolve_remote_ip + +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no iptables rules or container changes will be made" + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_SUMMARY Status ━━━ +# ----------------------------------------------------------------------------------------------- +if [[ "$SHOW_STATUS" == true ]]; then + echo "" + echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" + echo "$ICON_HOST Local: $LOCAL_SERVER_NAME" + echo "$ICON_HOST Remote: $REMOTE_SERVER_NAME ($REMOTE_SERVER)" + echo "$ICON_FAILOVER Block wait: ${FAILOVER_TEST_BLOCK_WAIT}s" + echo "$ICON_FAILOVER Handback wait: ${FAILOVER_TEST_HANDBACK_WAIT}s" + echo "$ICON_GEAR Dry Run: $DRY_RUN" + + # Current failover state + if [[ -f "$FAILOVER_STATE_FILE" ]]; then + CURRENT_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2) + echo "$ICON_FAILOVER Current state: ${CURRENT_STATE:-unknown}" + else + echo "$ICON_FAILOVER Current state: no state file" + fi + echo "━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +# ----------------------------------------------------------------------------------------------- +# PHASE TRACKING +# ----------------------------------------------------------------------------------------------- +PHASES_PASS=() +PHASES_FAIL=() +TOTAL_START=$(date +%s) + +phase_pass() { PHASES_PASS+=("$1"); success "$ICON_DONE Phase: $1 — PASSED"; } +phase_fail() { PHASES_FAIL+=("$1"); error "$ICON_ERROR Phase: $1 — FAILED"; } + +# ----------------------------------------------------------------------------------------------- +# ━━━ PHASE 1 — Pre-flight ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo " $ICON_SHIELD FAILOVER TEST — $(date '+%Y-%m-%d %H:%M:%S')" +echo " $ICON_HOST Local: $LOCAL_SERVER_NAME" +echo " $ICON_HOST Remote: $REMOTE_SERVER_NAME ($REMOTE_SERVER)" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo "" +echo "━━━ $ICON_SHIELD Phase 1 — Pre-flight ━━━" + +# Check remote reachable +info "Checking remote reachability..." +if ping_remote; then + success "Remote $REMOTE_SERVER_NAME is reachable" +else + error "Remote $REMOTE_SERVER_NAME is not reachable — cannot run test" + phase_fail "Pre-flight" + exit 1 +fi + +# Check internet reachable +info "Checking internet connectivity..." +if ping_internet; then + success "Internet is reachable" +else + error "No internet connectivity — cannot run test" + phase_fail "Pre-flight" + exit 1 +fi + +# Check current failover state is NORMAL +if [[ -f "$FAILOVER_STATE_FILE" ]]; then + CURRENT_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2) + if [[ "$CURRENT_STATE" != "NORMAL" ]]; then + error "Failover state is $CURRENT_STATE — must be NORMAL before running test" + phase_fail "Pre-flight" + exit 1 + fi + success "Failover state is NORMAL" +else + warn "No state file found — assuming NORMAL (first run)" +fi + +phase_pass "Pre-flight" + +# ----------------------------------------------------------------------------------------------- +# ━━━ PHASE 2 — Block Remote ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_PING Phase 2 — Block Remote Connectivity ━━━" +warn "Adding iptables rule — dropping all traffic to $REMOTE_SERVER" + +if [[ "$DRY_RUN" == false ]]; then + iptables -I OUTPUT -d "$REMOTE_SERVER" -j DROP + IPTABLES_RULE_ACTIVE=true + success "iptables rule active — $REMOTE_SERVER_NAME appears unreachable" + + # Verify block is working + if ! ping -c1 -W2 "$REMOTE_SERVER" &>/dev/null; then + success "Connectivity block confirmed — ping to remote fails as expected" + phase_pass "Block Remote" + else + error "iptables rule did not block connectivity — ping still succeeds" + phase_fail "Block Remote" + exit 1 + fi +else + warn "DRY RUN — would block $REMOTE_SERVER with iptables" + phase_pass "Block Remote" +fi + +# ----------------------------------------------------------------------------------------------- +# ━━━ PHASE 3 — Failover Detection ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_FAILOVER Phase 3 — Failover Detection ━━━" +info "Waiting ${FAILOVER_TEST_BLOCK_WAIT}s for failover.sh to detect outage..." +info "failover.sh check interval is ${FAILOVER_CHECK_INTERVAL}s" + +if [[ "$DRY_RUN" == false ]]; then + sleep "$FAILOVER_TEST_BLOCK_WAIT" + + # Check state file updated to FAILOVER + if [[ -f "$FAILOVER_STATE_FILE" ]]; then + NEW_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2) + if [[ "$NEW_STATE" == "FAILOVER" ]]; then + success "State changed to FAILOVER — outage detected correctly" + phase_pass "Failover Detection" + else + error "State is $NEW_STATE — expected FAILOVER after ${FAILOVER_TEST_BLOCK_WAIT}s" + warn "failover.sh may not be running — check User Scripts plugin" + phase_fail "Failover Detection" + fi + else + error "No state file found after wait — failover.sh may not be running" + phase_fail "Failover Detection" + fi +else + warn "DRY RUN — would wait ${FAILOVER_TEST_BLOCK_WAIT}s and check for FAILOVER state" + phase_pass "Failover Detection" +fi + +# ----------------------------------------------------------------------------------------------- +# ━━━ PHASE 4 — Container Start Verification ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_CONTAINERS Phase 4 — Failover Containers Started ━━━" + +# Determine which containers should have started on this host +if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then + EXPECTED_CONTAINERS=("${FAILOVER_HOST1_STARTS_FOR_HOST2[@]}") +else + EXPECTED_CONTAINERS=("${FAILOVER_HOST2_STARTS_FOR_HOST1[@]}") +fi + +if [[ "$DRY_RUN" == false ]]; then + CONTAINERS_OK=true + for container in "${EXPECTED_CONTAINERS[@]}"; do + [[ -z "$container" ]] && continue + STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null) + if [[ "$STATUS" == "true" ]]; then + success "$ICON_RUNNING $container is running locally" + else + error "$ICON_NOT_RUNNING $container is NOT running locally" + CONTAINERS_OK=false + fi + done + + if [[ "$CONTAINERS_OK" == true ]]; then + phase_pass "Container Start" + else + phase_fail "Container Start" + fi +else + warn "DRY RUN — would verify these containers started: ${EXPECTED_CONTAINERS[*]}" + phase_pass "Container Start" +fi + +# ----------------------------------------------------------------------------------------------- +# ━━━ PHASE 5 — Restore Connectivity ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_PING Phase 5 — Restore Remote Connectivity ━━━" +info "Removing iptables block — remote becomes reachable again" + +if [[ "$DRY_RUN" == false ]]; then + iptables -D OUTPUT -d "$REMOTE_SERVER" -j DROP 2>/dev/null + IPTABLES_RULE_ACTIVE=false + success "iptables rule removed" + + # Verify connectivity restored + sleep 3 + if ping_remote; then + success "Remote $REMOTE_SERVER_NAME is reachable again" + phase_pass "Restore Connectivity" + else + error "Remote still unreachable after removing iptables rule" + phase_fail "Restore Connectivity" + fi +else + warn "DRY RUN — would remove iptables rule" + phase_pass "Restore Connectivity" +fi + +# ----------------------------------------------------------------------------------------------- +# ━━━ PHASE 6 — Handback ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_FAILOVER Phase 6 — Handback ━━━" +info "Waiting ${FAILOVER_TEST_HANDBACK_WAIT}s for failover.sh to confirm handback..." +info "Requires $FAILOVER_HANDBACK_STRIKES consecutive remote-up checks at ${FAILOVER_CHECK_INTERVAL}s intervals" +info "Estimated minimum wait: $(( FAILOVER_HANDBACK_STRIKES * FAILOVER_CHECK_INTERVAL ))s" + +if [[ "$DRY_RUN" == false ]]; then + sleep "$FAILOVER_TEST_HANDBACK_WAIT" + + if [[ -f "$FAILOVER_STATE_FILE" ]]; then + FINAL_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2) + if [[ "$FINAL_STATE" == "NORMAL" ]]; then + success "State returned to NORMAL — handback completed" + phase_pass "Handback" + else + error "State is $FINAL_STATE — expected NORMAL after handback wait" + phase_fail "Handback" + fi + else + error "No state file found" + phase_fail "Handback" + fi +else + warn "DRY RUN — would wait ${FAILOVER_TEST_HANDBACK_WAIT}s and verify NORMAL state" + phase_pass "Handback" +fi + +# ----------------------------------------------------------------------------------------------- +# ━━━ PHASE 7 — Container Handback Verification ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_CONTAINERS Phase 7 — Failover Containers Stopped Locally ━━━" + +if [[ "$DRY_RUN" == false ]]; then + HANDBACK_OK=true + for container in "${EXPECTED_CONTAINERS[@]}"; do + [[ -z "$container" ]] && continue + STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null) + if [[ "$STATUS" != "true" ]]; then + success "$ICON_NOT_RUNNING $container stopped locally — handed back" + else + error "$ICON_RUNNING $container still running locally — handback may have failed" + HANDBACK_OK=false + fi + done + + if [[ "$HANDBACK_OK" == true ]]; then + phase_pass "Container Handback" + else + phase_fail "Container Handback" + fi +else + warn "DRY RUN — would verify failover containers stopped locally after handback" + phase_pass "Container Handback" +fi + +TOTAL_END=$(date +%s) + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_SUMMARY Test Report ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━━━ $ICON_SUMMARY FAILOVER TEST REPORT ━━━━━" +echo "$ICON_HOST Local: $LOCAL_SERVER_NAME" +echo "$ICON_HOST Remote: $REMOTE_SERVER_NAME" +echo "$ICON_TIME Duration: $(format_duration $((TOTAL_END - TOTAL_START)))" +echo "" +echo " Phase Results:" +for phase in "${PHASES_PASS[@]}"; do + echo " $ICON_SUCCESS $phase" +done +for phase in "${PHASES_FAIL[@]}"; do + echo " $ICON_ERROR $phase" +done +echo "" + +PASS_COUNT=${#PHASES_PASS[@]} +FAIL_COUNT=${#PHASES_FAIL[@]} +TOTAL_PHASES=$(( PASS_COUNT + FAIL_COUNT )) + +if [[ "$DRY_RUN" == true ]]; then + echo "$ICON_WARN Status: DRY RUN — no changes made" +elif [[ "$FAIL_COUNT" -eq 0 ]]; then + echo "$ICON_DONE Status: $ICON_SUCCESS ALL $TOTAL_PHASES PHASES PASSED" + notify "Failover test PASSED on $(hostname) — all $TOTAL_PHASES phases completed successfully" "Failover Test" "normal" +else + echo "$ICON_ERROR Status: $FAIL_COUNT/$TOTAL_PHASES PHASES FAILED" + notify "Failover test FAILED on $(hostname) — $FAIL_COUNT/$TOTAL_PHASES phases failed: ${PHASES_FAIL[*]}" "Failover Test" "warning" +fi +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + +[[ "$FAIL_COUNT" -gt 0 ]] && exit 1 +exit 0 \ No newline at end of file diff --git a/Master.conf b/Master.conf index 5200d12..813f39e 100644 --- a/Master.conf +++ b/Master.conf @@ -53,7 +53,7 @@ # ── TRANSCODES ───────────────────────────────────────────────────────────────────────────── # TRANSCODE MANAGER Ramdisk and SSD fallback transcode management # -# ── MONITOR ──────────────────────────────────────────────────────────────────────────────── +# ── MONITORS ──────────────────────────────────────────────────────────────────────────────── # CERTIFICATE MONITOR SSL certificate expiry monitoring # BACKUP VERIFY Random sample checksum verification against remote # SMART HEALTH Drive SMART attribute monitoring @@ -294,83 +294,64 @@ declare -A PROFILE_SKIP_DISK_CHECK=( # Each server runs Failover/failover.sh independently — no coordination between servers. # All decisions are based solely on two ping checks: remote reachable + internet reachable. # -# How it works: -# Every FAILOVER_CHECK_INTERVAL seconds each server pings the remote and pings the internet. -# Based on those two results it determines its state and takes the appropriate action. -# No SSH, no signaling — each server acts autonomously based only on what it can see. -# # States: # NORMAL — remote up, internet up — own containers only, silent operation # FAILOVER — remote down, internet up — start remote's containers locally (additive) -# Own normal containers keep running — failover containers added on top # NO_INTERNET — internet down — stop public-facing containers, wait for recovery +# DARK — remote down + internet down — same actions as NO_INTERNET # -# Handback sequence when remote returns after FAILOVER: -# 1. Strike confirmation — FAILOVER_HANDBACK_STRIKES consecutive remote-up checks -# Prevents handing back during a brief network blip -# 2. Pre-flight checks — remote array started, Docker daemon healthy, rootfs not full -# 3. Rsync data back via rsync.sh — uses existing profile system for options -# 4. Start failover containers on remote via SSH -# 5. Stop failover containers locally — only after remote confirmed started -# 6. Return to NORMAL state -# -# Script runs on BOTH servers — detect_hosts() selects the correct arrays automatically. -# Comment out any container or rsync job to disable without removing the entry. - +# Handback: strike confirmation → pre-flight → rsync → start remote → stop local + EXTERNAL_IP="8.8.8.8" # external IP to ping for internet connectivity check - FAILOVER_CHECK_INTERVAL=120 # seconds between state checks — 120s = 2 minute polling - FAILOVER_HANDBACK_STRIKES=2 # consecutive remote-up confirmations required before handback - # 2 strikes at 120s interval = 4 minutes confirmation window + FAILOVER_CHECK_INTERVAL=120 # seconds between state checks + FAILOVER_HANDBACK_STRIKES=2 # consecutive remote-up confirmations before handback FAILOVER_STATE_FILE="/boot/config/failover_state.db" - # persists on /boot/ so it survives reboots - # script re-evaluates from live pings on restart - -# ━━━ HOST1 Failover Config (unRAID-Gmer4Lfe — Primary) ━━━ - -# Containers HOST1 starts locally when HOST2 goes down. -# These run ON TOP OF HOST1's normal containers — additive, not a replacement. + +# ━━━ Failover Test ━━━ +# Used by Failover/failover_test.sh — controlled simulation of the failover lifecycle. +# failover_test.sh blocks remote connectivity via iptables then observes failover.sh behavior. +# All failover logic stays in failover.sh — test script is the harness only. +# +# ⚠️ Run during a maintenance window — real containers start and stop during the test. +# Use --dry-run first to walk through phases without touching anything. + + # Seconds to hold the iptables block — must be longer than FAILOVER_CHECK_INTERVAL + # so failover.sh has time to detect the outage and change state + FAILOVER_TEST_BLOCK_WAIT=150 # 150s = FAILOVER_CHECK_INTERVAL + 30s buffer + + # Seconds to wait for handback after restoring connectivity + # Must cover FAILOVER_HANDBACK_STRIKES x FAILOVER_CHECK_INTERVAL plus rsync time + # 2 strikes x 120s = 240s minimum — add buffer for rsync handback jobs + FAILOVER_TEST_HANDBACK_WAIT=360 # 360s = 6 minutes — adjust if rsync takes longer + + FAILOVER_HOST1_STARTS_FOR_HOST2=( "Gmer4Lfe.com" "Gmer4Lfe.us" ) - -# Containers HOST1 stops when it loses internet connectivity. -# No point serving DDNS or public services if HOST1 itself has no internet. FAILOVER_HOST1_STOP_ON_NO_NET=( "Gmer4Lfe.com" "Gmer4Lfe.us" ) - -# Rsync jobs HOST1 runs before handing containers back to HOST2 after recovery. -# Uses rsync.sh with the existing profile system — basename matched to profile keys. -# Comment out jobs that are not yet ready or not needed for handback. FAILOVER_HOST1_RSYNC_JOBS=( # "/mnt/user/appdata-Failover/Jayred365" # "/mnt/user/Media_Server/Emby-Jayred" ) - -# ━━━ HOST2 Failover Config (unRAID-Jayred365 — Secondary) ━━━ - -# Containers HOST2 starts locally when HOST1 goes down. -# Emby starts on HOST2 so media keeps working during HOST1 outage. -# HOST1's DDNS containers start here so DNS updates to point at HOST2's IP. + +# HOST2 (unRAID-Jayred365 — Secondary) FAILOVER_HOST2_STARTS_FOR_HOST1=( "Emby" "Gmer4Lfe.com" "Gmer4Lfe.us" ) - -# Containers HOST2 stops when it loses internet connectivity. FAILOVER_HOST2_STOP_ON_NO_NET=( "Gmer4Lfe.com" "Gmer4Lfe.us" ) - -# Rsync jobs HOST2 runs before handing containers back to HOST1 after recovery. FAILOVER_HOST2_RSYNC_JOBS=( # "/mnt/user/appdata-Failover/Gmer4Lfe" ) - + # ============================================================================================== # ── DOCKER ESSENTIALS ───────────────────────────────────────────────────────────────────────── # ============================================================================================== @@ -702,7 +683,7 @@ MEDIA_MAINTENANCE_JOBS=( TRANSCODE_EMBY_CONTAINER="Emby" # exact Docker container name — case sensitive # ============================================================================================== -# ── MONITOR ─────────────────────────────────────────────────────────────────────────────────── +# ── MONITORS ─────────────────────────────────────────────────────────────────────────────────── # ============================================================================================== # Monitoring scripts — watch and report only, never take action. # Lives in Monitor/ folder — distinct from unRAID_Essentials (which acts) and diff --git a/Media/radarr_cleanup.sh b/Media/radarr_cleanup.sh index e69de29..8fa84ec 100644 --- a/Media/radarr_cleanup.sh +++ b/Media/radarr_cleanup.sh @@ -0,0 +1,281 @@ +#!/bin/bash +# ----------------------------------------------------------------------------------------------- +# --------------------------------- Radarr Cleanup Script -------------------------------------- +# ----------------------------------------------------------------------------------------------- +# Removes orphaned movie files from the library that Radarr no longer tracks. +# Uses the Radarr API to build a complete list of tracked movie file paths then compares +# against what exists on disk — anything not tracked and older than RADARR_ORPHAN_AGE +# days is considered an orphan and deleted. +# +# File classification: +# TRACKED — Radarr API knows about this exact file path → leave it alone +# PROTECTED — matches RADARR_PROTECTED_PATTERNS → never delete (artwork, subtitles, .nfo) +# ORPHAN — video file, not tracked, older than RADARR_ORPHAN_AGE days → delete +# JUNK — not a video extension, not protected → delete regardless of age +# RECENT — not tracked, under RADARR_ORPHAN_AGE days old → skip (may be mid-import) +# +# Why protected patterns matter: +# Radarr generates movie artwork (*.jpg), metadata (*.nfo) and manages subtitles +# (*.srt, *.sub, *.ass) but does not include these in its tracked file API response. +# Without protection these would be classified as orphans and deleted. +# +# All configuration in Master.conf under Arr Cleanup section. +# Supports --dry-run to preview what would be deleted without making changes. +# ----------------------------------------------------------------------------------------------- + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "$SCRIPT_DIR/../Master.conf" +source "$SCRIPT_DIR/../common.sh" + +parse_args "$@" + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_GEAR Setup ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_GEAR Setup ━━━" + +if [[ "$EUID" -ne 0 ]]; then + error "Must be run as root" + exit 1 +fi + +success "Running as root" + +if ! command -v curl >/dev/null 2>&1; then + error "curl not found — required for Radarr API calls" + exit 1 +fi + +if ! command -v jq >/dev/null 2>&1; then + error "jq not found — required for JSON parsing" + notify "Radarr cleanup failed on $(hostname) — jq not installed" "Radarr Cleanup" "warning" + exit 1 +fi + +require_var RADARR_URL +require_var RADARR_API_KEY +require_var RADARR_MOVIES_ROOT + +if [[ ! -d "$RADARR_MOVIES_ROOT" ]]; then + error "Movies root not found: $RADARR_MOVIES_ROOT" + notify "Radarr cleanup failed on $(hostname) — movies root not found: $RADARR_MOVIES_ROOT" "Radarr Cleanup" "warning" + exit 1 +fi + +success "Config validated" + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_SUMMARY Status ━━━ +# ----------------------------------------------------------------------------------------------- +if [[ "$SHOW_STATUS" == true ]]; then + echo "" + echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" + echo "$ICON_GEAR Radarr URL: $RADARR_URL" + echo "$ICON_GEAR Movies root: $RADARR_MOVIES_ROOT" + echo "$ICON_TIME Orphan age: ${RADARR_ORPHAN_AGE} days" + echo "$ICON_GEAR Extensions: ${RADARR_EXTENSIONS[*]}" + echo "$ICON_GEAR Protected patterns: ${RADARR_PROTECTED_PATTERNS[*]}" + echo "$ICON_GEAR Dry Run: $DRY_RUN" + echo "━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be deleted" + +# ----------------------------------------------------------------------------------------------- +# HELPERS +# ----------------------------------------------------------------------------------------------- + +radarr_api() { + local endpoint="$1" + local response http_code body + + response=$(curl -sf \ + --max-time 30 \ + -H "X-Api-Key: $RADARR_API_KEY" \ + -w "\n%{http_code}" \ + "${RADARR_URL}/api/v3/${endpoint}" 2>/dev/null) + + http_code=$(echo "$response" | tail -1) + body=$(echo "$response" | head -n -1) + + if [[ "$http_code" != "200" ]]; then + error "Radarr API returned HTTP $http_code for endpoint: $endpoint" + return 1 + fi + + echo "$body" +} + +is_video_file() { + local ext="${1##*.}" + ext="${ext,,}" + for valid_ext in "${RADARR_EXTENSIONS[@]}"; do + [[ "$ext" == "$valid_ext" ]] && return 0 + done + return 1 +} + +is_protected_file() { + local filename + filename=$(basename "$1") + for pattern in "${RADARR_PROTECTED_PATTERNS[@]}"; do + # shellcheck disable=SC2254 + case "$filename" in + $pattern) return 0 ;; + esac + done + return 1 +} + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_SYNC Fetching Radarr Tracked Files ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_SYNC Fetching Radarr Tracked Files ━━━" + +info "Querying Radarr API: $RADARR_URL" + +MOVIEFILE_RESPONSE=$(radarr_api "moviefile") || { + error "Failed to fetch movie files from Radarr — check URL and API key" + notify "Radarr cleanup failed on $(hostname) — API unreachable" "Radarr Cleanup" "warning" + exit 1 +} + +TMP_DIR="/tmp/radarr_cleanup_$$" +mkdir -p "$TMP_DIR" +trap "rm -rf $TMP_DIR" EXIT + +TRACKED_FILE="$TMP_DIR/tracked_paths.txt" +echo "$MOVIEFILE_RESPONSE" | jq -r '.[].path' 2>/dev/null | sort > "$TRACKED_FILE" + +TRACKED_COUNT=$(wc -l < "$TRACKED_FILE") +success "Radarr tracks $TRACKED_COUNT movie files" + +if [[ "$TRACKED_COUNT" -eq 0 ]]; then + warn "No tracked files returned — Radarr may not have scanned yet or library is empty" + warn "Aborting to prevent mass deletion" + notify "Radarr cleanup aborted on $(hostname) — no tracked files returned from API" "Radarr Cleanup" "warning" + exit 1 +fi + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_CLEAN Scanning Movies Root ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_CLEAN Scanning Movies Root ━━━" +info "Root: $RADARR_MOVIES_ROOT" +info "Orphan age: ${RADARR_ORPHAN_AGE} days" +info "Protected: ${RADARR_PROTECTED_PATTERNS[*]}" +echo "" + +START=$(date +%s) +ORPHAN_COUNT=0 +JUNK_COUNT=0 +RECENT_COUNT=0 +PROTECTED_COUNT=0 +ORPHAN_BYTES=0 +JUNK_BYTES=0 + +AGE_SECONDS=$(( RADARR_ORPHAN_AGE * 86400 )) +NOW=$(date +%s) + +while IFS= read -r filepath; do + [[ -z "$filepath" ]] && continue + + if grep -qF "$filepath" "$TRACKED_FILE" 2>/dev/null; then + log "TRACKED: $filepath" + continue + fi + + if is_protected_file "$filepath"; then + log "PROTECTED: $filepath" + ((PROTECTED_COUNT++)) + continue + fi + + FILE_SIZE=$(stat -c%s "$filepath" 2>/dev/null || echo 0) + + if is_video_file "$filepath"; then + FILE_MTIME=$(stat -c %Y "$filepath" 2>/dev/null || echo 0) + FILE_AGE=$(( NOW - FILE_MTIME )) + + if [[ "$FILE_AGE" -lt "$AGE_SECONDS" ]]; then + log "RECENT (skipping): $filepath" + ((RECENT_COUNT++)) + continue + fi + + warn "$ICON_TRASH ORPHAN: $filepath" + if [[ "$DRY_RUN" == false ]]; then + rm -f "$filepath" && { + ((ORPHAN_COUNT++)) + ORPHAN_BYTES=$((ORPHAN_BYTES + FILE_SIZE)) + } || error "Failed to delete: $filepath" + else + ((ORPHAN_COUNT++)) + ORPHAN_BYTES=$((ORPHAN_BYTES + FILE_SIZE)) + fi + else + log "JUNK: $filepath" + if [[ "$DRY_RUN" == false ]]; then + rm -f "$filepath" && { + ((JUNK_COUNT++)) + JUNK_BYTES=$((JUNK_BYTES + FILE_SIZE)) + } || error "Failed to delete: $filepath" + else + ((JUNK_COUNT++)) + JUNK_BYTES=$((JUNK_BYTES + FILE_SIZE)) + fi + fi + +done < <(find "$RADARR_MOVIES_ROOT" -type f 2>/dev/null) + +if [[ "$DRY_RUN" == false ]]; then + echo "" + info "Cleaning up empty folders..." + find "$RADARR_MOVIES_ROOT" -mindepth 1 -type d -empty -delete 2>/dev/null + success "Empty folders removed" +fi + +END=$(date +%s) + +format_bytes() { + local bytes=$1 + if (( bytes > 1073741824 )); then + awk "BEGIN {printf \"%.1fGB\", $bytes / 1073741824}" + elif (( bytes > 1048576 )); then + awk "BEGIN {printf \"%.1fMB\", $bytes / 1048576}" + else + echo "${bytes}B" + fi +} + +ORPHAN_HUMAN=$(format_bytes $ORPHAN_BYTES) +JUNK_HUMAN=$(format_bytes $JUNK_BYTES) +TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT )) + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_SUMMARY Summary ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━━━ $ICON_SUMMARY RADARR CLEANUP SUMMARY ━━━━━" +echo "$ICON_SYNC Tracked by Radarr: $TRACKED_COUNT files" +echo "$ICON_SHIELD Protected: $PROTECTED_COUNT files (artwork, subtitles, metadata)" +echo "$ICON_TRASH Orphans removed: $ORPHAN_COUNT files ($ORPHAN_HUMAN)" +echo "$ICON_TRASH Junk removed: $JUNK_COUNT files ($JUNK_HUMAN)" +echo "$ICON_TIME Recent skipped: $RECENT_COUNT files (under ${RADARR_ORPHAN_AGE} days)" +echo "$ICON_TIME Duration: $(format_duration $((END - START)))" +echo "" +if [[ "$DRY_RUN" == true ]]; then + echo "$ICON_WARN Status: DRY RUN — no files deleted" +elif [[ "$TOTAL_REMOVED" -eq 0 ]]; then + echo "$ICON_DONE Status: $ICON_SUCCESS CLEAN — nothing to remove" + notify "Radarr cleanup complete on $(hostname) — library is clean" "Radarr Cleanup" "normal" +else + echo "$ICON_DONE Status: $ICON_SUCCESS DONE — $TOTAL_REMOVED files removed" + notify "Radarr cleanup on $(hostname) — removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" "Radarr Cleanup" "normal" +fi +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" \ No newline at end of file diff --git a/Monitors/emby_session_report.sh b/Monitors/emby_session_report.sh index e69de29..cfbe6ce 100644 --- a/Monitors/emby_session_report.sh +++ b/Monitors/emby_session_report.sh @@ -0,0 +1,156 @@ +#!/bin/bash +# ----------------------------------------------------------------------------------------------- +# --------------------------------- Emby Session Report ---------------------------------------- +# ----------------------------------------------------------------------------------------------- +# Generates a weekly usage report from the Emby media server via its API. +# Queries activity logs, session history and library stats to produce a +# human-readable summary of what was watched, by whom and how. +# +# Report includes: +# Total streams during the report period +# Transcode vs direct play ratio +# Live TV usage +# Top N most watched content +# Most active users +# Peak concurrent streams +# +# No persistent writes — queries API fresh each run. +# All configuration in Master.conf under Emby Session Report section. +# Supports --dry-run to test API connectivity without sending notification. +# ----------------------------------------------------------------------------------------------- + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "$SCRIPT_DIR/../Master.conf" +source "$SCRIPT_DIR/../common.sh" + +parse_args "$@" + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_GEAR Setup ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_GEAR Setup ━━━" + +if ! command -v curl >/dev/null 2>&1; then + error "curl not found — required for Emby API calls" + exit 1 +fi + +if ! command -v jq >/dev/null 2>&1; then + error "jq not found — required for JSON parsing" + notify "Emby report failed on $(hostname) — jq not installed" "Emby Report" "warning" + exit 1 +fi + +require_var EMBY_URL +require_var EMBY_API_KEY + +success "Config validated" +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — API will be queried but no notification sent" + +# ----------------------------------------------------------------------------------------------- +# API HELPER +# ----------------------------------------------------------------------------------------------- +emby_api() { + local endpoint="$1" + local response http_code body + + response=$(curl -sf \ + --max-time 15 \ + -H "X-Emby-Token: $EMBY_API_KEY" \ + -w "\n%{http_code}" \ + "${EMBY_URL}/${endpoint}" 2>/dev/null) + + http_code=$(echo "$response" | tail -1) + body=$(echo "$response" | head -n -1) + + if [[ "$http_code" != "200" ]]; then + error "Emby API returned HTTP $http_code for: $endpoint" + return 1 + fi + + echo "$body" +} + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_EMBY Emby Session Report ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_EMBY Emby Session Report — $(date '+%Y-%m-%d %H:%M:%S') ━━━" +echo "$ICON_EMBY URL: $EMBY_URL" +echo "$ICON_TIME Period: Last ${EMBY_REPORT_DAYS} days" +echo "" + +START=$(date +%s) + +# Test connectivity +info "Testing Emby API connectivity..." +SYSTEM_INFO=$(emby_api "System/Info" 2>/dev/null) || { + error "Cannot connect to Emby at $EMBY_URL" + notify "Emby report failed on $(hostname) — cannot connect to Emby" "Emby Report" "warning" + exit 1 +} + +SERVER_NAME=$(echo "$SYSTEM_INFO" | jq -r '.ServerName // "Unknown"' 2>/dev/null) +SERVER_VERSION=$(echo "$SYSTEM_INFO" | jq -r '.Version // "Unknown"' 2>/dev/null) +success "Connected to: $SERVER_NAME (v$SERVER_VERSION)" +echo "" + +# Calculate date range +REPORT_START=$(date -d "${EMBY_REPORT_DAYS} days ago" '+%Y-%m-%dT00:00:00') + +# ── Active Sessions ────────────────────────────────────────────────────────────────────────── +echo "━━━ $ICON_EMBY Active Sessions ━━━" +SESSIONS=$(emby_api "Sessions" 2>/dev/null) || { warn "Could not fetch sessions"; SESSIONS="[]"; } + +ACTIVE_COUNT=$(echo "$SESSIONS" | jq '[.[] | select(.NowPlayingItem != null)] | length' 2>/dev/null || echo 0) +TRANSCODE_COUNT=$(echo "$SESSIONS" | jq '[.[] | select(.NowPlayingItem != null) | select(.TranscodingInfo != null)] | length' 2>/dev/null || echo 0) +DIRECT_COUNT=$(( ACTIVE_COUNT - TRANSCODE_COUNT )) + +echo " $ICON_EMBY Active streams: $ACTIVE_COUNT" +echo " $ICON_EMBY Direct play: $DIRECT_COUNT" +echo " $ICON_EMBY Transcoding: $TRANSCODE_COUNT" +echo "" + +# ── Library Stats ──────────────────────────────────────────────────────────────────────────── +echo "━━━ $ICON_EMBY Library ━━━" +ITEMS=$(emby_api "Items/Counts" 2>/dev/null) || { warn "Could not fetch library counts"; ITEMS="{}"; } + +MOVIE_COUNT=$(echo "$ITEMS" | jq '.MovieCount // 0' 2>/dev/null || echo 0) +EPISODE_COUNT=$(echo "$ITEMS" | jq '.EpisodeCount // 0' 2>/dev/null || echo 0) +SONG_COUNT=$(echo "$ITEMS" | jq '.SongCount // 0' 2>/dev/null || echo 0) + +echo " $ICON_EMBY Movies: $MOVIE_COUNT" +echo " $ICON_EMBY Episodes: $EPISODE_COUNT" +echo " $ICON_EMBY Songs: $SONG_COUNT" +echo "" + +# ── Ramdisk Status (from state file) ──────────────────────────────────────────────────────── +echo "━━━ $ICON_RAM Transcode Location ━━━" +if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then + RAMDISK_USED_KB=$(df "$RAMDISK_PATH" --output=used | tail -1 | tr -d ' ') + RAMDISK_USED_GB=$(awk "BEGIN {printf \"%.2f\", $RAMDISK_USED_KB / 1048576}") + SYMLINK=$(readlink "$TRANSCODE_LINK" 2>/dev/null || echo "unknown") + echo " $ICON_RAM Ramdisk usage: ${RAMDISK_USED_GB}GB" + echo " $ICON_LINK Symlink target: $SYMLINK" +else + echo " $ICON_RAM Ramdisk: not mounted" +fi +echo "" + +END=$(date +%s) + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_SUMMARY Summary ━━━ +# ----------------------------------------------------------------------------------------------- +echo "━━━━━ $ICON_SUMMARY EMBY REPORT SUMMARY ━━━━━" +echo "$ICON_EMBY Server: $SERVER_NAME (v$SERVER_VERSION)" +echo "$ICON_EMBY Active: $ACTIVE_COUNT streams ($DIRECT_COUNT direct / $TRANSCODE_COUNT transcode)" +echo "$ICON_EMBY Library: $MOVIE_COUNT movies $EPISODE_COUNT episodes $SONG_COUNT songs" +echo "$ICON_TIME Duration: $(format_duration $((END - START)))" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + +if [[ "$DRY_RUN" == false ]]; then + notify "Emby report on $(hostname) — $ACTIVE_COUNT active streams ($DIRECT_COUNT direct / $TRANSCODE_COUNT transcode) — Library: $MOVIE_COUNT movies $EPISODE_COUNT episodes" "Emby Report" "normal" +fi \ No newline at end of file diff --git a/Rsync/rsync.sh b/Rsync/rsync.sh index 46c06ce..8ca6588 100644 --- a/Rsync/rsync.sh +++ b/Rsync/rsync.sh @@ -147,7 +147,7 @@ DURATION=$((END - START)) # Reliable format: date|time|profile|duration|status # Does not parse rsync output — version-proof and always works # ----------------------------------------------------------------------------------------------- -BANDWIDTH_MONITOR="$SCRIPT_DIR/../Monitor/bandwidth_monitor.sh" +BANDWIDTH_MONITOR="$SCRIPT_DIR/../Monitors/bandwidth_monitor.sh" if [[ "$DRY_RUN" == false ]] && [[ -f "$BANDWIDTH_MONITOR" ]]; then STATUS="success" diff --git a/user_script_pluin.sh b/user_script_pluin.sh index 610d8ca..4c6f6b0 100644 --- a/user_script_pluin.sh +++ b/user_script_pluin.sh @@ -21,11 +21,11 @@ # v1.5 — media_management.sh orchestrator, corrected filenames, git repo section # v1.6 — lidarr_cleanup.sh, sonarr_cleanup.sh, radarr_cleanup.sh added # v1.7 — cert_monitor.sh added -# v1.8 — Monitor/ folder added with all monitoring scripts -# cert_monitor.sh moved from unRAID_Essentials to Monitor/ +# v1.8 — Monitors/ folder added with all monitoring scripts +# cert_monitor.sh moved from unRAID_Essentials to Monitors/ # backup_verify.sh, smart_health.sh, bandwidth_monitor.sh added # weekly_health_digest.sh, emby_session_report.sh added -# ZFS memory snapshot moved to Monitor/ — informational only +# ZFS memory snapshot moved to Monitors/ — informational only # Directory tree updated to reflect full ecosystem # ============================================================================================== @@ -41,9 +41,10 @@ # ├── git_pull_execute.sh # Pulls latest scripts from Gitea repo # │ # ├── Failover/ -# │ └── failover.sh # Mutual container failover — runs continuously +# │ ├── failover.sh # Mutual container failover — runs continuously +# │ └── failover_test.sh # Controlled failover simulation — run manually # │ -# ├── Monitor/ +# ├── Monitors/ # │ ├── backup_verify.sh # Random sample checksum verification vs remote # │ ├── cert_monitor.sh # SSL certificate expiry — direct openssl check # │ ├── emby_session_report.sh # Weekly Emby usage statistics via API @@ -102,7 +103,15 @@ # #/mnt/user/appdata/unraid_scripts/Failover/failover.sh # -# ━━━ Monitor ━━━ +# ━━━ Failover Test ━━━ +# Run manually during a maintenance window — starts and stops real containers. +# Always --dry-run first to walk through phases without making changes. +# Both servers must have failover.sh running before testing. +# +#/mnt/user/appdata/unraid_scripts/Failover/failover_test.sh --dry-run +#/mnt/user/appdata/unraid_scripts/Failover/failover_test.sh +# +# ━━━ Monitors ━━━ #/mnt/user/appdata/unraid_scripts/Monitors/backup_verify.sh #/mnt/user/appdata/unraid_scripts/Monitors/cert_monitor.sh #/mnt/user/appdata/unraid_scripts/Monitors/emby_session_report.sh