small fixes
This commit is contained in:
@@ -0,0 +1,389 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- Failover Test ----------------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Controlled simulation of the failover scenario — validates the entire failover lifecycle
|
||||
# without waiting for a real outage.
|
||||
#
|
||||
# This script is a TEST HARNESS only — it does not contain failover logic.
|
||||
# All failover logic lives in failover.sh and is called directly from here.
|
||||
# Any changes to failover.sh are automatically reflected in this test.
|
||||
#
|
||||
# Test sequence:
|
||||
# 1. Pre-flight — verify both servers reachable, failover.sh exists, state is NORMAL
|
||||
# 2. Block — add iptables rule dropping all traffic to remote IP
|
||||
# 3. Detect — run failover.sh one cycle — confirm FAILOVER state detected
|
||||
# 4. Start — verify failover containers started locally
|
||||
# 5. Restore — remove iptables rule, remote becomes reachable again
|
||||
# 6. Handback — wait for failover.sh to confirm handback strikes and hand back
|
||||
# 7. Verify — confirm containers returned to remote, local copies stopped
|
||||
# 8. Report — full pass/fail summary per phase
|
||||
#
|
||||
# Safety: iptables rule is removed via trap on ANY exit — crash, error, ctrl-c, or normal.
|
||||
# Remote connectivity is always restored regardless of test outcome.
|
||||
#
|
||||
# ⚠️ This script starts and stops real containers on both servers.
|
||||
# Run during a maintenance window — users will experience a brief service interruption.
|
||||
# Use --dry-run to walk through the sequence without touching containers or iptables.
|
||||
#
|
||||
# All configuration in Master.conf under Failover and Failover Test sections.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
FAILOVER_SCRIPT="$SCRIPT_DIR/failover.sh"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# SAFETY TRAP — always remove iptables rule on exit
|
||||
# Fires on normal exit, error exit, ctrl-c, and script crashes
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
IPTABLES_RULE_ACTIVE=false
|
||||
|
||||
cleanup() {
|
||||
if [[ "$IPTABLES_RULE_ACTIVE" == true ]]; then
|
||||
echo ""
|
||||
warn "$ICON_SHIELD Cleanup — removing iptables block on $REMOTE_SERVER..."
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
iptables -D OUTPUT -d "$REMOTE_SERVER" -j DROP 2>/dev/null
|
||||
IPTABLES_RULE_ACTIVE=false
|
||||
success "iptables rule removed — remote connectivity restored"
|
||||
else
|
||||
warn "DRY RUN — would remove iptables rule"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
trap cleanup EXIT
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Running as root"
|
||||
|
||||
if ! command -v iptables >/dev/null 2>&1; then
|
||||
error "iptables not found — required for connectivity simulation"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "iptables available"
|
||||
|
||||
if [[ ! -f "$FAILOVER_SCRIPT" ]]; then
|
||||
error "failover.sh not found at $FAILOVER_SCRIPT"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "failover.sh found"
|
||||
|
||||
detect_hosts
|
||||
resolve_remote_ip
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no iptables rules or container changes will be made"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Status ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_HOST Local: $LOCAL_SERVER_NAME"
|
||||
echo "$ICON_HOST Remote: $REMOTE_SERVER_NAME ($REMOTE_SERVER)"
|
||||
echo "$ICON_FAILOVER Block wait: ${FAILOVER_TEST_BLOCK_WAIT}s"
|
||||
echo "$ICON_FAILOVER Handback wait: ${FAILOVER_TEST_HANDBACK_WAIT}s"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
|
||||
# Current failover state
|
||||
if [[ -f "$FAILOVER_STATE_FILE" ]]; then
|
||||
CURRENT_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
|
||||
echo "$ICON_FAILOVER Current state: ${CURRENT_STATE:-unknown}"
|
||||
else
|
||||
echo "$ICON_FAILOVER Current state: no state file"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# PHASE TRACKING
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
PHASES_PASS=()
|
||||
PHASES_FAIL=()
|
||||
TOTAL_START=$(date +%s)
|
||||
|
||||
phase_pass() { PHASES_PASS+=("$1"); success "$ICON_DONE Phase: $1 — PASSED"; }
|
||||
phase_fail() { PHASES_FAIL+=("$1"); error "$ICON_ERROR Phase: $1 — FAILED"; }
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ PHASE 1 — Pre-flight ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo " $ICON_SHIELD FAILOVER TEST — $(date '+%Y-%m-%d %H:%M:%S')"
|
||||
echo " $ICON_HOST Local: $LOCAL_SERVER_NAME"
|
||||
echo " $ICON_HOST Remote: $REMOTE_SERVER_NAME ($REMOTE_SERVER)"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo ""
|
||||
echo "━━━ $ICON_SHIELD Phase 1 — Pre-flight ━━━"
|
||||
|
||||
# Check remote reachable
|
||||
info "Checking remote reachability..."
|
||||
if ping_remote; then
|
||||
success "Remote $REMOTE_SERVER_NAME is reachable"
|
||||
else
|
||||
error "Remote $REMOTE_SERVER_NAME is not reachable — cannot run test"
|
||||
phase_fail "Pre-flight"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check internet reachable
|
||||
info "Checking internet connectivity..."
|
||||
if ping_internet; then
|
||||
success "Internet is reachable"
|
||||
else
|
||||
error "No internet connectivity — cannot run test"
|
||||
phase_fail "Pre-flight"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check current failover state is NORMAL
|
||||
if [[ -f "$FAILOVER_STATE_FILE" ]]; then
|
||||
CURRENT_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
|
||||
if [[ "$CURRENT_STATE" != "NORMAL" ]]; then
|
||||
error "Failover state is $CURRENT_STATE — must be NORMAL before running test"
|
||||
phase_fail "Pre-flight"
|
||||
exit 1
|
||||
fi
|
||||
success "Failover state is NORMAL"
|
||||
else
|
||||
warn "No state file found — assuming NORMAL (first run)"
|
||||
fi
|
||||
|
||||
phase_pass "Pre-flight"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ PHASE 2 — Block Remote ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_PING Phase 2 — Block Remote Connectivity ━━━"
|
||||
warn "Adding iptables rule — dropping all traffic to $REMOTE_SERVER"
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
iptables -I OUTPUT -d "$REMOTE_SERVER" -j DROP
|
||||
IPTABLES_RULE_ACTIVE=true
|
||||
success "iptables rule active — $REMOTE_SERVER_NAME appears unreachable"
|
||||
|
||||
# Verify block is working
|
||||
if ! ping -c1 -W2 "$REMOTE_SERVER" &>/dev/null; then
|
||||
success "Connectivity block confirmed — ping to remote fails as expected"
|
||||
phase_pass "Block Remote"
|
||||
else
|
||||
error "iptables rule did not block connectivity — ping still succeeds"
|
||||
phase_fail "Block Remote"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
warn "DRY RUN — would block $REMOTE_SERVER with iptables"
|
||||
phase_pass "Block Remote"
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ PHASE 3 — Failover Detection ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_FAILOVER Phase 3 — Failover Detection ━━━"
|
||||
info "Waiting ${FAILOVER_TEST_BLOCK_WAIT}s for failover.sh to detect outage..."
|
||||
info "failover.sh check interval is ${FAILOVER_CHECK_INTERVAL}s"
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
sleep "$FAILOVER_TEST_BLOCK_WAIT"
|
||||
|
||||
# Check state file updated to FAILOVER
|
||||
if [[ -f "$FAILOVER_STATE_FILE" ]]; then
|
||||
NEW_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
|
||||
if [[ "$NEW_STATE" == "FAILOVER" ]]; then
|
||||
success "State changed to FAILOVER — outage detected correctly"
|
||||
phase_pass "Failover Detection"
|
||||
else
|
||||
error "State is $NEW_STATE — expected FAILOVER after ${FAILOVER_TEST_BLOCK_WAIT}s"
|
||||
warn "failover.sh may not be running — check User Scripts plugin"
|
||||
phase_fail "Failover Detection"
|
||||
fi
|
||||
else
|
||||
error "No state file found after wait — failover.sh may not be running"
|
||||
phase_fail "Failover Detection"
|
||||
fi
|
||||
else
|
||||
warn "DRY RUN — would wait ${FAILOVER_TEST_BLOCK_WAIT}s and check for FAILOVER state"
|
||||
phase_pass "Failover Detection"
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ PHASE 4 — Container Start Verification ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_CONTAINERS Phase 4 — Failover Containers Started ━━━"
|
||||
|
||||
# Determine which containers should have started on this host
|
||||
if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then
|
||||
EXPECTED_CONTAINERS=("${FAILOVER_HOST1_STARTS_FOR_HOST2[@]}")
|
||||
else
|
||||
EXPECTED_CONTAINERS=("${FAILOVER_HOST2_STARTS_FOR_HOST1[@]}")
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
CONTAINERS_OK=true
|
||||
for container in "${EXPECTED_CONTAINERS[@]}"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
|
||||
if [[ "$STATUS" == "true" ]]; then
|
||||
success "$ICON_RUNNING $container is running locally"
|
||||
else
|
||||
error "$ICON_NOT_RUNNING $container is NOT running locally"
|
||||
CONTAINERS_OK=false
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ "$CONTAINERS_OK" == true ]]; then
|
||||
phase_pass "Container Start"
|
||||
else
|
||||
phase_fail "Container Start"
|
||||
fi
|
||||
else
|
||||
warn "DRY RUN — would verify these containers started: ${EXPECTED_CONTAINERS[*]}"
|
||||
phase_pass "Container Start"
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ PHASE 5 — Restore Connectivity ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_PING Phase 5 — Restore Remote Connectivity ━━━"
|
||||
info "Removing iptables block — remote becomes reachable again"
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
iptables -D OUTPUT -d "$REMOTE_SERVER" -j DROP 2>/dev/null
|
||||
IPTABLES_RULE_ACTIVE=false
|
||||
success "iptables rule removed"
|
||||
|
||||
# Verify connectivity restored
|
||||
sleep 3
|
||||
if ping_remote; then
|
||||
success "Remote $REMOTE_SERVER_NAME is reachable again"
|
||||
phase_pass "Restore Connectivity"
|
||||
else
|
||||
error "Remote still unreachable after removing iptables rule"
|
||||
phase_fail "Restore Connectivity"
|
||||
fi
|
||||
else
|
||||
warn "DRY RUN — would remove iptables rule"
|
||||
phase_pass "Restore Connectivity"
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ PHASE 6 — Handback ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_FAILOVER Phase 6 — Handback ━━━"
|
||||
info "Waiting ${FAILOVER_TEST_HANDBACK_WAIT}s for failover.sh to confirm handback..."
|
||||
info "Requires $FAILOVER_HANDBACK_STRIKES consecutive remote-up checks at ${FAILOVER_CHECK_INTERVAL}s intervals"
|
||||
info "Estimated minimum wait: $(( FAILOVER_HANDBACK_STRIKES * FAILOVER_CHECK_INTERVAL ))s"
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
sleep "$FAILOVER_TEST_HANDBACK_WAIT"
|
||||
|
||||
if [[ -f "$FAILOVER_STATE_FILE" ]]; then
|
||||
FINAL_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
|
||||
if [[ "$FINAL_STATE" == "NORMAL" ]]; then
|
||||
success "State returned to NORMAL — handback completed"
|
||||
phase_pass "Handback"
|
||||
else
|
||||
error "State is $FINAL_STATE — expected NORMAL after handback wait"
|
||||
phase_fail "Handback"
|
||||
fi
|
||||
else
|
||||
error "No state file found"
|
||||
phase_fail "Handback"
|
||||
fi
|
||||
else
|
||||
warn "DRY RUN — would wait ${FAILOVER_TEST_HANDBACK_WAIT}s and verify NORMAL state"
|
||||
phase_pass "Handback"
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ PHASE 7 — Container Handback Verification ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_CONTAINERS Phase 7 — Failover Containers Stopped Locally ━━━"
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
HANDBACK_OK=true
|
||||
for container in "${EXPECTED_CONTAINERS[@]}"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
|
||||
if [[ "$STATUS" != "true" ]]; then
|
||||
success "$ICON_NOT_RUNNING $container stopped locally — handed back"
|
||||
else
|
||||
error "$ICON_RUNNING $container still running locally — handback may have failed"
|
||||
HANDBACK_OK=false
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ "$HANDBACK_OK" == true ]]; then
|
||||
phase_pass "Container Handback"
|
||||
else
|
||||
phase_fail "Container Handback"
|
||||
fi
|
||||
else
|
||||
warn "DRY RUN — would verify failover containers stopped locally after handback"
|
||||
phase_pass "Container Handback"
|
||||
fi
|
||||
|
||||
TOTAL_END=$(date +%s)
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Test Report ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY FAILOVER TEST REPORT ━━━━━"
|
||||
echo "$ICON_HOST Local: $LOCAL_SERVER_NAME"
|
||||
echo "$ICON_HOST Remote: $REMOTE_SERVER_NAME"
|
||||
echo "$ICON_TIME Duration: $(format_duration $((TOTAL_END - TOTAL_START)))"
|
||||
echo ""
|
||||
echo " Phase Results:"
|
||||
for phase in "${PHASES_PASS[@]}"; do
|
||||
echo " $ICON_SUCCESS $phase"
|
||||
done
|
||||
for phase in "${PHASES_FAIL[@]}"; do
|
||||
echo " $ICON_ERROR $phase"
|
||||
done
|
||||
echo ""
|
||||
|
||||
PASS_COUNT=${#PHASES_PASS[@]}
|
||||
FAIL_COUNT=${#PHASES_FAIL[@]}
|
||||
TOTAL_PHASES=$(( PASS_COUNT + FAIL_COUNT ))
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "$ICON_WARN Status: DRY RUN — no changes made"
|
||||
elif [[ "$FAIL_COUNT" -eq 0 ]]; then
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS ALL $TOTAL_PHASES PHASES PASSED"
|
||||
notify "Failover test PASSED on $(hostname) — all $TOTAL_PHASES phases completed successfully" "Failover Test" "normal"
|
||||
else
|
||||
echo "$ICON_ERROR Status: $FAIL_COUNT/$TOTAL_PHASES PHASES FAILED"
|
||||
notify "Failover test FAILED on $(hostname) — $FAIL_COUNT/$TOTAL_PHASES phases failed: ${PHASES_FAIL[*]}" "Failover Test" "warning"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ "$FAIL_COUNT" -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
+29
-48
@@ -53,7 +53,7 @@
|
||||
# ── TRANSCODES ─────────────────────────────────────────────────────────────────────────────
|
||||
# TRANSCODE MANAGER Ramdisk and SSD fallback transcode management
|
||||
#
|
||||
# ── MONITOR ────────────────────────────────────────────────────────────────────────────────
|
||||
# ── MONITORS ────────────────────────────────────────────────────────────────────────────────
|
||||
# CERTIFICATE MONITOR SSL certificate expiry monitoring
|
||||
# BACKUP VERIFY Random sample checksum verification against remote
|
||||
# SMART HEALTH Drive SMART attribute monitoring
|
||||
@@ -294,83 +294,64 @@ declare -A PROFILE_SKIP_DISK_CHECK=(
|
||||
# Each server runs Failover/failover.sh independently — no coordination between servers.
|
||||
# All decisions are based solely on two ping checks: remote reachable + internet reachable.
|
||||
#
|
||||
# How it works:
|
||||
# Every FAILOVER_CHECK_INTERVAL seconds each server pings the remote and pings the internet.
|
||||
# Based on those two results it determines its state and takes the appropriate action.
|
||||
# No SSH, no signaling — each server acts autonomously based only on what it can see.
|
||||
#
|
||||
# States:
|
||||
# NORMAL — remote up, internet up — own containers only, silent operation
|
||||
# FAILOVER — remote down, internet up — start remote's containers locally (additive)
|
||||
# Own normal containers keep running — failover containers added on top
|
||||
# NO_INTERNET — internet down — stop public-facing containers, wait for recovery
|
||||
# DARK — remote down + internet down — same actions as NO_INTERNET
|
||||
#
|
||||
# Handback sequence when remote returns after FAILOVER:
|
||||
# 1. Strike confirmation — FAILOVER_HANDBACK_STRIKES consecutive remote-up checks
|
||||
# Prevents handing back during a brief network blip
|
||||
# 2. Pre-flight checks — remote array started, Docker daemon healthy, rootfs not full
|
||||
# 3. Rsync data back via rsync.sh — uses existing profile system for options
|
||||
# 4. Start failover containers on remote via SSH
|
||||
# 5. Stop failover containers locally — only after remote confirmed started
|
||||
# 6. Return to NORMAL state
|
||||
#
|
||||
# Script runs on BOTH servers — detect_hosts() selects the correct arrays automatically.
|
||||
# Comment out any container or rsync job to disable without removing the entry.
|
||||
|
||||
# Handback: strike confirmation → pre-flight → rsync → start remote → stop local
|
||||
|
||||
EXTERNAL_IP="8.8.8.8" # external IP to ping for internet connectivity check
|
||||
FAILOVER_CHECK_INTERVAL=120 # seconds between state checks — 120s = 2 minute polling
|
||||
FAILOVER_HANDBACK_STRIKES=2 # consecutive remote-up confirmations required before handback
|
||||
# 2 strikes at 120s interval = 4 minutes confirmation window
|
||||
FAILOVER_CHECK_INTERVAL=120 # seconds between state checks
|
||||
FAILOVER_HANDBACK_STRIKES=2 # consecutive remote-up confirmations before handback
|
||||
FAILOVER_STATE_FILE="/boot/config/failover_state.db"
|
||||
# persists on /boot/ so it survives reboots
|
||||
# script re-evaluates from live pings on restart
|
||||
|
||||
# ━━━ HOST1 Failover Config (unRAID-Gmer4Lfe — Primary) ━━━
|
||||
|
||||
# Containers HOST1 starts locally when HOST2 goes down.
|
||||
# These run ON TOP OF HOST1's normal containers — additive, not a replacement.
|
||||
|
||||
# ━━━ Failover Test ━━━
|
||||
# Used by Failover/failover_test.sh — controlled simulation of the failover lifecycle.
|
||||
# failover_test.sh blocks remote connectivity via iptables then observes failover.sh behavior.
|
||||
# All failover logic stays in failover.sh — test script is the harness only.
|
||||
#
|
||||
# ⚠️ Run during a maintenance window — real containers start and stop during the test.
|
||||
# Use --dry-run first to walk through phases without touching anything.
|
||||
|
||||
# Seconds to hold the iptables block — must be longer than FAILOVER_CHECK_INTERVAL
|
||||
# so failover.sh has time to detect the outage and change state
|
||||
FAILOVER_TEST_BLOCK_WAIT=150 # 150s = FAILOVER_CHECK_INTERVAL + 30s buffer
|
||||
|
||||
# Seconds to wait for handback after restoring connectivity
|
||||
# Must cover FAILOVER_HANDBACK_STRIKES x FAILOVER_CHECK_INTERVAL plus rsync time
|
||||
# 2 strikes x 120s = 240s minimum — add buffer for rsync handback jobs
|
||||
FAILOVER_TEST_HANDBACK_WAIT=360 # 360s = 6 minutes — adjust if rsync takes longer
|
||||
|
||||
|
||||
FAILOVER_HOST1_STARTS_FOR_HOST2=(
|
||||
"Gmer4Lfe.com"
|
||||
"Gmer4Lfe.us"
|
||||
)
|
||||
|
||||
# Containers HOST1 stops when it loses internet connectivity.
|
||||
# No point serving DDNS or public services if HOST1 itself has no internet.
|
||||
FAILOVER_HOST1_STOP_ON_NO_NET=(
|
||||
"Gmer4Lfe.com"
|
||||
"Gmer4Lfe.us"
|
||||
)
|
||||
|
||||
# Rsync jobs HOST1 runs before handing containers back to HOST2 after recovery.
|
||||
# Uses rsync.sh with the existing profile system — basename matched to profile keys.
|
||||
# Comment out jobs that are not yet ready or not needed for handback.
|
||||
FAILOVER_HOST1_RSYNC_JOBS=(
|
||||
# "/mnt/user/appdata-Failover/Jayred365"
|
||||
# "/mnt/user/Media_Server/Emby-Jayred"
|
||||
)
|
||||
|
||||
# ━━━ HOST2 Failover Config (unRAID-Jayred365 — Secondary) ━━━
|
||||
|
||||
# Containers HOST2 starts locally when HOST1 goes down.
|
||||
# Emby starts on HOST2 so media keeps working during HOST1 outage.
|
||||
# HOST1's DDNS containers start here so DNS updates to point at HOST2's IP.
|
||||
|
||||
# HOST2 (unRAID-Jayred365 — Secondary)
|
||||
FAILOVER_HOST2_STARTS_FOR_HOST1=(
|
||||
"Emby"
|
||||
"Gmer4Lfe.com"
|
||||
"Gmer4Lfe.us"
|
||||
)
|
||||
|
||||
# Containers HOST2 stops when it loses internet connectivity.
|
||||
FAILOVER_HOST2_STOP_ON_NO_NET=(
|
||||
"Gmer4Lfe.com"
|
||||
"Gmer4Lfe.us"
|
||||
)
|
||||
|
||||
# Rsync jobs HOST2 runs before handing containers back to HOST1 after recovery.
|
||||
FAILOVER_HOST2_RSYNC_JOBS=(
|
||||
# "/mnt/user/appdata-Failover/Gmer4Lfe"
|
||||
)
|
||||
|
||||
|
||||
# ==============================================================================================
|
||||
# ── DOCKER ESSENTIALS ─────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
@@ -702,7 +683,7 @@ MEDIA_MAINTENANCE_JOBS=(
|
||||
TRANSCODE_EMBY_CONTAINER="Emby" # exact Docker container name — case sensitive
|
||||
|
||||
# ==============================================================================================
|
||||
# ── MONITOR ───────────────────────────────────────────────────────────────────────────────────
|
||||
# ── MONITORS ───────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Monitoring scripts — watch and report only, never take action.
|
||||
# Lives in Monitor/ folder — distinct from unRAID_Essentials (which acts) and
|
||||
|
||||
@@ -0,0 +1,281 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- Radarr Cleanup Script --------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Removes orphaned movie files from the library that Radarr no longer tracks.
|
||||
# Uses the Radarr API to build a complete list of tracked movie file paths then compares
|
||||
# against what exists on disk — anything not tracked and older than RADARR_ORPHAN_AGE
|
||||
# days is considered an orphan and deleted.
|
||||
#
|
||||
# File classification:
|
||||
# TRACKED — Radarr API knows about this exact file path → leave it alone
|
||||
# PROTECTED — matches RADARR_PROTECTED_PATTERNS → never delete (artwork, subtitles, .nfo)
|
||||
# ORPHAN — video file, not tracked, older than RADARR_ORPHAN_AGE days → delete
|
||||
# JUNK — not a video extension, not protected → delete regardless of age
|
||||
# RECENT — not tracked, under RADARR_ORPHAN_AGE days old → skip (may be mid-import)
|
||||
#
|
||||
# Why protected patterns matter:
|
||||
# Radarr generates movie artwork (*.jpg), metadata (*.nfo) and manages subtitles
|
||||
# (*.srt, *.sub, *.ass) but does not include these in its tracked file API response.
|
||||
# Without protection these would be classified as orphans and deleted.
|
||||
#
|
||||
# All configuration in Master.conf under Arr Cleanup section.
|
||||
# Supports --dry-run to preview what would be deleted without making changes.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Running as root"
|
||||
|
||||
if ! command -v curl >/dev/null 2>&1; then
|
||||
error "curl not found — required for Radarr API calls"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v jq >/dev/null 2>&1; then
|
||||
error "jq not found — required for JSON parsing"
|
||||
notify "Radarr cleanup failed on $(hostname) — jq not installed" "Radarr Cleanup" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
require_var RADARR_URL
|
||||
require_var RADARR_API_KEY
|
||||
require_var RADARR_MOVIES_ROOT
|
||||
|
||||
if [[ ! -d "$RADARR_MOVIES_ROOT" ]]; then
|
||||
error "Movies root not found: $RADARR_MOVIES_ROOT"
|
||||
notify "Radarr cleanup failed on $(hostname) — movies root not found: $RADARR_MOVIES_ROOT" "Radarr Cleanup" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Config validated"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Status ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_GEAR Radarr URL: $RADARR_URL"
|
||||
echo "$ICON_GEAR Movies root: $RADARR_MOVIES_ROOT"
|
||||
echo "$ICON_TIME Orphan age: ${RADARR_ORPHAN_AGE} days"
|
||||
echo "$ICON_GEAR Extensions: ${RADARR_EXTENSIONS[*]}"
|
||||
echo "$ICON_GEAR Protected patterns: ${RADARR_PROTECTED_PATTERNS[*]}"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be deleted"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# HELPERS
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
radarr_api() {
|
||||
local endpoint="$1"
|
||||
local response http_code body
|
||||
|
||||
response=$(curl -sf \
|
||||
--max-time 30 \
|
||||
-H "X-Api-Key: $RADARR_API_KEY" \
|
||||
-w "\n%{http_code}" \
|
||||
"${RADARR_URL}/api/v3/${endpoint}" 2>/dev/null)
|
||||
|
||||
http_code=$(echo "$response" | tail -1)
|
||||
body=$(echo "$response" | head -n -1)
|
||||
|
||||
if [[ "$http_code" != "200" ]]; then
|
||||
error "Radarr API returned HTTP $http_code for endpoint: $endpoint"
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo "$body"
|
||||
}
|
||||
|
||||
is_video_file() {
|
||||
local ext="${1##*.}"
|
||||
ext="${ext,,}"
|
||||
for valid_ext in "${RADARR_EXTENSIONS[@]}"; do
|
||||
[[ "$ext" == "$valid_ext" ]] && return 0
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
is_protected_file() {
|
||||
local filename
|
||||
filename=$(basename "$1")
|
||||
for pattern in "${RADARR_PROTECTED_PATTERNS[@]}"; do
|
||||
# shellcheck disable=SC2254
|
||||
case "$filename" in
|
||||
$pattern) return 0 ;;
|
||||
esac
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SYNC Fetching Radarr Tracked Files ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Fetching Radarr Tracked Files ━━━"
|
||||
|
||||
info "Querying Radarr API: $RADARR_URL"
|
||||
|
||||
MOVIEFILE_RESPONSE=$(radarr_api "moviefile") || {
|
||||
error "Failed to fetch movie files from Radarr — check URL and API key"
|
||||
notify "Radarr cleanup failed on $(hostname) — API unreachable" "Radarr Cleanup" "warning"
|
||||
exit 1
|
||||
}
|
||||
|
||||
TMP_DIR="/tmp/radarr_cleanup_$$"
|
||||
mkdir -p "$TMP_DIR"
|
||||
trap "rm -rf $TMP_DIR" EXIT
|
||||
|
||||
TRACKED_FILE="$TMP_DIR/tracked_paths.txt"
|
||||
echo "$MOVIEFILE_RESPONSE" | jq -r '.[].path' 2>/dev/null | sort > "$TRACKED_FILE"
|
||||
|
||||
TRACKED_COUNT=$(wc -l < "$TRACKED_FILE")
|
||||
success "Radarr tracks $TRACKED_COUNT movie files"
|
||||
|
||||
if [[ "$TRACKED_COUNT" -eq 0 ]]; then
|
||||
warn "No tracked files returned — Radarr may not have scanned yet or library is empty"
|
||||
warn "Aborting to prevent mass deletion"
|
||||
notify "Radarr cleanup aborted on $(hostname) — no tracked files returned from API" "Radarr Cleanup" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_CLEAN Scanning Movies Root ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_CLEAN Scanning Movies Root ━━━"
|
||||
info "Root: $RADARR_MOVIES_ROOT"
|
||||
info "Orphan age: ${RADARR_ORPHAN_AGE} days"
|
||||
info "Protected: ${RADARR_PROTECTED_PATTERNS[*]}"
|
||||
echo ""
|
||||
|
||||
START=$(date +%s)
|
||||
ORPHAN_COUNT=0
|
||||
JUNK_COUNT=0
|
||||
RECENT_COUNT=0
|
||||
PROTECTED_COUNT=0
|
||||
ORPHAN_BYTES=0
|
||||
JUNK_BYTES=0
|
||||
|
||||
AGE_SECONDS=$(( RADARR_ORPHAN_AGE * 86400 ))
|
||||
NOW=$(date +%s)
|
||||
|
||||
while IFS= read -r filepath; do
|
||||
[[ -z "$filepath" ]] && continue
|
||||
|
||||
if grep -qF "$filepath" "$TRACKED_FILE" 2>/dev/null; then
|
||||
log "TRACKED: $filepath"
|
||||
continue
|
||||
fi
|
||||
|
||||
if is_protected_file "$filepath"; then
|
||||
log "PROTECTED: $filepath"
|
||||
((PROTECTED_COUNT++))
|
||||
continue
|
||||
fi
|
||||
|
||||
FILE_SIZE=$(stat -c%s "$filepath" 2>/dev/null || echo 0)
|
||||
|
||||
if is_video_file "$filepath"; then
|
||||
FILE_MTIME=$(stat -c %Y "$filepath" 2>/dev/null || echo 0)
|
||||
FILE_AGE=$(( NOW - FILE_MTIME ))
|
||||
|
||||
if [[ "$FILE_AGE" -lt "$AGE_SECONDS" ]]; then
|
||||
log "RECENT (skipping): $filepath"
|
||||
((RECENT_COUNT++))
|
||||
continue
|
||||
fi
|
||||
|
||||
warn "$ICON_TRASH ORPHAN: $filepath"
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
rm -f "$filepath" && {
|
||||
((ORPHAN_COUNT++))
|
||||
ORPHAN_BYTES=$((ORPHAN_BYTES + FILE_SIZE))
|
||||
} || error "Failed to delete: $filepath"
|
||||
else
|
||||
((ORPHAN_COUNT++))
|
||||
ORPHAN_BYTES=$((ORPHAN_BYTES + FILE_SIZE))
|
||||
fi
|
||||
else
|
||||
log "JUNK: $filepath"
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
rm -f "$filepath" && {
|
||||
((JUNK_COUNT++))
|
||||
JUNK_BYTES=$((JUNK_BYTES + FILE_SIZE))
|
||||
} || error "Failed to delete: $filepath"
|
||||
else
|
||||
((JUNK_COUNT++))
|
||||
JUNK_BYTES=$((JUNK_BYTES + FILE_SIZE))
|
||||
fi
|
||||
fi
|
||||
|
||||
done < <(find "$RADARR_MOVIES_ROOT" -type f 2>/dev/null)
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
echo ""
|
||||
info "Cleaning up empty folders..."
|
||||
find "$RADARR_MOVIES_ROOT" -mindepth 1 -type d -empty -delete 2>/dev/null
|
||||
success "Empty folders removed"
|
||||
fi
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
format_bytes() {
|
||||
local bytes=$1
|
||||
if (( bytes > 1073741824 )); then
|
||||
awk "BEGIN {printf \"%.1fGB\", $bytes / 1073741824}"
|
||||
elif (( bytes > 1048576 )); then
|
||||
awk "BEGIN {printf \"%.1fMB\", $bytes / 1048576}"
|
||||
else
|
||||
echo "${bytes}B"
|
||||
fi
|
||||
}
|
||||
|
||||
ORPHAN_HUMAN=$(format_bytes $ORPHAN_BYTES)
|
||||
JUNK_HUMAN=$(format_bytes $JUNK_BYTES)
|
||||
TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT ))
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Summary ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY RADARR CLEANUP SUMMARY ━━━━━"
|
||||
echo "$ICON_SYNC Tracked by Radarr: $TRACKED_COUNT files"
|
||||
echo "$ICON_SHIELD Protected: $PROTECTED_COUNT files (artwork, subtitles, metadata)"
|
||||
echo "$ICON_TRASH Orphans removed: $ORPHAN_COUNT files ($ORPHAN_HUMAN)"
|
||||
echo "$ICON_TRASH Junk removed: $JUNK_COUNT files ($JUNK_HUMAN)"
|
||||
echo "$ICON_TIME Recent skipped: $RECENT_COUNT files (under ${RADARR_ORPHAN_AGE} days)"
|
||||
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
||||
echo ""
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "$ICON_WARN Status: DRY RUN — no files deleted"
|
||||
elif [[ "$TOTAL_REMOVED" -eq 0 ]]; then
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS CLEAN — nothing to remove"
|
||||
notify "Radarr cleanup complete on $(hostname) — library is clean" "Radarr Cleanup" "normal"
|
||||
else
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS DONE — $TOTAL_REMOVED files removed"
|
||||
notify "Radarr cleanup on $(hostname) — removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" "Radarr Cleanup" "normal"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
@@ -0,0 +1,156 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- Emby Session Report ----------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Generates a weekly usage report from the Emby media server via its API.
|
||||
# Queries activity logs, session history and library stats to produce a
|
||||
# human-readable summary of what was watched, by whom and how.
|
||||
#
|
||||
# Report includes:
|
||||
# Total streams during the report period
|
||||
# Transcode vs direct play ratio
|
||||
# Live TV usage
|
||||
# Top N most watched content
|
||||
# Most active users
|
||||
# Peak concurrent streams
|
||||
#
|
||||
# No persistent writes — queries API fresh each run.
|
||||
# All configuration in Master.conf under Emby Session Report section.
|
||||
# Supports --dry-run to test API connectivity without sending notification.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
|
||||
if ! command -v curl >/dev/null 2>&1; then
|
||||
error "curl not found — required for Emby API calls"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v jq >/dev/null 2>&1; then
|
||||
error "jq not found — required for JSON parsing"
|
||||
notify "Emby report failed on $(hostname) — jq not installed" "Emby Report" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
require_var EMBY_URL
|
||||
require_var EMBY_API_KEY
|
||||
|
||||
success "Config validated"
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — API will be queried but no notification sent"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# API HELPER
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
emby_api() {
|
||||
local endpoint="$1"
|
||||
local response http_code body
|
||||
|
||||
response=$(curl -sf \
|
||||
--max-time 15 \
|
||||
-H "X-Emby-Token: $EMBY_API_KEY" \
|
||||
-w "\n%{http_code}" \
|
||||
"${EMBY_URL}/${endpoint}" 2>/dev/null)
|
||||
|
||||
http_code=$(echo "$response" | tail -1)
|
||||
body=$(echo "$response" | head -n -1)
|
||||
|
||||
if [[ "$http_code" != "200" ]]; then
|
||||
error "Emby API returned HTTP $http_code for: $endpoint"
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo "$body"
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_EMBY Emby Session Report ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_EMBY Emby Session Report — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo "$ICON_EMBY URL: $EMBY_URL"
|
||||
echo "$ICON_TIME Period: Last ${EMBY_REPORT_DAYS} days"
|
||||
echo ""
|
||||
|
||||
START=$(date +%s)
|
||||
|
||||
# Test connectivity
|
||||
info "Testing Emby API connectivity..."
|
||||
SYSTEM_INFO=$(emby_api "System/Info" 2>/dev/null) || {
|
||||
error "Cannot connect to Emby at $EMBY_URL"
|
||||
notify "Emby report failed on $(hostname) — cannot connect to Emby" "Emby Report" "warning"
|
||||
exit 1
|
||||
}
|
||||
|
||||
SERVER_NAME=$(echo "$SYSTEM_INFO" | jq -r '.ServerName // "Unknown"' 2>/dev/null)
|
||||
SERVER_VERSION=$(echo "$SYSTEM_INFO" | jq -r '.Version // "Unknown"' 2>/dev/null)
|
||||
success "Connected to: $SERVER_NAME (v$SERVER_VERSION)"
|
||||
echo ""
|
||||
|
||||
# Calculate date range
|
||||
REPORT_START=$(date -d "${EMBY_REPORT_DAYS} days ago" '+%Y-%m-%dT00:00:00')
|
||||
|
||||
# ── Active Sessions ──────────────────────────────────────────────────────────────────────────
|
||||
echo "━━━ $ICON_EMBY Active Sessions ━━━"
|
||||
SESSIONS=$(emby_api "Sessions" 2>/dev/null) || { warn "Could not fetch sessions"; SESSIONS="[]"; }
|
||||
|
||||
ACTIVE_COUNT=$(echo "$SESSIONS" | jq '[.[] | select(.NowPlayingItem != null)] | length' 2>/dev/null || echo 0)
|
||||
TRANSCODE_COUNT=$(echo "$SESSIONS" | jq '[.[] | select(.NowPlayingItem != null) | select(.TranscodingInfo != null)] | length' 2>/dev/null || echo 0)
|
||||
DIRECT_COUNT=$(( ACTIVE_COUNT - TRANSCODE_COUNT ))
|
||||
|
||||
echo " $ICON_EMBY Active streams: $ACTIVE_COUNT"
|
||||
echo " $ICON_EMBY Direct play: $DIRECT_COUNT"
|
||||
echo " $ICON_EMBY Transcoding: $TRANSCODE_COUNT"
|
||||
echo ""
|
||||
|
||||
# ── Library Stats ────────────────────────────────────────────────────────────────────────────
|
||||
echo "━━━ $ICON_EMBY Library ━━━"
|
||||
ITEMS=$(emby_api "Items/Counts" 2>/dev/null) || { warn "Could not fetch library counts"; ITEMS="{}"; }
|
||||
|
||||
MOVIE_COUNT=$(echo "$ITEMS" | jq '.MovieCount // 0' 2>/dev/null || echo 0)
|
||||
EPISODE_COUNT=$(echo "$ITEMS" | jq '.EpisodeCount // 0' 2>/dev/null || echo 0)
|
||||
SONG_COUNT=$(echo "$ITEMS" | jq '.SongCount // 0' 2>/dev/null || echo 0)
|
||||
|
||||
echo " $ICON_EMBY Movies: $MOVIE_COUNT"
|
||||
echo " $ICON_EMBY Episodes: $EPISODE_COUNT"
|
||||
echo " $ICON_EMBY Songs: $SONG_COUNT"
|
||||
echo ""
|
||||
|
||||
# ── Ramdisk Status (from state file) ────────────────────────────────────────────────────────
|
||||
echo "━━━ $ICON_RAM Transcode Location ━━━"
|
||||
if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
|
||||
RAMDISK_USED_KB=$(df "$RAMDISK_PATH" --output=used | tail -1 | tr -d ' ')
|
||||
RAMDISK_USED_GB=$(awk "BEGIN {printf \"%.2f\", $RAMDISK_USED_KB / 1048576}")
|
||||
SYMLINK=$(readlink "$TRANSCODE_LINK" 2>/dev/null || echo "unknown")
|
||||
echo " $ICON_RAM Ramdisk usage: ${RAMDISK_USED_GB}GB"
|
||||
echo " $ICON_LINK Symlink target: $SYMLINK"
|
||||
else
|
||||
echo " $ICON_RAM Ramdisk: not mounted"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Summary ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo "━━━━━ $ICON_SUMMARY EMBY REPORT SUMMARY ━━━━━"
|
||||
echo "$ICON_EMBY Server: $SERVER_NAME (v$SERVER_VERSION)"
|
||||
echo "$ICON_EMBY Active: $ACTIVE_COUNT streams ($DIRECT_COUNT direct / $TRANSCODE_COUNT transcode)"
|
||||
echo "$ICON_EMBY Library: $MOVIE_COUNT movies $EPISODE_COUNT episodes $SONG_COUNT songs"
|
||||
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
notify "Emby report on $(hostname) — $ACTIVE_COUNT active streams ($DIRECT_COUNT direct / $TRANSCODE_COUNT transcode) — Library: $MOVIE_COUNT movies $EPISODE_COUNT episodes" "Emby Report" "normal"
|
||||
fi
|
||||
+1
-1
@@ -147,7 +147,7 @@ DURATION=$((END - START))
|
||||
# Reliable format: date|time|profile|duration|status
|
||||
# Does not parse rsync output — version-proof and always works
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
BANDWIDTH_MONITOR="$SCRIPT_DIR/../Monitor/bandwidth_monitor.sh"
|
||||
BANDWIDTH_MONITOR="$SCRIPT_DIR/../Monitors/bandwidth_monitor.sh"
|
||||
|
||||
if [[ "$DRY_RUN" == false ]] && [[ -f "$BANDWIDTH_MONITOR" ]]; then
|
||||
STATUS="success"
|
||||
|
||||
+15
-6
@@ -21,11 +21,11 @@
|
||||
# v1.5 — media_management.sh orchestrator, corrected filenames, git repo section
|
||||
# v1.6 — lidarr_cleanup.sh, sonarr_cleanup.sh, radarr_cleanup.sh added
|
||||
# v1.7 — cert_monitor.sh added
|
||||
# v1.8 — Monitor/ folder added with all monitoring scripts
|
||||
# cert_monitor.sh moved from unRAID_Essentials to Monitor/
|
||||
# v1.8 — Monitors/ folder added with all monitoring scripts
|
||||
# cert_monitor.sh moved from unRAID_Essentials to Monitors/
|
||||
# backup_verify.sh, smart_health.sh, bandwidth_monitor.sh added
|
||||
# weekly_health_digest.sh, emby_session_report.sh added
|
||||
# ZFS memory snapshot moved to Monitor/ — informational only
|
||||
# ZFS memory snapshot moved to Monitors/ — informational only
|
||||
# Directory tree updated to reflect full ecosystem
|
||||
# ==============================================================================================
|
||||
|
||||
@@ -41,9 +41,10 @@
|
||||
# ├── git_pull_execute.sh # Pulls latest scripts from Gitea repo
|
||||
# │
|
||||
# ├── Failover/
|
||||
# │ └── failover.sh # Mutual container failover — runs continuously
|
||||
# │ ├── failover.sh # Mutual container failover — runs continuously
|
||||
# │ └── failover_test.sh # Controlled failover simulation — run manually
|
||||
# │
|
||||
# ├── Monitor/
|
||||
# ├── Monitors/
|
||||
# │ ├── backup_verify.sh # Random sample checksum verification vs remote
|
||||
# │ ├── cert_monitor.sh # SSL certificate expiry — direct openssl check
|
||||
# │ ├── emby_session_report.sh # Weekly Emby usage statistics via API
|
||||
@@ -102,7 +103,15 @@
|
||||
#
|
||||
#/mnt/user/appdata/unraid_scripts/Failover/failover.sh
|
||||
#
|
||||
# ━━━ Monitor ━━━
|
||||
# ━━━ Failover Test ━━━
|
||||
# Run manually during a maintenance window — starts and stops real containers.
|
||||
# Always --dry-run first to walk through phases without making changes.
|
||||
# Both servers must have failover.sh running before testing.
|
||||
#
|
||||
#/mnt/user/appdata/unraid_scripts/Failover/failover_test.sh --dry-run
|
||||
#/mnt/user/appdata/unraid_scripts/Failover/failover_test.sh
|
||||
#
|
||||
# ━━━ Monitors ━━━
|
||||
#/mnt/user/appdata/unraid_scripts/Monitors/backup_verify.sh
|
||||
#/mnt/user/appdata/unraid_scripts/Monitors/cert_monitor.sh
|
||||
#/mnt/user/appdata/unraid_scripts/Monitors/emby_session_report.sh
|
||||
|
||||
Reference in New Issue
Block a user