From 11872b802b8685b2d2c8c54cf90dc587ef510cdb Mon Sep 17 00:00:00 2001 From: FailedProxy Date: Tue, 28 Apr 2026 17:04:09 -0400 Subject: [PATCH] big logic update to rsync and rsync called scripts --- Master.conf | 20 +- Orchestrators/daily_sync_maintenance.sh | 35 +- Rsync/rsync.sh | 24 +- common.sh | 280 ++++- safe_master.conf | 1407 +++++++++++++++++++++++ user_script_plug-in.sh | 2 - 6 files changed, 1718 insertions(+), 50 deletions(-) create mode 100644 safe_master.conf diff --git a/Master.conf b/Master.conf index a9c0f58..56e00b0 100644 --- a/Master.conf +++ b/Master.conf @@ -167,7 +167,7 @@ ARRAY_START_SCRIPTS=( "Docker_Essentials/docker_network_connect.sh" # ensure networks exist + connect containers "unRAID_Essentials/system_watchdog.sh" # system health monitor — continuous loop "Docker_Essentials/docker_watchdog.sh" # container health monitor — continuous loop -# "Failover/failover.sh" # mutual failover — continuous loop + "Failover/failover.sh" # mutual failover — continuous loop ) # ━━━ Daily Sync Maintenance ━━━ @@ -255,9 +255,9 @@ MEDIA_MANAGEMENT_JOBS=( "Media/media_shares_permissions.sh" # apply permissions — runs first "Media/media_cleaner.sh anime" # remove junk from anime shares "Media/media_cleaner.sh media" # remove junk from media shares -# "Media/lidarr_cleanup.sh" # remove orphaned music files -# "Media/sonarr_cleanup.sh" # remove orphaned TV files -# "Media/radarr_cleanup.sh" # remove orphaned movie files + "Media/lidarr_cleanup.sh" # remove orphaned music files + "Media/sonarr_cleanup.sh" # remove orphaned TV files + "Media/radarr_cleanup.sh" # remove orphaned movie files "Docker_Essentials/downloaders_reset.sh" # clear stuck states + purge old history ) @@ -403,16 +403,8 @@ declare -A PROFILE_EXCLUDE_DIRS=( [emby-failover]="logs transcodes cache metadata *.db-wal *.db-shm crash* plugins root" ) -# Skip per-disk space check for these profiles — appdata syncs go to cache/appdata -# not to array disks, so disk space check is irrelevant and just slows things down -declare -A PROFILE_SKIP_DISK_CHECK=( - [arrs_stack]=true - [critical-data]=true - [gmer4lfe]=true - [important-data]=true - [emby]=true - [emby-failover]=true -) +# Skip per-disk space check is no longer needed — check_remote_disks() auto-detects +# XFS and ZFS filesystem types from disks.ini, no manual configuration required # ============================================================================================== # ── FAILOVER ────────────────────────────────────────────────────────────────────────────────── diff --git a/Orchestrators/daily_sync_maintenance.sh b/Orchestrators/daily_sync_maintenance.sh index a593291..864f070 100644 --- a/Orchestrators/daily_sync_maintenance.sh +++ b/Orchestrators/daily_sync_maintenance.sh @@ -17,6 +17,10 @@ # 9. radarr_cleanup.sh — remove orphaned movie files # 10. docker_daily_restart.sh — restart containers that need daily restart # +# What triggers weekly_health_digest.sh: +# NOT this script — weekly_health_digest.sh runs on its own schedule (Saturday) +# This script writes no stats — it just syncs and maintains +# # Configuration in Master.conf: # DAILY_MAINTENANCE_SCRIPTS — pre/post-sync scripts (git pull, docker restart) # MEDIA_MANAGEMENT_JOBS — media maintenance jobs run after sync @@ -144,6 +148,8 @@ echo "" SHARE_INDEX=0 +ABORT_ALL_SYNCS=false + for SHARE in "${ALL_SHARES[@]}"; do SHARE_INDEX=$((SHARE_INDEX + 1)) SHARE_NAME=$(basename "$SHARE") @@ -151,14 +157,33 @@ for SHARE in "${ALL_SHARES[@]}"; do echo "━━━ $ICON_SYNC Share $SHARE_INDEX of $SHARE_COUNT: $SHARE_NAME ━━━" - if bash "$RSYNC_SCRIPT" "$SHARE"; then - SHARE_END=$(date +%s) - SHARE_TIMES+=("$SHARE_NAME:$((SHARE_END - SHARE_START))") + if [[ "$ABORT_ALL_SYNCS" == true ]]; then + warn "$SHARE_NAME — skipped (drive temps CRITICAL earlier in window)" + FAIL+=("$SHARE_NAME:temp-critical") + echo "" + continue + fi + + bash "$RSYNC_SCRIPT" "$SHARE" + RSYNC_EXIT=$? + + SHARE_END=$(date +%s) + SHARE_TIMES+=("$SHARE_NAME:$((SHARE_END - SHARE_START))") + + if [[ "$RSYNC_EXIT" -eq 0 ]]; then PASS+=("$SHARE_NAME") echo "$ICON_DONE $SHARE_NAME complete" + elif [[ "$RSYNC_EXIT" -eq 1 ]]; then + # Temp warning — skip this profile, continue to next + FAIL+=("$SHARE_NAME:temp-warn") + warn "$SHARE_NAME skipped — drive temps too high" + elif [[ "$RSYNC_EXIT" -eq 2 ]]; then + # Temp critical — abort all remaining syncs + FAIL+=("$SHARE_NAME:temp-critical") + ABORT_ALL_SYNCS=true + error "$SHARE_NAME aborted — drive temps CRITICAL, stopping all remaining syncs" + notify "Daily sync aborted on $(hostname) — drive temps CRITICAL during $SHARE_NAME sync" "Daily Sync" "warning" else - SHARE_END=$(date +%s) - SHARE_TIMES+=("$SHARE_NAME:$((SHARE_END - SHARE_START))") FAIL+=("$SHARE_NAME") error "$SHARE_NAME failed — continuing to next share" fi diff --git a/Rsync/rsync.sh b/Rsync/rsync.sh index 0661a17..b27ad07 100644 --- a/Rsync/rsync.sh +++ b/Rsync/rsync.sh @@ -80,9 +80,6 @@ read -r -a EXCLUDE_DIRS <<< "${PROFILE_EXCLUDE_DIRS[$PROFILE_N # Local and remote use the same container list — same naming scheme on both servers LOCAL_CRITICAL_CONTAINER_NAMES=("${CRITICAL_CONTAINER_NAMES[@]}") -# Disk check toggle -SKIP_DISK_CHECK=${PROFILE_SKIP_DISK_CHECK[$PROFILE_NAME]:-false} - [[ "$SHOW_STATUS" == true ]] && show_status && exit 0 # ----------------------------------------------------------------------------------------------- @@ -91,15 +88,24 @@ SKIP_DISK_CHECK=${PROFILE_SKIP_DISK_CHECK[$PROFILE_NAME]:-false} echo "" echo "━━━ $ICON_SHIELD Pre-flight Checks ━━━" +# Disk temp check — before touching remote or moving any data +# Returns: 0=OK 1=warn(skip this profile) 2=crit(abort all remaining) +check_local_disk_temps +TEMP_RESULT=$? +if [[ "$TEMP_RESULT" -eq 2 ]]; then + error "Drive temps CRITICAL — aborting sync for all remaining profiles" + exit 2 # caller (daily_sync_maintenance.sh) sees exit 2 → stops all syncs +elif [[ "$TEMP_RESULT" -eq 1 ]]; then + warn "Drive temps too high — skipping profile [$PROFILE_NAME]" + exit 1 # caller sees exit 1 → skips this profile, continues to next +else + success "Drive temps OK — $TEMP_CHECK_RESULT" +fi + check_connectivity check_remote_rootfs check_remote_share "$DIRECTORY" - -if [[ "$SKIP_DISK_CHECK" == "true" ]]; then - info "$ICON_DISK Disk check skipped for profile [$PROFILE_NAME] — ZFS pool on remote" -else - check_remote_disks "$DIRECTORY" -fi +check_remote_disks "$DIRECTORY" # ----------------------------------------------------------------------------------------------- # ━━━ $ICON_STOP $ICON_CONTAINERS Containers ━━━ diff --git a/common.sh b/common.sh index 9ee3907..976adc0 100644 --- a/common.sh +++ b/common.sh @@ -469,9 +469,122 @@ check_remote_share() { } # ----------------------------------------------------------------------------------------------- -# REMOTE DISK CHECK — fatal -# Verifies all physical disks backing a share are online on the remote server. -# Skipped when PROFILE_SKIP_DISK_CHECK is true (ZFS pools have no /mnt/disk* structure). +# get_unraid_temp_thresholds — read disk temp thresholds from unRAID's dynamix.cfg +# Sets globals: UNRAID_DISK_HOT UNRAID_DISK_MAX UNRAID_SSD_HOT UNRAID_SSD_MAX +# Falls back to safe defaults if file not found +# ----------------------------------------------------------------------------------------------- +get_unraid_temp_thresholds() { + local cfg="/boot/config/plugins/dynamix/dynamix.cfg" + if [[ -f "$cfg" ]]; then + UNRAID_DISK_HOT=$(grep '^hot=' "$cfg" 2>/dev/null | cut -d= -f2 | tr -d '"') + UNRAID_DISK_MAX=$(grep '^max=' "$cfg" 2>/dev/null | cut -d= -f2 | tr -d '"') + UNRAID_SSD_HOT=$(grep '^hotssd=' "$cfg" 2>/dev/null | cut -d= -f2 | tr -d '"') + UNRAID_SSD_MAX=$(grep '^maxssd=' "$cfg" 2>/dev/null | cut -d= -f2 | tr -d '"') + fi + # Safe defaults if not found + UNRAID_DISK_HOT="${UNRAID_DISK_HOT:-45}" + UNRAID_DISK_MAX="${UNRAID_DISK_MAX:-55}" + UNRAID_SSD_HOT="${UNRAID_SSD_HOT:-60}" + UNRAID_SSD_MAX="${UNRAID_SSD_MAX:-70}" +} + +# ----------------------------------------------------------------------------------------------- +# check_local_disk_temps — check local disk temps before rsync +# Reads temps and rotational flag from /var/local/emhttp/disks.ini +# Uses unRAID's own thresholds from dynamix.cfg +# +# Returns: +# 0 = all temps OK +# 1 = warn threshold exceeded (skip this profile) +# 2 = critical threshold exceeded (abort all remaining profiles) +# +# Sets global TEMP_CHECK_RESULT with human readable summary +# ----------------------------------------------------------------------------------------------- +check_local_disk_temps() { + get_unraid_temp_thresholds + + local disks_ini="/var/local/emhttp/disks.ini" + if [[ ! -f "$disks_ini" ]]; then + warn "disks.ini not found — skipping temp check" + TEMP_CHECK_RESULT="temp check skipped (disks.ini not found)" + return 0 + fi + + local worst_result=0 + local hot_drives=() + local crit_drives=() + local current_name="" + local current_device="" + local current_rotational="" + local current_temp="" + + check_drive() { + [[ -z "$current_name" ]] || [[ -z "$current_temp" ]] && return + [[ "$current_temp" -eq 0 ]] && return # spun down + + local warn_thresh crit_thresh + if [[ "$current_rotational" == "0" ]]; then + warn_thresh="$UNRAID_SSD_HOT" + crit_thresh="$UNRAID_SSD_MAX" + else + warn_thresh="$UNRAID_DISK_HOT" + crit_thresh="$UNRAID_DISK_MAX" + fi + + if [[ "$current_temp" -ge "$crit_thresh" ]]; then + crit_drives+=("${current_name}(${current_device}):${current_temp}°C≥${crit_thresh}°C") + [[ $worst_result -lt 2 ]] && worst_result=2 + elif [[ "$current_temp" -ge "$warn_thresh" ]]; then + hot_drives+=("${current_name}(${current_device}):${current_temp}°C≥${warn_thresh}°C") + [[ $worst_result -lt 1 ]] && worst_result=1 + fi + } + + while IFS= read -r ini_line; do + if echo "$ini_line" | grep -qE '^\["(disk[0-9]+|parity[0-9]?|cache[0-9]?)"\]'; then + # Save previous drive before starting new one + check_drive + current_name=$(echo "$ini_line" | grep -o '"[^"]*"' | head -1 | tr -d '"') + current_device="" + current_rotational="1" # default HDD + current_temp="" + elif echo "$ini_line" | grep -q '^device='; then + current_device=$(echo "$ini_line" | cut -d= -f2 | tr -d '"') + elif echo "$ini_line" | grep -q '^rotational='; then + current_rotational=$(echo "$ini_line" | cut -d= -f2 | tr -d '"') + elif echo "$ini_line" | grep -q '^temp='; then + current_temp=$(echo "$ini_line" | cut -d= -f2 | tr -d '"') + current_temp="${current_temp//[^0-9]/}" + current_temp="${current_temp:-0}" + fi + done < "$disks_ini" + check_drive # process last drive + + if [[ ${#crit_drives[@]} -gt 0 ]]; then + TEMP_CHECK_RESULT="CRITICAL temps: ${crit_drives[*]}" + error "$ICON_WARN Drive temp CRITICAL — aborting all remaining syncs: ${crit_drives[*]}" + notify "Rsync aborted on $(hostname) — drive temp CRITICAL: ${crit_drives[*]}" "Rsync Temp Check" "warning" + return 2 + elif [[ ${#hot_drives[@]} -gt 0 ]]; then + TEMP_CHECK_RESULT="HOT drives: ${hot_drives[*]}" + warn "$ICON_WARN Drive temp WARNING — skipping this profile: ${hot_drives[*]}" + notify "Rsync profile skipped on $(hostname) — drive temp WARNING: ${hot_drives[*]}" "Rsync Temp Check" "normal" + return 1 + else + TEMP_CHECK_RESULT="all normal (HDD warn:${UNRAID_DISK_HOT}°C crit:${UNRAID_DISK_MAX}°C SSD warn:${UNRAID_SSD_HOT}°C crit:${UNRAID_SSD_MAX}°C)" + return 0 + fi +} + +# ----------------------------------------------------------------------------------------------- +# check_remote_disks — verify all disks backing a share are healthy on the remote server +# Auto-detects filesystem type from disks.ini — handles XFS and ZFS correctly +# No SKIP_DISK_CHECK needed — detection is automatic +# +# XFS array disks: checks mountpoint is active via mountpoint -q +# ZFS disks/pools: checks zpool status is ONLINE +# Shares spanning multiple disks: all must pass +# # Usage: check_remote_disks "/mnt/user/Movies" # ----------------------------------------------------------------------------------------------- check_remote_disks() { @@ -481,34 +594,104 @@ check_remote_disks() { info "$ICON_DISK Checking disks backing $share_name on $REMOTE_SERVER_NAME..." - DISK_PATHS=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ - "ls -d /mnt/disk*/$share_name 2>/dev/null" 2>/dev/null) + # Get disk→fsType mapping from remote disks.ini + local disks_ini_content + disks_ini_content=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ + "cat /var/local/emhttp/disks.ini 2>/dev/null" 2>/dev/null) - if [[ -z "$DISK_PATHS" ]]; then - error "$ICON_DISK No disks found backing $share_name on $REMOTE_SERVER_NAME" + if [[ -z "$disks_ini_content" ]]; then + error "$ICON_DISK Cannot read disks.ini from $REMOTE_SERVER_NAME" exit 1 fi - local all_ok=true - while IFS= read -r disk_share_path; do - local disk_mount disk_name - disk_mount=$(dirname "$disk_share_path") - disk_name=$(basename "$disk_mount") - MOUNTED=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ - "mountpoint -q '$disk_mount' && echo yes || echo no" 2>/dev/null) - if [[ "$MOUNTED" == "yes" ]]; then - info "$ICON_DISK $disk_name $ICON_RUNNING — $share_name present" + # Find which disk(s) back this share on remote + local backing_disks + backing_disks=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ + "ls -d /mnt/disk*/$share_name 2>/dev/null | awk -F/ '{print \$3}'" 2>/dev/null) + + # Also check ZFS standalone pools (cache, gaming, media-servers etc.) + local zfs_pool_paths + zfs_pool_paths=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ + "zpool list -H -o name 2>/dev/null | while read pool; do + [[ -d \"/mnt/\${pool}/$share_name\" ]] && echo \"\$pool\" + done" 2>/dev/null) + + if [[ -z "$backing_disks" ]] && [[ -z "$zfs_pool_paths" ]]; then + # Check cache pool directly + local on_cache + on_cache=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ + "[[ -d '/mnt/cache/$share_name' ]] && echo yes" 2>/dev/null) + if [[ "$on_cache" == "yes" ]]; then + backing_disks="" + zfs_pool_paths="cache" else - error "$ICON_DISK $disk_name $ICON_STOPPED — $share_name missing" - all_ok=false + error "$ICON_DISK No disks found backing $share_name on $REMOTE_SERVER_NAME" + exit 1 fi - done <<< "$DISK_PATHS" + fi + + local all_ok=true + + # Check array disks (XFS or ZFS single-disk-in-array) + if [[ -n "$backing_disks" ]]; then + while IFS= read -r disk_name; do + [[ -z "$disk_name" ]] && continue + + # Get fsType for this disk from disks.ini + local fs_type + fs_type=$(echo "$disks_ini_content" | awk -F= -v disk="$disk_name" ' + /^\["'"'"'?/ { current=substr($0,3,length($0)-4) } + current==disk && /^fsType=/ { print $2; exit } + ' | tr -d '"') + fs_type="${fs_type:-xfs}" + + if [[ "$fs_type" == "zfs" ]]; then + # ZFS single disk in array — check zpool status + local pool_health + pool_health=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ + "zpool list -H -o health '$disk_name' 2>/dev/null" 2>/dev/null) + if [[ "$pool_health" == "ONLINE" ]]; then + info "$ICON_DISK $disk_name (ZFS) $ICON_RUNNING — $share_name ONLINE" + else + error "$ICON_DISK $disk_name (ZFS) $ICON_STOPPED — pool ${pool_health:-offline}" + all_ok=false + fi + else + # XFS — check mountpoint + local mounted + mounted=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ + "mountpoint -q '/mnt/$disk_name' && echo yes || echo no" 2>/dev/null) + if [[ "$mounted" == "yes" ]]; then + info "$ICON_DISK $disk_name (XFS) $ICON_RUNNING — $share_name present" + else + error "$ICON_DISK $disk_name (XFS) $ICON_STOPPED — not mounted" + all_ok=false + fi + fi + done <<< "$backing_disks" + fi + + # Check ZFS standalone pools + if [[ -n "$zfs_pool_paths" ]]; then + while IFS= read -r pool_name; do + [[ -z "$pool_name" ]] && continue + local pool_health + pool_health=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ + "zpool list -H -o health '$pool_name' 2>/dev/null" 2>/dev/null) + if [[ "$pool_health" == "ONLINE" ]]; then + info "$ICON_DISK $pool_name (ZFS pool) $ICON_RUNNING — $share_name ONLINE" + else + error "$ICON_DISK $pool_name (ZFS pool) $ICON_STOPPED — pool ${pool_health:-offline}" + all_ok=false + fi + done <<< "$zfs_pool_paths" + fi if [[ "$all_ok" == false ]]; then error "One or more disks backing $share_name are offline on $REMOTE_SERVER_NAME" exit 1 fi - success "All disks backing $share_name are online" + success "All disks backing $share_name are online ✅" } # ----------------------------------------------------------------------------------------------- @@ -905,6 +1088,63 @@ translate_path() { fi } +# ----------------------------------------------------------------------------------------------- +# check_arr_version — verify arr major version matches tested version in Master.conf +# Exits the calling script if version doesn't match — prevents running against untested API +# +# Usage: +# check_arr_version "$RADARR_URL" "$RADARR_API_KEY" "v3" "$RADARR_VERSION_MAJOR" "Radarr" +# +# Arguments: +# $1 = arr base URL +# $2 = API key +# $3 = API path prefix (v3 or v1) +# $4 = expected major version number +# $5 = arr name for error messages +# ----------------------------------------------------------------------------------------------- +check_arr_version() { + local url="$1" + local api_key="$2" + local api_prefix="$3" + local expected_major="$4" + local arr_name="${5:-Arr}" + + local status_response version major_version + + status_response=$(curl -sf --max-time 10 \ + -H "X-Api-Key: $api_key" \ + "${url}/api/${api_prefix}/system/status" 2>/dev/null) + + if [[ -z "$status_response" ]]; then + warn "$arr_name version check failed — could not reach system/status endpoint" + warn "Proceeding without version verification — monitor for API errors" + return 0 + fi + + version=$(echo "$status_response" | \ + grep -o '"version":"[^"]*"' | \ + grep -o '[0-9][^"]*' | head -1) + + if [[ -z "$version" ]]; then + warn "$arr_name version check failed — could not parse version from response" + warn "Proceeding without version verification — monitor for API errors" + return 0 + fi + + major_version="${version%%.*}" + + if [[ "$major_version" == "$expected_major" ]]; then + success "$arr_name version: $version (major $major_version — tested ✅)" + return 0 + else + error "$arr_name version mismatch — running v${major_version}, tested against v${expected_major}" + error "The API endpoint structure may have changed — exiting to protect your library" + error "Update ${arr_name^^}_VERSION_MAJOR in Master.conf after verifying the script works with v${major_version}" + notify "$arr_name version mismatch on $(hostname) — running v${major_version}, script tested against v${expected_major}" "$arr_name Cleanup" "warning" + exit 1 + fi +} + check_api() { local url="$1" local service="${2:-API}" diff --git a/safe_master.conf b/safe_master.conf new file mode 100644 index 0000000..1fe15fb --- /dev/null +++ b/safe_master.conf @@ -0,0 +1,1407 @@ +#!/bin/bash +# ============================================================================================== +# ================================= MASTER CONFIGURATION ======================================= +# ============================================================================================== +# All user-facing variables for the unRAID script ecosystem. +# Scripts source this file — edit here, changes apply everywhere on next git pull. +# +# ── HOW THIS FILE WORKS ─────────────────────────────────────────────────────────────────────── +# Every script sources Master.conf and common.sh at startup. +# Change a value here and it affects all scripts that use it — no hunting through files. +# To disable something: comment it out with # rather than deleting it. +# To add a new rsync profile: add a key to each PROFILE_* array. +# To add or remove orchestrator jobs: edit the arrays in the ORCHESTRATORS section. +# +# ── INDEX ───────────────────────────────────────────────────────────────────────────────────── +# +# Section Description +# ─────────────────────────────────────────────────────────────────────────────────────────── +# HOST CONFIGURATION Server hostnames, SSH keys, Emby connection details, DATA_DIR +# LOGGING Enable or disable verbose logging +# NOTIFICATIONS unRAID native and Discord webhook settings +# GIT / REPO Gitea repository and SSH settings +# +# ── ORCHESTRATORS ────────────────────────────────────────────────────────────────────────── +# ARRAY START Scripts launched at array start (array_start.sh) +# DAILY SYNC MAINTENANCE Job list + media shares (daily_sync_maintenance.sh) +# WEEKLY SYNC MAINTENANCE Job list + sync jobs + sync settings (weekly_sync_maintenance.sh) +# MEDIA MANAGEMENT Job list for media_management.sh +# +# ── RSYNC ────────────────────────────────────────────────────────────────────────────────── +# RSYNC DEFAULTS Global fallback rsync options and limits +# REMOTE HEALTH CHECKS Rootfs threshold for pre-flight abort +# RSYNC PROFILE SYSTEM Per-profile overrides for appdata syncs +# +# ── FAILOVER ─────────────────────────────────────────────────────────────────────────────── +# FAILOVER Mutual container failover between two servers +# FAILOVER TEST Simulated outage settings for failover_test.sh +# DDNS Script-controlled DDNS — absolute rules +# INTERNET LOSS Containers to stop when internet is lost +# TIERED CONTAINER LISTS What each server runs for the other per tier +# TIER DELAY SETTINGS How long before each tier activates (minutes) +# RSYNC WRITEBACK JOBS Appdata synced back to primary on handback +# +# ── DOCKER ESSENTIALS ────────────────────────────────────────────────────────────────────── +# DOCKER DAILY RESTART Containers restarted daily +# DOCKER WEEKLY RESTART Containers restarted weekly +# DOCKER WATCHDOG Continuous two-tier self-healing container monitoring +# DOCKER NETWORK CONNECT Connect containers to extra networks on array start +# +# ── UNRAID ESSENTIALS ────────────────────────────────────────────────────────────────────── +# REBOOT User warning delay before scheduled reboot +# MOVER Mover stop timeout +# SYSLOG FILTER Docker veth noise filter file path +# PHP-FPM PHP-FPM max children config +# CLEAR LOGS System log file paths +# WEBGUI WATCHDOG WebGUI nginx + emhttp monitoring and restart +# +# ── MEDIA ────────────────────────────────────────────────────────────────────────────────── +# MEDIA PERMISSIONS Share list, mode and owner for permissions script +# MEDIA CLEANER Anime and media folder lists and file patterns +# ARR CLEANUP Lidarr, Sonarr, Radarr orphan file cleanup +# ARR FAILED/STALLED RECOVERY Auto blocklist + re-search failed imports and stalled downloads +# +# ── TRANSCODES ───────────────────────────────────────────────────────────────────────────── +# TRANSCODE MANAGER Ramdisk and SSD fallback transcode management +# TRANSCODE SERVER ARRAY Multi-server session monitoring (Emby, Jellyfin, Plex) +# +# ── MONITORS ─────────────────────────────────────────────────────────────────────────────── +# CERTIFICATE MONITOR SSL certificate expiry monitoring +# BACKUP VERIFY Random sample checksum verification against remote +# SMART HEALTH Drive SMART attribute monitoring +# ZFS MEMORY SNAPSHOT Weekly ZFS health and memory diagnostic report +# BANDWIDTH MONITOR Daily rsync transfer logging and weekly summary +# HEALTH DIGEST Aggregated system health digest — always/smart/weekly +# EMBY SESSION REPORT Weekly Emby usage statistics via API +# +# ── SYSTEM WATCHDOG ──────────────────────────────────────────────────────────────────────── +# SYSTEM WATCHDOG Continuous system health monitoring — last line of defense +# +# ============================================================================================== + +# ============================================================================================== +# ── HOST CONFIGURATION ──────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +# ━━━ Hosts ━━━ +# Hostnames must match exact Docker/unRAID hostnames — case sensitive. +# Used by detect_hosts() in common.sh to determine which server is local and which is remote. +# Both servers run identical scripts — host detection makes them bidirectional. + HOST1="your-host1-hostname" + HOST2="your-host2-hostname" + +# Data directory — persistent script state and statistics files. +# Array share — survives reboots, no flash drive wear. +# Created automatically if it doesn't exist. +# Only truly critical files (failover state, watchdog reboot log) stay on /boot/config. + DATA_DIR="/mnt/user/appdata/unraid_scripts/data" + +# SSH keys for server-to-server rsync and failover container operations. +# Both keys must be in /root/.ssh/ and authorised in the remote server's authorized_keys. + HOST1_SSH_KEY="/root/.ssh/your_key" + HOST2_SSH_KEY="/root/.ssh/your_key" + +# ━━━ Emby ━━━ +# Defined once here — referenced by transcode_manager.sh, emby_session_report.sh, +# emby_database_repair.sh, weekly_sync_maintenance.sh, and TRANSCODE_SERVERS array. +# API key: Emby Dashboard → API Keys → + New Key + HOST1_EMBY_CONTAINER="Emby" + HOST1_EMBY_URL="http://localhost:8096" + HOST1_EMBY_API_KEY="your-emby-api-key" + + HOST2_EMBY_CONTAINER="Emby-Secondary" + HOST2_EMBY_URL="http://localhost:8096" # same port — different server, different key + HOST2_EMBY_API_KEY="your-emby-api-key" + +# ============================================================================================== +# ── LOGGING ─────────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +# Controls verbose [LOG] output across all scripts. +# true = show detailed [LOG] lines — useful for debugging or first-time setup +# false = show only user-facing output — cleaner for scheduled runs + ENABLE_LOGGING=true + +# ============================================================================================== +# ── NOTIFICATIONS ───────────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +# unRAID native notification system — integrates with the bell icon in the WebGUI. +# normal = job completed successfully / warning = something failed or needs attention + NOTIFY_UNRAID=true + +# Discord webhook URL — leave blank to disable + DISCORD_WEBHOOK="" + +# ============================================================================================== +# ── GIT / REPO ──────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +# Gitea self-hosted repository — used by git_pull_execute.sh. +# Detects Gitea container location at runtime — works through failover automatically. +# Falls back to GITEA_DOMAIN if local and Tailscale both fail. + GITEA_CONTAINER="Gitea" # exact Docker container name + GITEA_REPO_PATH="youruser/Unraid_Scripts.git" # repo path on Gitea server + GITEA_DOMAIN="" # e.g. git.gmer4lfe.com — requires NPM + DNS setup + TARGET_DIR="/mnt/user/appdata/unraid_scripts" # where scripts are cloned to + GITEA_SSH_KEY="/root/.ssh/your_gitea_key" # SSH key for authenticating to Gitea + SSH_PORT=221 # Gitea SSH port (default 22, Gitea often uses 222/221) + +# ============================================================================================== +# ── ORCHESTRATORS ────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# All orchestrator job lists live here — edit arrays to add/remove scripts. +# No changes to orchestrator scripts needed when adding or removing jobs. + +# ━━━ Array Start ━━━ +# Scripts launched by array_start.sh when the array comes online. +# Launched in order — each as a background process. +# One-shot scripts (ramdisk, syslog, fpm, network) run and exit naturally. +# Continuous scripts (watchdogs, failover) run until array stops. + +ARRAY_START_SCRIPTS=( + "Transcodes/ramdisk_setup.sh" # creates ramdisk + symlink before Emby starts + "unRAID_Essentials/docker_syslog_filter.sh" # suppress veth noise before logs fill + "unRAID_Essentials/php_fpm_max_children.sh" # WebGUI performance tuning + "unRAID_Essentials/inotify_tuning.sh" # bump inotify limits — containers miss events if exhausted + "Docker_Essentials/docker_network_connect.sh" # ensure networks exist + connect containers + "unRAID_Essentials/system_watchdog.sh" # system health monitor — continuous loop + "Docker_Essentials/docker_watchdog.sh" # container health monitor — continuous loop + "Failover/failover.sh" # mutual failover — continuous loop +) + +# ━━━ Daily Sync Maintenance ━━━ +# daily_sync_maintenance.sh runs the media share sync built into the script first, +# then iterates DAILY_MAINTENANCE_SCRIPTS for additional jobs. +# Schedule: 0 1 * * * (1am daily) + +DAILY_MAINTENANCE_SCRIPTS=( + "git_pull_execute.sh" # pull latest scripts — always runs first + "Docker_Essentials/docker_daily_restart.sh" # daily container restarts +) + +# Media shares synced daily by daily_sync_maintenance.sh. +# Each server syncs only the shares it owns (source of truth) — direction is automatic. +# HOST1 pushes its truth shares to HOST2. HOST2 pushes its truth shares to HOST1. +# Never both pushing the same share — one server is always the truth holder. +# These shares use DEFAULT_RSYNC_OPTS — no profile entry needed. +# For shares needing custom options or container stops — create a profile in the RSYNC section. + +HOST1_DAILY_SYNC_SHARES=( + /mnt/user/Books + /mnt/user/Intros + /mnt/user/Kids_Movies + /mnt/user/Kids_Tv_Shows + /mnt/user/Movies + /mnt/user/Music + /mnt/user/Music_Videos + /mnt/user/Nextcloud + /mnt/user/stand-up_comedy + /mnt/user/Sports + /mnt/user/Tv_Shows + /mnt/user/Anime_Shows-Old + /mnt/user/Anime_Movies-Old +) + +HOST2_DAILY_SYNC_SHARES=( + /mnt/user/Anime_Movies + /mnt/user/Anime_Shows +) + +# Personal encrypted shares — synced for offsite backup, independent of media shares. +# ZFS encrypted at dataset level — remote receives encrypted blocks, cannot read content. +# See README-Rsync_Setup.md for ZFS encryption setup before uncommenting. +HOST1_PERSONAL_SHARES=( + # /mnt/user/Host1-Personal # uncomment after creating encrypted dataset +) + +HOST2_PERSONAL_SHARES=( + # /mnt/user/Host2-Personal # uncomment after creating encrypted dataset +) + +# ━━━ Weekly Sync Maintenance ━━━ +# weekly_sync_maintenance.sh handles the critical sync built into the script first: +# stop containers both sides → pull updates → sync Emby + Critical-Data → restart +# Then iterates WEEKLY_MAINTENANCE_SCRIPTS for additional jobs. +# Schedule: 30 2 * * 0 (Sunday 2:30am) + +WEEKLY_MAINTENANCE_SCRIPTS=( + "Docker_Essentials/docker_weekly_restart.sh" # weekly container restarts after sync +) + +# Shares synced by weekly_sync_maintenance.sh during the maintenance window. +# Containers are stopped both sides before these sync — full clean state guaranteed. +# Profiles drive container stops, excludes, and options — configure in RSYNC section. +# Order matters — Emby first, then auth stack. +WEEKLY_SYNC_JOBS=( + "/mnt/user/Media_Server/Emby" # emby profile — full clean mirror + "/mnt/user/appdata-Failover/Critical-Data" # critical-data profile — auth stack +) + +# Container update toggles for the weekly sync window. +# Containers are already stopped for the sync — updates pull at no extra downtime. +# Both false → sync only, no updates. +# Toggle false temporarily to skip updates without changing the schedule. + CRITICAL_SYNC_UPDATES=true # pull container updates locally + CRITICAL_SYNC_UPDATES_REMOTE=true # pull container updates on remote via SSH + +# ━━━ Media Management ━━━ +# Job list run directly by daily_sync_maintenance.sh after the media share sync. +# Runs sequentially — permissions first, then cleaners, then arr cleanup. +# Comment out any job to disable without removing it. +# Each individual script can still be run manually for one-off maintenance. + +MEDIA_MANAGEMENT_JOBS=( + "Media/media_shares_permissions.sh" # apply permissions — runs first + "Media/media_cleaner.sh anime" # remove junk from anime shares + "Media/media_cleaner.sh media" # remove junk from media shares + "Media/lidarr_cleanup.sh" # remove orphaned music files + "Media/sonarr_cleanup.sh" # remove orphaned TV files + "Media/radarr_cleanup.sh" # remove orphaned movie files + "Docker_Essentials/downloaders_reset.sh" # clear stuck states + purge old history +) + +# ============================================================================================== +# ── RSYNC ───────────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +# ━━━ Rsync Defaults ━━━ +# Global fallback values used when no profile match is found. +# Media shares in HOST*_DAILY_SYNC_SHARES always use these globals — no profile needed. +# Appdata shares match profiles by directory basename (lowercased). +# If a profile key exists it overrides the global. If missing the global is used. + + BW_LIMIT=12500 # KB/s — 12500 ≈ 100Mbit — network transfer speed cap + RETRY_COUNT=3 # retry attempts if rsync fails before giving up + SLEEP=300 # seconds between retry attempts + CRITICAL_CONTAINER_NAMES=() # containers to stop on REMOTE before rsync — profiles override + DELAYED_CONTAINERS=() # containers needing delay before starting — profiles override + CONTAINER_DELAY=5 # seconds to wait before starting delayed containers + EXCLUDE_DIRS=() # directories to exclude from transfer — profiles override + +# --delete removes files on remote that no longer exist on source (mirror behaviour) +# --inplace writes directly to destination — better for large files, avoids temp copies +# --no-whole-file forces delta transfer even on fast connections — sends only changed blocks + DEFAULT_RSYNC_OPTS=(-av --info=progress2 --human-readable --bwlimit="$BW_LIMIT" --delete --inplace --no-whole-file) + +# ━━━ Remote Health Checks ━━━ +# Pre-flight check — aborts if remote rootfs (/) usage is at or above this percentage. +# When remote array is down, rsync writes land on rootfs — fills fast and crashes the server. + ROOTFS_WARN=75 + +# ━━━ Rsync Profile System ━━━ +# Profiles allow per-share rsync behaviour without touching script logic. +# Profile key matched by basename of directory passed to rsync.sh (lowercased). +# Override with --profile=name flag. +# +# IMPORTANT: PROFILE_RSYNC_OPTS does NOT inherit DEFAULT_RSYNC_OPTS. +# List ALL desired options explicitly when defining a profile. +# +# Current profiles: +# arrs_stack — arr databases — lower bandwidth, containers stopped for consistency +# critical-data — auth stack — containers stopped both sides, Authelia delayed start +# gmer4lfe — server-specific appdata — no container stops needed +# important-data — NextCloud + Postgres — NextCloud delayed start after Postgres +# emby — weekly clean sync — both Emby stopped, full mirror, minimal excludes +# called by weekly_sync_maintenance.sh only — do NOT schedule separately +# emby-failover — frequent dirty sync — Emby stays running, WAL excluded, critical data only +# also used for failover writeback on handback + +declare -A PROFILE_RSYNC_OPTS=( + [arrs_stack]="-av --info=progress2 --human-readable --bwlimit=$BW_LIMIT --delete --inplace" + [critical-data]="-av --human-readable --bwlimit=$BW_LIMIT --delete" + [gmer4lfe]="-av --info=progress2 --bwlimit=$BW_LIMIT" + [important-data]="-av --human-readable --bwlimit=$BW_LIMIT" + [emby]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file" + [emby-failover]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file" +) + +# Per-profile bandwidth limits in KB/s — overrides global BW_LIMIT for that profile only +# Lower for shares running alongside other jobs, higher for time-sensitive critical data +declare -A PROFILE_BW_LIMIT=( + [arrs_stack]=5000 # lower — runs alongside other syncs, avoids saturating link + [critical-data]=9500 # high — small dataset, get it synced fast and clean + [gmer4lfe]=8000 + [important-data]=9500 # high — database sync needs to be fast + [emby]=8000 # medium — large full mirror, steady transfer + [emby-failover]=9500 # high — small critical dataset, sync as fast as possible +) + +# Retry attempts per profile — how many times to retry before giving up on a failed sync +declare -A PROFILE_RETRY_COUNT=( + [arrs_stack]=3 + [critical-data]=3 + [gmer4lfe]=3 + [important-data]=3 + [emby]=3 + [emby-failover]=3 +) + +# Seconds to wait between retry attempts +# emby-failover shorter — frequent sync, faster retry on transient failures +declare -A PROFILE_SLEEP=( + [arrs_stack]=300 + [critical-data]=300 + [gmer4lfe]=300 + [important-data]=300 + [emby]=300 + [emby-failover]=120 # shorter — frequent dirty sync, retry faster +) + +# Containers stopped on BOTH LOCAL and REMOTE servers before rsync. +# Local stops first — flushes databases cleanly before pushing data out. +# Remote stops next — prevents writes to destination while receiving. +# Only containers that were running get restarted — stopped containers stay stopped. +# Same container names on both servers — consistent naming is required by this ecosystem. +# If a container is not found on a server it is skipped gracefully, not errored. +# SPACE-SEPARATED STRINGS — converted to array at runtime +declare -A PROFILE_CRITICAL_CONTAINER_NAMES=( + [arrs_stack]="Sonarr Lidarr Readarr Radarr Prowlarr Bazarr Pinchflat" + [critical-data]="Mariadb-Authelia Mariadb-Authelia-Secondary Redis-Authelia Redis-Authelia-Secondary Lldap NginxProxyManager Authelia Authelia-Secondary" + [gmer4lfe]="Organizrv2 UptimeKuma VaultWarden" + [important-data]="Postgres-NextCloud NextCloud" + [emby]="Emby" # weekly clean sync — both Emby instances stopped, WAL checkpointed + [emby-failover]="" # dirty sync — Emby stays running both sides, WAL excluded from sync +) + +# Containers that need a delay before starting after rsync completes. +# Database containers must be accepting connections before dependent apps start. +# Authelia waits for Mariadb + Redis. NextCloud waits for Postgres. +# SPACE-SEPARATED STRINGS — converted to array at runtime +declare -A PROFILE_DELAYED_CONTAINERS=( + [arrs_stack]="" + [critical-data]="Authelia Authelia-Secondary" # wait for Mariadb + Redis to be ready + [gmer4lfe]="" + [important-data]="NextCloud" # wait for Postgres to accept connections + [emby]="" + [emby-failover]="" +) + +# Seconds to wait before starting delayed containers +# 15s gives Mariadb, Redis, and LLDAP time to accept connections before Authelia starts +declare -A PROFILE_CONTAINER_DELAY=( + [arrs_stack]=5 + [critical-data]=15 # Mariadb + Redis need time to accept connections + [gmer4lfe]=5 + [important-data]=10 # Postgres needs time before NextCloud + [emby]=5 + [emby-failover]=5 +) + +# Directories excluded from rsync transfer per profile +# emby-failover excludes WAL files — safe to sync while Emby is running +# emby clean sync only excludes logs, transcodes, cache — full metadata mirror +# SPACE-SEPARATED STRINGS — converted to array at runtime +declare -A PROFILE_EXCLUDE_DIRS=( + [arrs_stack]="logs *.tmp" + [gmer4lfe]="logs *.tmp" + [important-data]="logs *.tmp" + [critical-data]="logs *.tmp *.log nginx/temp nginx/cache __pycache__ notification.txt" + [emby]="logs transcodes cache crash*" + # emby-failover: Emby running, WAL excluded — only safe critical data synced + # users.db, library.db, authentication.db, config/ — everything else excluded + [emby-failover]="logs transcodes cache metadata *.db-wal *.db-shm crash* plugins root" +) + +# Skip per-disk space check for these profiles — appdata syncs go to cache/appdata +# not to array disks, so disk space check is irrelevant and just slows things down +declare -A PROFILE_SKIP_DISK_CHECK=( + [arrs_stack]=true + [critical-data]=true + [gmer4lfe]=true + [important-data]=true + [emby]=true + [emby-failover]=true +) + +# ============================================================================================== +# ── FAILOVER ────────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# Mutual container failover between two unRAID servers. +# Each server runs Failover/failover.sh independently via array_start.sh. +# All decisions based on two pings: remote reachable + internet reachable. +# +# States: NORMAL | FAILOVER | NO_INTERNET | DARK +# +# DDNS rules — absolute: +# Internet loss → stop own DDNS immediately +# Failover → start remote DDNS first (Tier 1) +# Handback → stop remote DDNS → rsync → start containers → start local DDNS last +# +# Tiers: +# Tier 1 — Immediate — vital services + Live TV +# Tier 2 — configurable delay — productivity services +# Tier 3 — configurable delay — secondary services +# Tier 4 — configurable delay — arrs + downloaders + + EXTERNAL_IP="8.8.8.8" + FAILOVER_CHECK_INTERVAL=120 + FAILOVER_HANDBACK_STRIKES=2 + FAILOVER_STATE_FILE="/boot/config/failover_state.db" + FAILOVER_ENABLED=false # set true when HOST2 is back online and tested + # false = suppresses "not running" warnings in status scripts + +# ━━━ Failover Test ━━━ + FAILOVER_TEST_BLOCK_WAIT=150 + FAILOVER_TEST_HANDBACK_WAIT=360 + +# ━━━ DDNS ━━━ +HOST1_DDNS_CONTAINERS=( + "yourdomain.com" +) + +HOST2_DDNS_CONTAINERS=( + "yourdomain.us" +) + +# ━━━ Internet Loss ━━━ +FAILOVER_HOST1_STOP_ON_NO_NET=( + "yourdomain.com" +) + +FAILOVER_HOST2_STOP_ON_NO_NET=( + "yourdomain.us" +) + +# ━━━ Tiered Container Lists ━━━ + +# HOST1 runs for HOST2 +FAILOVER_HOST1_RUNS_FOR_HOST2_IMMEDIATE=( + "yourdomain.us" + "VaultWarden-Secondary" + # "container-placeholder" +) + +FAILOVER_HOST1_RUNS_FOR_HOST2_2HR=( + # "container-placeholder" +) + +FAILOVER_HOST1_RUNS_FOR_HOST2_6HR=( + # "container-placeholder" +) + +FAILOVER_HOST1_RUNS_FOR_HOST2_18HR=( + # "container-placeholder" +) + +# HOST2 runs for HOST1 +FAILOVER_HOST2_RUNS_FOR_HOST1_IMMEDIATE=( + "yourdomain.com" + "Emby" + "VaultWarden" + "Dispatcharr" + "Dispatcharr-Basic" + "Dispatcharr-Iptv-Users" + "ErsatzTV-Emby" +) + +FAILOVER_HOST2_RUNS_FOR_HOST1_2HR=( + "Postgres-NextCloud" + "NextCloud" + "PostgreSQL_Immich" + "Immich" + # "container-placeholder" +) + +FAILOVER_HOST2_RUNS_FOR_HOST1_6HR=( + "Gitea" + # "container-placeholder" +) + +FAILOVER_HOST2_RUNS_FOR_HOST1_18HR=( + "Sonarr" + "Radarr" + "Lidarr" + "Readarr" + "Prowlarr" + "Bazarr" + "SABnzbd" + "Qbittorrent" + "LidaTube" + "Pinchflat" + "ChannelTube" + # "container-placeholder" +) + +# ━━━ Tier Delay Settings ━━━ +# How long the primary server must be down before each tier activates — in minutes. +# Tier 1 is always immediate — Live TV and media can't wait. +# Set independently per host — adjust based on hardware and what's worth starting. +# Longer delays = less resource usage on covering server but slower recovery. +# +# HOST1's containers running on HOST2 (HOST1 is down): +HOST1_TIER2_DELAY=240 # 4 hours — NextCloud, Immich — can wait +HOST1_TIER3_DELAY=720 # 12 hours — secondary services — Gitea etc. +HOST1_TIER4_DELAY=1440 # 24 hours — full workflow — arrs and downloaders + +# HOST2's containers running on HOST1 (HOST2 is down): +HOST2_TIER2_DELAY=240 +HOST2_TIER3_DELAY=720 +HOST2_TIER4_DELAY=1440 + +# ━━━ Rsync Writeback Jobs ━━━ +# Syncs critical appdata BACK to primary server during handback after failover. +# Containers are stopped before writeback runs — clean source, no competing writes. +# Purpose: primary comes back online with the state that built up during its outage +# (watch states, auth changes, library updates that happened on HOST2) +# +# HOST*_TIER1_WRITEBACK_DELAY: +# Short outages skip Tier 1 writeback — primary state is more reliable than dirty sync data +# Only writeback if outage lasted longer than this many minutes +# 60 minutes = if HOST1 was down less than 1hr, don't bother writing back Emby +# +# Tier 4 writeback automatically syncs HOST*_DAILY_SYNC_SHARES back — no need to list those here +# Only add paths that are NOT in DAILY_SYNC_SHARES and need writeback after extended outage + +HOST1_TIER1_WRITEBACK_DELAY=60 # minutes — skip Emby writeback if outage under 1hr +HOST2_TIER1_WRITEBACK_DELAY=60 + +# HOST1 writeback — run by HOST2 during HOST1 handback +FAILOVER_HOST1_WRITEBACK_TIER1=( + "/mnt/user/Media_Server/Emby" # watch states, playstates built up during outage +) + +FAILOVER_HOST1_WRITEBACK_TIER2=( + "/mnt/user/appdata-Failover/Important-Data" # NextCloud + Postgres — files added during outage +) + +FAILOVER_HOST1_WRITEBACK_TIER3=( + # "location-placeholder" +) + +FAILOVER_HOST1_WRITEBACK_TIER4=( + # Edge cases outside HOST1_DAILY_SYNC_SHARES + "/mnt/user/appdata-Failover/Arrs_Stack" # arr databases — downloads queued during outage +) + +# HOST2 writeback — run by HOST1 during HOST2 handback +FAILOVER_HOST2_WRITEBACK_TIER1=( + # "/mnt/user/appdata-Failover/Host2-Emby" +) + +FAILOVER_HOST2_WRITEBACK_TIER2=( + # "/mnt/user/appdata-Failover/Host2-Important" +) + +FAILOVER_HOST2_WRITEBACK_TIER3=( + # "location-placeholder" +) + +FAILOVER_HOST2_WRITEBACK_TIER4=( + # Edge cases outside HOST2_DAILY_SYNC_SHARES + "/mnt/user/appdata-Failover/Arrs_Stack" +) + +# ============================================================================================== +# ── DOCKER ESSENTIALS ───────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +# ━━━ Downloaders Reset ━━━ +# Daily maintenance reset for all download clients. +# Called by daily_sync_maintenance.sh via MEDIA_MANAGEMENT_JOBS before container restarts. +# Clears stuck states, purges old history, prepares each downloader for a clean daily cycle. +# +# Retention period — applies to: slskd failed imports, SABnzbd completed and failed history + DOWNLOADER_RETENTION_DAYS=7 + +# ── slskd ── +# Clears stuck/errored searches, dead transfer records, purges expired failed imports +# SLSKD_FAILED_IMPORTS_DIR: where Soularr moves albums Lidarr rejected + HOST1_SLSKD_URL="http://localhost:8980" + HOST1_SLSKD_API_KEY="your-slskd-api-key" + HOST1_SLSKD_FAILED_IMPORTS_DIR="/mnt/user/Temp_Storage/Slskd/completed/failed_imports" + +# ── SABnzbd ── +# Clears completed history, failed history, and stalled paused queue items + HOST1_SABNZBD_URL="http://localhost:8180" + HOST1_SABNZBD_API_KEY="your-sabnzbd-api-key" + +# ── qBittorrent ── +# Last chance failsafe — deletes torrents older than QBIT_FAILSAFE_MIN_DAYS +# qBittorrent's own rules handle normal cleanup (ratio >= 1.25 OR 45 days inactive) +# This catches anything missed after extended time +# deleteFiles=false — removes torrent from qBit but leaves files on disk +# Radarr/Sonarr manage actual files independently +# QBIT_FAILSAFE_MIN_RATIO=0 disables ratio gate — age is the only condition + HOST1_QBIT_URL="http://localhost:8080" + HOST1_QBIT_USERNAME="admin" + HOST1_QBIT_PASSWORD="your-qbit-password" + QBIT_FAILSAFE_MIN_DAYS=180 + QBIT_FAILSAFE_MIN_RATIO=0 # 0 = age only, no ratio requirement + +# ━━━ Docker Daily Restart ━━━ +# Containers restarted every day by docker_daily_restart.sh via daily_sync_maintenance.sh. +# These containers run better with a daily restart — not just "keeping things fresh". +# Dispatcharr specifically degrades over time without restart — daily is intentional. +# Schedule is set in daily_sync_maintenance.sh — runs at 1am as part of daily window. +# Case-sensitive — must match exact Docker container names. +DAILY_RESTART_CONTAINERS=( + "NginxProxyManager" + "Authelia" + "Dispatcharr-Iptv-Users" + "Dispatcharr" # Live TV scheduler — degrades without daily restart + "Dispatcharr-Basic" + "ErsatzTV-Emby" +) + +# ━━━ Docker Weekly Restart ━━━ +# Less critical services restarted weekly by docker_weekly_restart.sh. +# Called by weekly_sync_maintenance.sh Sunday 2:30am — containers already stopped +# for the weekly sync window so restart adds zero extra downtime. +# Weekly restarts also catch any pending image updates not applied during weekly sync. +WEEKLY_RESTART_CONTAINERS=( + "NextCloud" + "Organizrv2" + "AdGuard-Home" + "Immich" +) + +# ━━━ Docker Watchdog ━━━ +# Continuous two-tier self-healing container monitoring. +# Started by array_start.sh — runs until array stops. +# Re-sources Master.conf each cycle — add/remove containers without restarting watchdog. +# Silent when all healthy — only logs when something needs attention. +# Heartbeat fires periodically as proof of life even when everything is healthy. +# +# Tier 1 — strict monitoring of explicitly configured containers: +# Memory hard limits — immediate restart if container exceeds limit +# CPU thresholds — strike system, restart after CPU_FAIL_LIMIT sustained strikes +# HTTP responsiveness — strike system, restart after RESP_FAIL_LIMIT failed checks +# Required containers — must always be running, strike + skip list with auto-clear +# +# Tier 2 — global health scan of ALL running containers: +# Unhealthy status — Docker HEALTHCHECK unhealthy → restart +# OOM killed — kernel killed container → restart + notify +# Crash loop detection — RestartCount climbing → notify, critical above limit +# Dead containers — remove and restart +# Unexpected exits — non-zero exit code → restart + +# Memory hard limits in MB — immediate restart if exceeded +# Container restarted the moment it crosses this line — no strike system +# 20GB=20480 16GB=16384 12GB=12288 10GB=10240 +# 8GB=8192 6GB=6144 4GB=4096 2GB=2048 1GB=1024 +declare -A WATCHDOG_CONTAINERS=( + ["Emby"]=16384 + ["LidaTube"]=6144 + ["Tdarr"]=6144 + ["Code-Server"]=1024 +) + +# HTTP health check URLs — checked every cycle, strike system before restart +# Container must respond with HTTP 200 within CURL_TIMEOUT seconds +# Per-host — HOST1 and HOST2 may run different containers on different ports +declare -A HOST1_WATCHDOG_CONTAINER_URLS=( + ["Emby"]="http://localhost:8096" +) + +declare -A HOST2_WATCHDOG_CONTAINER_URLS=( + ["Emby"]="http://localhost:8096" +) + +# Required containers — must always be running +# Strike system: SYS_WATCHDOG_STRIKE_LIMIT strikes before restart attempt +# Persistent skip list: added after WATCHDOG_CONTAINER_RESTART_LIMIT restarts in window +# Skip list auto-clears when container recovers — no manual intervention needed +# Per-host — each server has different critical containers +HOST1_WATCHDOG_REQUIRED_CONTAINERS=( + "NginxProxyManager" + "Lldap" + "Authelia" + "Mariadb-Authelia" + "Redis-Authelia" + "Authelia-Secondary" + "Redis-Authelia-Secondary" +) + +HOST2_WATCHDOG_REQUIRED_CONTAINERS=( + "NginxProxyManager" + # add HOST2 required containers here +) + +# Strike state file — /tmp resets on reboot which is correct +# Fresh start after reboot means no stale strikes carrying over + WATCHDOG_STATE_FILE="/tmp/container_watchdog_state.db" + +# CPU thresholds — normalised against total core count automatically at runtime +# SOFT = warn only, HARD = strike toward restart +# CPU_FAIL_LIMIT = consecutive HARD strikes before restart + SOFT_CPU_THRESHOLD=80 # warn at this % of total system CPU + HARD_CPU_THRESHOLD=85 # strike at this % of total system CPU + CPU_FAIL_LIMIT=2 # consecutive hard CPU strikes before container restart + +# Memory soft threshold — warn when container reaches this % of its WATCHDOG_CONTAINERS hard limit +# Does not trigger restart — informational only + SOFT_MEM_THRESHOLD=80 + +# HTTP responsiveness — consecutive failed checks before restart +# CURL_TIMEOUT = seconds before curl gives up on a single check + RESP_FAIL_LIMIT=2 # consecutive failed checks before restart + CURL_TIMEOUT=5 # seconds per check before timeout + +# How often the watchdog runs its checks +# 900 = 15 minutes — long enough to not be noisy, short enough to catch issues quickly +# Containers have this long to recover before next check + DOCKER_WATCHDOG_INTERVAL=900 # seconds between watchdog cycles + +# Heartbeat — proof of life logged periodically even when everything is healthy +# Useful to confirm the watchdog is still running without flooding logs + DOCKER_WATCHDOG_HEARTBEAT=true # true = log heartbeat / false = completely silent + DOCKER_WATCHDOG_HEARTBEAT_HOURS=1 # how often to log heartbeat (hours) + +# Tier 2 master toggle — set false to disable global container scanning entirely +# When false only WATCHDOG_CONTAINERS and required containers are monitored + WATCHDOG_SCAN_ALL=true + +# Containers to skip in Tier 2 scan entirely +# Useful for containers that legitimately exit/restart frequently +WATCHDOG_SCAN_IGNORE=( + "DashGate" + "PIA-WG-Config-Generator" + "Aperture" + "Aperture-Kids" + "pgvector-18-Apeture-Kids" + "Pgvector18-Aperture" +) + +# Individual Tier 2 check toggles — disable specific checks without disabling Tier 2 + WATCHDOG_RESTART_UNHEALTHY=true # restart containers with Docker HEALTHCHECK = unhealthy + WATCHDOG_RESTART_DEAD=true # restart containers in dead state + WATCHDOG_RESTART_CRASHED=true # restart containers that exited with non-zero code + WATCHDOG_NOTIFY_OOM=true # notify + restart OOM killed containers + WATCHDOG_NOTIFY_CRASHLOOP=true # notify when Docker RestartCount keeps climbing + +# Crash loop threshold — notify critical if Docker has restarted this many times total +# Above this number the notification escalates to critical — manual intervention needed + WATCHDOG_CRASH_LIMIT=5 + +# Startup grace period — skip restarts while system is still booting after array start +# Prevents watchdog from restarting containers that are legitimately still initializing + WATCHDOG_STARTUP_GRACE=600 # seconds after boot before watchdog acts on failures + +# Restart loop protection — stops hammering a broken container +# If watchdog restarts a container more than LIMIT times in WINDOW hours → skip list +# Skip list auto-clears when container recovers healthy + WATCHDOG_CONTAINER_RESTART_LIMIT=3 # max watchdog restarts allowed in window + WATCHDOG_CONTAINER_RESTART_WINDOW=1 # rolling window in hours + WATCHDOG_CONTAINER_RESTART_LOG="$DATA_DIR/container_restart_history.db" + # rolling restart history for loop detection + +# Notification batching — one clean summary per cycle instead of one ping per event +# true = batch all events into one notification at end of cycle +# false = send one notification per event (noisy on busy systems) + WATCHDOG_BATCH_NOTIFY=true + +# Dependency ordering — skip restarting a container if its dependency is also down +# Prevents restarting Authelia before its database is ready +# Space-separated list of dependencies per container +declare -A WATCHDOG_DEPENDENCIES=( + ["Authelia"]="Mariadb-Authelia Redis-Authelia" + ["Authelia-Secondary"]="Mariadb-Authelia Redis-Authelia-Secondary" + ["NextCloud"]="Postgres-NextCloud" +) + +# ━━━ Docker Network Connect ━━━ +# Connects containers to extra Docker networks on array start via array_start.sh. +# Useful for containers that need their own custom network but also need to be +# reachable from your main custom bridge network. +# Every container in NETWORK_CONNECT_CONTAINERS is connected to every network in +# NETWORK_CONNECT_NETWORKS — containers not found are skipped gracefully. +NETWORK_CONNECT_CONTAINERS=( + "memcached" + "Npm-CrowdSec" +) + +NETWORK_CONNECT_NETWORKS=( + "high-availability" # must exist before array start — create in Docker settings +) + +# ============================================================================================== +# ── UNRAID ESSENTIALS ───────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +# ━━━ inotify Tuning ━━━ +# Linux inotify limits — applied at every array start by inotify_tuning.sh +# Default unRAID values are very low — with many Docker containers watching files +# (Sonarr, Radarr, Lidarr, NextCloud etc.) you can silently exhaust the limit. +# Symptoms: containers miss file events, downloads not detected, library not updated. +# These settings are lost on reboot — reapplied automatically at array start. + INOTIFY_MAX_INSTANCES=1024 # default: 128 — max inotify instances per user + INOTIFY_MAX_WATCHES=524288 # default: 8192 — max files watched per instance + INOTIFY_MAX_QUEUED_EVENTS=32768 # default: 16384 — max events queued before dropping + +# ━━━ System Tuning Monitor ━━━ +# Tracks inotify and php-fpm usage over time — read by sunday_morning_coffee_report.sh +# Snapshot written every 6 hours by system_tuning_monitor.sh +# Log bounded to TUNING_LOG_RETENTION days — auto-purges on each write + INOTIFY_WARN_PCT=80 # warn if inotify instances exceed this % of limit + PHP_FPM_WARN_PCT=80 # warn if php-fpm workers exceed this % of max_children + TUNING_MONITOR_LOG="$DATA_DIR/system_tuning_history.db" + TUNING_LOG_RETENTION=30 # days — enough for monthly trend visibility + +# ━━━ Reboot ━━━ +# Seconds of warning broadcast to logged-in users before server_reboot.sh reboots. +# Gives users time to save work — 300s = 5 minutes + REBOOT_SLEEP=300 + +# ━━━ Mover ━━━ +# Seconds to wait before mover_stop.sh sends SIGTERM to the mover process. +# Gives mover time to finish current file transfer before being interrupted. + MOVER_STOP_TIMEOUT=300 + +# ━━━ Syslog Filter ━━━ +# Path for the rsyslog filter file that suppresses Docker veth interface noise. +# Docker creates a new veth interface for each container — generates hundreds of +# log lines per hour that have no diagnostic value. Filter removes them at source. + FILTER_FILE="/etc/rsyslog.d/ignore-docker-veth.conf" + +# ━━━ PHP-FPM ━━━ +# Higher max_children allows more concurrent PHP requests to the unRAID WebGUI. +# Default is very low — increasing it prevents WebGUI slowdowns under load. +# 250 is safe for servers with 32GB+ RAM. + PHP_CONF="/etc/php-fpm.d/www.conf" + PHP_MAX_CHILDREN=250 + +# ━━━ Clear Logs ━━━ +# System log files cleared weekly to prevent rootfs fill over time. +# These grow continuously — without clearing they eventually consume all rootfs space. + LOG_FILES=(/var/log/syslog /var/log/messages /var/log/dmesg) + +# ━━━ WebGUI Watchdog ━━━ +# Monitors unRAID WebGUI responsiveness — escalates through nginx restart → emhttp restart. +# Separate from docker_watchdog — this monitors the unRAID UI itself, not containers. +# WEBGUI_NGINX_WAIT = seconds after nginx restart before rechecking +# WEBGUI_EMHTTP_WAIT = seconds after emhttp restart before rechecking + WEBGUI_URL="http://localhost" + WEBGUI_TIMEOUT=5 # seconds before curl gives up on WebGUI check + WEBGUI_NGINX_WAIT=15 # seconds after nginx restart before rechecking + WEBGUI_EMHTTP_WAIT=30 # seconds after emhttp restart before rechecking + +# ============================================================================================== +# ── MEDIA ───────────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +# ━━━ Media Permissions ━━━ +# Applied recursively to all shares in MEDIA_PERMISSION_SHARES by media_shares_permissions.sh. +# Runs first in MEDIA_MANAGEMENT_JOBS — arr cleanup scripts depend on correct ownership. +# 777 mode = read/write/execute for all users — standard for unRAID media shares +# nobody:users = standard unRAID media share ownership + PERMISSIONS_MODE="777" + PERMISSIONS_OWNER="nobody:users" + +MEDIA_PERMISSION_SHARES=( + /mnt/user/Anime_Movies + /mnt/user/Anime_Movies-Old + /mnt/user/Anime_Shows + /mnt/user/Anime_Shows-Old + /mnt/user/appcache + /mnt/user/Books + /mnt/user/Downloads + /mnt/user/Games + /mnt/user/Intros + /mnt/user/Kids_Movies + /mnt/user/Kids_Tv_Shows + /mnt/user/Movie_Recordings + /mnt/user/Movies + /mnt/user/Music + /mnt/user/Music_Videos + /mnt/user/Photo + /mnt/user/Sports + /mnt/user/stand-up_comedy + /mnt/user/Temp_Storage + /mnt/user/Tv_Recordings + /mnt/user/Tv_Shows + /mnt/user/YouTube +) + +# ━━━ Media Cleaner ━━━ +# Removes junk files from media shares — two profiles: anime and media. +# Called via MEDIA_MANAGEMENT_JOBS. Run manually: Media/media_cleaner.sh anime|media + +ANIME_CLEAN_FOLDERS=( + /mnt/user/Anime_Movies + /mnt/user/Anime_Movies-Old + /mnt/user/Anime_Shows + /mnt/user/Anime_Shows-Old +) + +MEDIA_CLEAN_FOLDERS=( + /mnt/user/Kids_Movies + /mnt/user/Kids_Tv_Shows + /mnt/user/Movies + /mnt/user/Music + /mnt/user/Sports + /mnt/user/stand-up_comedy + /mnt/user/Tv_Shows +) + +ANIME_FILE_PATTERNS=( + '*.sfv' '*.md5' '*.sha1' '*.txt' '*.url' '*.lnk' + '*.rar' '*.zip' '*.info' '*.torrent' '*.sample*' '*.proof*' + '*sync-conflict*' '*.scr' '*.srr' '*.exe' '*.webp' + '*.log' '*.json' +) + +MEDIA_FILE_PATTERNS=( + '*.sfv' '*.md5' '*.sha1' '*.txt' '*.url' '*.lnk' + '*.rar' '*.zip' '*.info' '*.torrent' '*.sample*' '*.proof*' + '*sync-conflict*' '*.scr' '*.srr' '*.exe' '*.webp' + '*.log' '*.json' '*.iso' '*.lrc' +) + +# ━━━ Arr Cleanup ━━━ +# Orphan file cleanup via Lidarr, Sonarr, and Radarr APIs. +# Compares tracked file paths from API against disk — deletes untracked files older than ORPHAN_AGE. +# detect_hosts() selects correct URL, API key, and root path at runtime. +# +# Protected patterns are NEVER deleted — cover art, metadata, subtitles generated by the arr +# are not included in the tracked file API response but must not be deleted. +# +# API versions and endpoint patterns: +# Sonarr v4 → /api/v3/series (get IDs) → /api/v3/episodefile?seriesId=X per series +# Radarr v5 → /api/v3/movie (get IDs) → /api/v3/moviefile?movieId=X per movie +# Lidarr v3 → /api/v1/artist (get IDs) → /api/v1/trackFile?artistId=X per artist +# All require per-ID loops — bulk endpoints removed in newer versions +# +# Version checking — scripts verify the arr major version matches before running +# If the arr updates and breaks the API the script exits safely before touching files +# Update the MAJOR version here when the script is updated to support a new version +# MINOR = 0 means any minor version within that major is accepted + + SONARR_VERSION_MAJOR=4 # tested major version — script exits if major differs + RADARR_VERSION_MAJOR=6 # tested major version — script exits if major differs + LIDARR_VERSION_MAJOR=3 # tested major version — script exits if major differs + +# ── Lidarr ──────────────────────────────────────────────────────────────────────────────────── +HOST1_LIDARR_URL="http://localhost:8686" +HOST1_LIDARR_API_KEY="your-lidarr-api-key" +HOST1_LIDARR_MUSIC_ROOT="/mnt/user/Music-New" +LIDARR_LOCK_WARN_AGE=3600 # 1hr — large libraries take time, not stuck + +# Container path → host path translation +# Lidarr stores file paths using container paths — script scans host paths +# Add one entry per root folder configured in Lidarr Settings → Media Management → Root Folders +declare -A HOST1_LIDARR_PATH_MAP=( + ["/ext-music"]="/mnt/user/Music-New" +) +declare -A HOST2_LIDARR_PATH_MAP=( + # HOST2 does not run Lidarr — fill in if that changes + # ["/ext-music"]="/mnt/user/Music-New" +) + +LIDARR_ORPHAN_AGE=7 # days — files must be older than this before eligible for deletion + # protects files that may still be mid-import or recently downloaded +LIDARR_EXTENSIONS=("flac" "mp3" "m4a" "wav" "aac" "ogg" "opus" "wma") +LIDARR_PROTECTED_PATTERNS=( + # Metadata + "*.nfo" "*.tbn" + # Images — album art, artist images, Emby artwork + "*.jpg" "*.jpeg" "*.png" "*.webp" "*.svg" + "poster.*" "fanart.*" "backdrop.*" "clearlogo.*" + "banner.*" "thumb.*" "landscape.*" + "folder.*" "cover.*" "album.*" "artist.*" "disc.*" + # Lyrics + "*.lrc" +) + # NEVER deleted — cover art, metadata, lyrics + # Lidarr generates these but doesn't include them in trackFile API + # Without this protection cleanup would delete all your artwork +LIDARR_MAX_DELETE_GB=1 # require --i-know-what-im-doing if deletion exceeds this +LIDARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run + # protects against API returning partial data on a bad day +LIDARR_TRACKED_COUNT_FILE="$DATA_DIR/lidarr_tracked.count" + # persists last known tracked count for percentage comparison + +# ── Sonarr ──────────────────────────────────────────────────────────────────────────────────── +HOST1_SONARR_URL="http://localhost:8989" +HOST1_SONARR_API_KEY="your-sonarr-api-key" +HOST1_SONARR_TV_ROOT="/mnt/user/Tv_Shows" + +# Container path → host path translation +# Add one entry per root folder configured in Sonarr Settings → Media Management → Root Folders +# Note: stand-up_comedy in both Sonarr + Radarr — TV specials and movie specials, one folder +declare -A HOST1_SONARR_PATH_MAP=( + ["/tv"]="/mnt/user/Tv_Shows" + ["/ext-standup-comedy"]="/mnt/user/stand-up_comedy" + ["/kids tv"]="/mnt/user/Kids_Tv_Shows" + ["/ext-anime-shows"]="/mnt/user/Anime_Shows-Old" +) + +HOST2_SONARR_URL="http://localhost:8989" +HOST2_SONARR_API_KEY="your-sonarr-api-key" +HOST2_SONARR_TV_ROOT="/mnt/user/Anime_Shows" + +declare -A HOST2_SONARR_PATH_MAP=( + # Fill in when HOST2 is back online + # ["/tv"]="/mnt/user/Anime_Shows" +) + +SONARR_ORPHAN_AGE=7 # days — files must be older than this before eligible for deletion +SONARR_MAX_DELETE_GB=1 # require --i-know-what-im-doing if deletion exceeds this +SONARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "ts" "wmv" "mov") +SONARR_PROTECTED_PATTERNS=( + # Subtitles + "*.srt" "*.sub" "*.ass" "*.ssa" "*.idx" "*.vtt" + # Metadata + "*.nfo" "*.tbn" + # Images — cover art, posters, fanart, Emby artwork + "*.jpg" "*.jpeg" "*.png" "*.webp" "*.svg" + "poster.*" "fanart.*" "backdrop.*" "clearlogo.*" + "banner.*" "thumb.*" "landscape.*" + # Kodi/Emby extras — not tracked by Sonarr API + "*-trailer.*" "*-featurette.*" "*-behindthescenes.*" + "*-interview.*" "*-scene.*" "*-short.*" "*-deleted.*" + "*-clip.*" "*-other.*" + # Theme songs — stored in show folder, not tracked + "theme.mp3" "theme.flac" "theme.wav" "theme.m4a" "theme.mka" +) + +# ── Radarr ──────────────────────────────────────────────────────────────────────────────────── +HOST1_RADARR_URL="http://localhost:7878" +HOST1_RADARR_API_KEY="your-radarr-api-key" +HOST1_RADARR_MOVIES_ROOT="/mnt/user/Movies" + +# Container path → host path translation +# Add one entry per root folder configured in Radarr Settings → Media Management → Root Folders +# Note: stand-up_comedy in both Radarr + Sonarr — movie specials and TV specials, one folder +declare -A HOST1_RADARR_PATH_MAP=( + ["/movies"]="/mnt/user/Movies" + ["/kids movies"]="/mnt/user/Kids_Movies" + ["/ext-stand-up-comedy"]="/mnt/user/stand-up_comedy" + ["/anime-movies"]="/mnt/user/Anime_Movies-Old" +) + +HOST2_RADARR_URL="http://localhost:7878" +HOST2_RADARR_API_KEY="your-radarr-api-key" +HOST2_RADARR_MOVIES_ROOT="/mnt/user/Anime_Movies" + +declare -A HOST2_RADARR_PATH_MAP=( + # Fill in when HOST2 is back online + # ["/anime-movies"]="/mnt/user/Anime_Movies" +) + +RADARR_ORPHAN_AGE=7 # days — files must be older than this before eligible for deletion +RADARR_MAX_DELETE_GB=1 # require --i-know-what-im-doing if deletion exceeds this +RADARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "wmv" "mov") +RADARR_PROTECTED_PATTERNS=( + # Subtitles + "*.srt" "*.sub" "*.ass" "*.ssa" "*.idx" "*.vtt" + # Metadata + "*.nfo" "*.tbn" + # Images — cover art, posters, fanart, Emby artwork + "*.jpg" "*.jpeg" "*.png" "*.webp" "*.svg" + "poster.*" "fanart.*" "backdrop.*" "clearlogo.*" + "banner.*" "thumb.*" "landscape.*" + # Kodi/Emby extras — not tracked by Radarr API + "*-trailer.*" "*-featurette.*" "*-behindthescenes.*" + "*-interview.*" "*-scene.*" "*-short.*" "*-deleted.*" + "*-clip.*" "*-other.*" + # Theme songs — stored in movie folder, not tracked + "theme.mp3" "theme.flac" "theme.wav" "theme.m4a" "theme.mka" +) + +# ━━━ Arr Failed/Stalled Recovery ━━━ +# Auto blocklist + re-search failed imports and stalled downloads. +# Runs every 6 hours — schedule: 0 */6 * * * +# +# Targets four problem types: +# importFailed — downloaded but arr couldn't import +# importPending — downloaded, stuck waiting to import (won't self-resolve) +# error status — serious failure not covered above +# stalled — download stuck with no connections or progress +# +# Items newer than ARR_IMPORT_RECOVERY_AGE are skipped — gives arr time to retry first. +# API versions: Sonarr /api/v3/ — Radarr /api/v3/ — Lidarr /api/v1/ +# Lidarr runs on HOST1 only — exits cleanly on HOST2. + +ARR_IMPORT_RECOVERY_AGE=6 # hours — skip items newer than this + # gives the arr time to retry on its own before we intervene + # matches cron interval — items are eligible after one missed cycle + +# Per-arr enable/disable toggles — set false to temporarily disable without removing from cron +# Useful if an arr is having issues and you want to skip it for a few runs +HOST1_SONARR_RECOVERY=true # Tv_Shows import recovery +HOST1_RADARR_RECOVERY=true # Movies import recovery +HOST1_LIDARR_RECOVERY=true # Music import recovery — HOST1 only, exits cleanly on HOST2 +HOST2_SONARR_RECOVERY=true # Anime_Shows import recovery +HOST2_RADARR_RECOVERY=true # Anime_Movies import recovery + +# ============================================================================================== +# ── TRANSCODES ──────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# Session-based storage allocator using filesystem symlink indirection. +# ffmpeg resolves the symlink ONCE at session start — existing sessions are never affected. +# +# How it works: +# ramdisk_setup.sh — creates tmpfs and symlink at array start via array_start.sh +# transcode_management.sh — every 3min, runs cleanup then manager in correct order +# transcode_cleanup.sh — removes old inactive files +# transcode_manager.sh — manages symlink direction based on usage thresholds +# +# ⚠️ Docker mount — must use shared propagation: +# --mount type=bind,source=/mnt/ram-transcode,target=/ext-ram-transcode,bind-propagation=shared +# Standard rprivate bind mounts lock the inode — sessions drift to SSD permanently. + +# ━━━ Transcode Manager ━━━ +# tmpfs mount point — created at array start by ramdisk_setup.sh +# Must exist before Emby starts so the symlink resolves correctly + RAMDISK_PATH="/mnt/ramdisk_transcodes" + +# Ramdisk size ceiling — tmpfs only uses RAM actually needed, not the full size upfront +# Set this to a comfortable limit based on your typical concurrent stream count +# Real-world: 9 streams peaked at ~5.5GB — 8G gives comfortable headroom + RAMDISK_SIZE="8G" + +# Symlink that Emby points at — this path NEVER changes regardless of ramdisk/SSD state +# Emby resolves the symlink once per session at start — symlink flips are transparent +# Must match the container path configured in Emby's Extra Parameters + TRANSCODE_LINK="/mnt/ram-transcode" + +# SSD fallback location — where transcodes land when ramdisk is too full +# Must have enough free space to handle peak session load + TRANSCODE_SSD="/mnt/cache/Temp_Storage/Emby/Transcodes/" + +# Usage thresholds in GB — hysteresis gap between WARN and LOW prevents flip-flop +# RAMDISK_WARN_GB: flip symlink to SSD when ramdisk usage reaches this +# RAMDISK_LOW_GB: flip symlink back to ramdisk when usage drops to this +# Gap (6.8 - 5.5 = 1.3GB) means ramdisk must drop 1.3GB before flipping back +# Without hysteresis a session right at the threshold causes rapid flipping + RAMDISK_WARN_GB=6.8 + RAMDISK_LOW_GB=5.5 + +# Minimum free GB on SSD before allowing a flip to SSD +# Prevents flipping to SSD when it's almost full — that would be worse than a full ramdisk + RAMDISK_SSD_MIN_GB=20 + +# File age thresholds in minutes before cleanup eligibility +# TRANSCODE_MAX_AGE: HLS segment files older than this with no active session = clean up +# TRANSCODE_ORPHAN_AGE: files with no matching session at all = clean up + TRANSCODE_MAX_AGE=20 + TRANSCODE_ORPHAN_AGE=30 + +# Notify if symlink flips this many times in one hour +# Frequent flips indicate the ramdisk is too small or thresholds need adjustment + TRANSCODE_FLIP_WARN=3 + +# Permissions applied to ramdisk and SSD transcode directories + TRANSCODE_OWNER="nobody:users" + TRANSCODE_CHMOD="755" + +# Operating mode — controls symlink direction behaviour +# smart — auto-flips between ramdisk and SSD based on RAMDISK_WARN_GB / RAMDISK_LOW_GB +# hysteresis gap prevents flip-flop — default for production +# ramdisk — always uses ramdisk, never flips to SSD +# warns if RAMDISK_WARN_GB exceeded but holds position +# use during SSD maintenance or when SSD space is low +# ssd — always uses SSD, never flips to ramdisk +# use during ramdisk maintenance or after a ramdisk issue + TRANSCODE_MANAGER_MODE="smart" # smart | ramdisk | ssd + +# Daily statistics log — read by weekly_health_digest.sh for transcode summary +# Tracks peak usage, flip count, session ratio, files cleaned per day +# Bounded to TRANSCODE_LOG_RETENTION days — auto-purges old entries on each write + TRANSCODE_DAILY_LOG="$DATA_DIR/transcode_daily.db" + TRANSCODE_LOG_RETENTION=90 # days before old entries are purged + +# ━━━ Transcode Server Array ━━━ +# All media servers sharing the ramdisk transcode space. +# Format: "ContainerName|URL|APIKey|Type" — Type: emby | jellyfin | plex +# Entries with placeholder API keys are skipped automatically. +# ⚠️ Tdarr does NOT belong here — keep Tdarr on SSD. + +TRANSCODE_SERVERS=( + "${HOST1_EMBY_CONTAINER}|${HOST1_EMBY_URL}|${HOST1_EMBY_API_KEY}|emby" + # "${HOST2_EMBY_CONTAINER}|${HOST2_EMBY_URL}|${HOST2_EMBY_API_KEY}|emby" + # "Jellyfin|http://localhost:8097|jellyfin-api-key|jellyfin" + # "Plex|http://localhost:32400|plex-token|plex" +) + + TRANSCODE_CHECK_EMBY=true + +# ============================================================================================== +# ── MONITORS ────────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +# ━━━ Certificate Monitor ━━━ +# Checks SSL certificate expiry via direct openssl connection — no NPM dependency. +# Checks the actual certificate served by each domain, not what NPM thinks it has. +# CERT_WARN_DAYS = notify this many days before expiry +# CERT_CRIT_DAYS = escalate to critical this many days before expiry +# CERT_TIMEOUT = seconds before giving up on the openssl connection +CERT_MONITOR_DOMAINS=( + "yourdomain.com" + "yourdomain.us" +) + CERT_WARN_DAYS=30 # warn when cert expires within this many days + CERT_CRIT_DAYS=7 # critical alert within this many days + CERT_TIMEOUT=10 # seconds per domain check + +# ━━━ Backup Verify ━━━ +# Verifies rsync mirror health by comparing random file checksums between servers. +# Catches silent corruption or incomplete syncs that rsync itself wouldn't detect. +# Leave BACKUP_VERIFY_SHARES empty to use HOST*_DAILY_SYNC_SHARES automatically. +# BACKUP_VERIFY_SAMPLE = number of random files to checksum per share +# BACKUP_VERIFY_MIN_SIZE = skip files smaller than this (small files are rarely corrupted) +BACKUP_VERIFY_SHARES=( + # leave empty to use HOST*_DAILY_SYNC_SHARES automatically +) + BACKUP_VERIFY_SAMPLE=10 # random files to check per share + BACKUP_VERIFY_MIN_SIZE=1M # minimum file size to include in sample + +# ━━━ SMART Health ━━━ +# Monitors drive SMART attributes — discovers all drives automatically via /dev/sd* and /dev/nvme*. +# Reads live SMART data — no persistent writes. +# SMART_IGNORE_DRIVES = drives to skip (boot USB, drives without meaningful SMART data) + SMART_TEMP_WARN=45 # Celsius — warn above this temperature + SMART_TEMP_CRIT=55 # Celsius — critical above this temperature +SMART_IGNORE_DRIVES=( + "sda" # boot USB — SMART not meaningful on flash drives +) + +# ━━━ ZFS Memory Snapshot ━━━ +# Weekly ZFS pool health and memory diagnostic report — informational only, no action taken. +# ZFS_REPORT_ARC_WARN_PCT = warn if ARC is using more than this % of its max +# ZFS_REPORT_FREE_WARN_GB = warn if less than this GB free RAM +# ZFS_REPORT_AVAIL_WARN_GB = warn if less than this GB available on ZFS pool +# ZFS_REPORT_DOCKER_TOP = how many top Docker containers to show by memory usage +# ZFS_REPORT_IGNORE_POOLS = individual disk pools to skip (unRAID array disks as ZFS) + ZFS_REPORT_LOG="/var/log/zfs-weekly-health.log" + ZFS_REPORT_ARC_WARN_PCT=90 + ZFS_REPORT_FREE_WARN_GB=10 + ZFS_REPORT_AVAIL_WARN_GB=20 + ZFS_REPORT_DOCKER_TOP=10 +ZFS_REPORT_IGNORE_POOLS=( + "disk10" + "disk9" + "disk8" + "disk6" + "disk5" +) + +# ━━━ Bandwidth Monitor ━━━ +# Called automatically by rsync.sh after each sync — one bounded write per run. +# Tracks transfer size, duration and profile per sync for weekly summary reporting. +# BANDWIDTH_LOG_RETENTION = days to keep entries before auto-purging old records +# BANDWIDTH_WARN_GB = flag in weekly summary if a single sync exceeded this size + BANDWIDTH_LOG="$DATA_DIR/bandwidth_history.db" + BANDWIDTH_LOG_RETENTION=90 # days before old entries are purged + BANDWIDTH_WARN_GB=50 # flag syncs larger than this in weekly report + +# Stats files — written by cleanup and recovery scripts, read by coffee report +# All in DATA_DIR — array always running when these are written + ARR_CLEANUP_STATS="$DATA_DIR/arr_cleanup_stats.db" # lidarr/sonarr/radarr orphan stats + ARR_RECOVERY_STATS="$DATA_DIR/arr_recovery_stats.db" # blocklist + re-search stats + +# ━━━ Health Digest ━━━ +# Aggregated system health summary — reads existing state files, no new writes. +# Three profiles control when the digest email is sent: +# always — sends every run regardless of findings +# smart — sends only when DIGEST_SMART_ON_* conditions are found +# weekly — sends once per week on DIGEST_DAY only +# Smart profile triggers — set true to send digest when finding is detected: + DIGEST_PROFILE="weekly" # always | smart | weekly + DIGEST_DAY="Sunday" # day of week for weekly profile + DIGEST_SMART_ON_WATCHDOG=true # send if any watchdog strikes are active + DIGEST_SMART_ON_FAILOVER=true # send if failover state is not NORMAL + DIGEST_SMART_ON_CERT_WARN=true # send if any cert is under CERT_WARN_DAYS + DIGEST_SMART_ON_BANDWIDTH=true # send if any transfer exceeded BANDWIDTH_WARN_GB + +# ━━━ Emby Session Report ━━━ +# Weekly Emby usage statistics via API — no persistent writes, queries fresh each run. +# Shows top content, most active users, session counts over the report period. +# URL and API key pulled from HOST1/HOST2_EMBY_URL and HOST1/HOST2_EMBY_API_KEY +# defined in Host Configuration at the top of this file — no duplication needed. + EMBY_REPORT_DAYS=7 # days to include in the report period + EMBY_REPORT_TOP_N=10 # number of top content items to show + +# ============================================================================================== +# ── SYSTEM WATCHDOG ─────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# Continuous system health monitoring — last line of defense before a crash. +# Started by array_start.sh — runs until array stops. +# Re-sources Master.conf each cycle — config changes take effect on next cycle. +# Strike system: sustained threshold hits trigger reboot — single spikes ignored. +# Reboot loop protection: shuts down instead if reboot limit hit in rolling window. +# Silent when healthy — logs only when a threshold is triggered. + +# ━━━ State Files ━━━ + SYS_WATCHDOG_STATE_FILE="/tmp/system_watchdog_state.db" # /tmp resets on reboot ✅ + SYS_WATCHDOG_FAILED_FILE="/boot/config/system_watchdog_failed.db" # survives reboots + SYS_WATCHDOG_REBOOT_LOG="/boot/config/system_watchdog_reboots.db" # reboot loop detection + +# ━━━ Strike and Reboot Loop Settings ━━━ +# Strike system: a check must fail this many consecutive cycles before action is taken +# Single spikes (one bad reading) are ignored — sustained problems trigger reboot + SYS_WATCHDOG_STRIKE_LIMIT=2 # consecutive failures before reboot trigger + +# How often checks run — 300s = 5 minutes +# At STRIKE_LIMIT=2 and INTERVAL=300: problem must persist 10min before reboot + SYSTEM_WATCHDOG_INTERVAL=300 # seconds between watchdog cycles + +# Reboot loop protection — if system keeps rebooting something is seriously wrong +# After REBOOT_LIMIT reboots in REBOOT_WINDOW_HRS hours → shutdown instead of reboot +# Prevents infinite reboot loops when the underlying problem can't be fixed by rebooting + SYS_WATCHDOG_REBOOT_LIMIT=3 # max reboots before shutdown instead + SYS_WATCHDOG_REBOOT_WINDOW_HRS=12 # rolling window in hours + +# Heartbeat — proof of life logged periodically even when everything is healthy + SYSTEM_WATCHDOG_HEARTBEAT=true # true = log heartbeat / false = completely silent + SYSTEM_WATCHDOG_HEARTBEAT_HOURS=1 # how often to log heartbeat (hours) + +# ━━━ Thresholds ━━━ +# Set at "about to become unstable" levels — not "things are a bit high" +# These should be high enough that normal operation never triggers them + +# rootfs (/) usage percentage — when array is down rsync writes land on rootfs +# fills rapidly and can crash the server — 95% is almost too late, act fast + SYS_WATCHDOG_ROOTFS_PCT=95 + +# /var/log usage percentage — log spam can fill rootfs, indicates something broken + SYS_WATCHDOG_LOG_PCT=95 + +# Free RAM in GB — below this is critically low, OOM or swap imminent +# Your server has 128GB — 4GB free means something is consuming everything + SYS_WATCHDOG_MEM_GB=4 + +# ZFS ARC pinned percentage — ARC not releasing after reclaim = memory stuck +# SYS_WATCHDOG_ARC_RELEASE_PCT = after reclaim attempt, if still above this → trigger + SYS_WATCHDOG_ARC_PINNED_PCT=98 + SYS_WATCHDOG_ARC_RELEASE_PCT=95 + +# Load average multiplier — threshold = MULTIPLIER × CPU core count +# MULTIPLIER=3 on 16-core = load average of 48 before triggering +# Set high — transcoding causes legitimate high load spikes + SYS_WATCHDOG_LOAD_MULTIPLIER=3 + +# Zombie process count — large numbers indicate serious process management failure +# A few zombies are normal — 50 means something is very wrong + SYS_WATCHDOG_ZOMBIE_LIMIT=50 + +# CPU temperature in Celsius — sustained high temp causes throttling or kernel panic +# 95°C is close to tjmax on most CPUs — triggers before thermal shutdown + SYS_WATCHDOG_CPU_TEMP_MAX=95 + +# ━━━ Check Toggles ━━━ +# Disable individual checks without disabling the whole watchdog +# All enabled by default except load — transcoding causes legitimate load spikes + SYS_WATCHDOG_CHECK_ROOTFS=true + SYS_WATCHDOG_CHECK_LOG=true + SYS_WATCHDOG_CHECK_RAM=true + SYS_WATCHDOG_CHECK_ARC=true + SYS_WATCHDOG_CHECK_CPU_TEMP=true + SYS_WATCHDOG_CHECK_LOAD=false # disabled — load spikes during transcoding are normal + SYS_WATCHDOG_CHECK_ZOMBIES=true + SYS_WATCHDOG_CHECK_CONTAINERS=true # checks docker_watchdog persistent skip list + SYS_WATCHDOG_CHECK_DOCKER_DAEMON=true # checks if Docker daemon is responding + +# ━━━ Abort Toggles ━━━ +# Conditions that prevent reboot even when a threshold is hit +# true = abort reboot if this condition is active (conservative — avoid data loss) +# false = reboot anyway (aggressive — a clean reboot beats a hard crash) +# Philosophy: aborting is safer for data, rebooting is safer for stability + SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY=true # rebooting with bad pool risks data loss + SYS_WATCHDOG_ABORT_ON_PARITY=false # aborting parity is better than crashing mid-check + SYS_WATCHDOG_ABORT_ON_MOVER=false # aborting move is better than crashing mid-move + +# ============================================================================================== +# ──────────────────────── End Of User Variables ─────────────────────────────────────────────── +# ============================================================================================== \ No newline at end of file diff --git a/user_script_plug-in.sh b/user_script_plug-in.sh index e9ff7e2..37e39a4 100644 --- a/user_script_plug-in.sh +++ b/user_script_plug-in.sh @@ -221,9 +221,7 @@ #/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-Failover/Critical-Data #/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-Failover/Important-Data #/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Media_Server/Emby --profile=emby-failover -# ^^ schedule every 30-60min — dirty sync, Emby running, critical data only #/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Media_Server/Emby -# ^^ do NOT schedule — called by weekly_sync_maintenance.sh Sunday 2:30am only #/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-Failover/Gmer4Lfe # # ━━━ Rsync — Individual Media Shares (ad hoc) ━━━