From a638035a2f461038930b53de8e482189237359cc Mon Sep 17 00:00:00 2001 From: FailedProxy Date: Sun, 26 Apr 2026 04:29:09 -0400 Subject: [PATCH] updated docker network logic added a custome network check --- Docker_Essentials/docker_network_connect.sh | 1468 +++++++++++++++++-- Orchestrators/array_start.sh | 2 +- 2 files changed, 1338 insertions(+), 132 deletions(-) diff --git a/Docker_Essentials/docker_network_connect.sh b/Docker_Essentials/docker_network_connect.sh index 90fbe81..aa37860 100644 --- a/Docker_Essentials/docker_network_connect.sh +++ b/Docker_Essentials/docker_network_connect.sh @@ -1,159 +1,1365 @@ #!/bin/bash -# ----------------------------------------------------------------------------------------------- -# --------------------------------- Docker Network Connect ------------------------------------- -# ----------------------------------------------------------------------------------------------- -# Connects specified Docker containers to extra networks on array start. -# Useful when containers need to communicate across networks they were not -# originally configured with — e.g. memcached needing access to nextcloud-aio network. +# ============================================================================================== +# ================================= MASTER CONFIGURATION ======================================= +# ============================================================================================== +# All user-facing variables for the unRAID script ecosystem. +# Scripts source this file — edit here, changes apply everywhere on next git pull. # -# Every container in NETWORK_CONNECT_CONTAINERS is connected to every network -# in NETWORK_CONNECT_NETWORKS. Already-connected containers are skipped cleanly. +# ── HOW THIS FILE WORKS ─────────────────────────────────────────────────────────────────────── +# Every script sources Master.conf and common.sh at startup. +# Change a value here and it affects all scripts that use it — no hunting through files. +# To disable something: comment it out with # rather than deleting it. +# To add a new rsync profile: add a key to each PROFILE_* array. +# To add or remove orchestrator jobs: edit the arrays in the ORCHESTRATORS section. # -# Run once at array start via User Scripts plugin. -# All configuration in Master.conf under Docker Network Connect section. -# Supports --dry-run to preview connections without making them. -# ----------------------------------------------------------------------------------------------- +# ── INDEX ───────────────────────────────────────────────────────────────────────────────────── +# +# Section Description +# ─────────────────────────────────────────────────────────────────────────────────────────── +# HOST CONFIGURATION Server hostnames, SSH keys, Emby connection details, DATA_DIR +# LOGGING Enable or disable verbose logging +# NOTIFICATIONS unRAID native and Discord webhook settings +# GIT / REPO Gitea repository and SSH settings +# +# ── ORCHESTRATORS ────────────────────────────────────────────────────────────────────────── +# ARRAY START Scripts launched at array start (array_start.sh) +# DAILY SYNC MAINTENANCE Job list + media shares (daily_sync_maintenance.sh) +# WEEKLY SYNC MAINTENANCE Job list + sync jobs + sync settings (weekly_sync_maintenance.sh) +# MEDIA MANAGEMENT Job list for media_management.sh +# +# ── RSYNC ────────────────────────────────────────────────────────────────────────────────── +# RSYNC DEFAULTS Global fallback rsync options and limits +# REMOTE HEALTH CHECKS Rootfs threshold for pre-flight abort +# RSYNC PROFILE SYSTEM Per-profile overrides for appdata syncs +# +# ── FAILOVER ─────────────────────────────────────────────────────────────────────────────── +# FAILOVER Mutual container failover between two servers +# FAILOVER TEST Simulated outage settings for failover_test.sh +# DDNS Script-controlled DDNS — absolute rules +# INTERNET LOSS Containers to stop when internet is lost +# TIERED CONTAINER LISTS What each server runs for the other per tier +# TIER DELAY SETTINGS How long before each tier activates (minutes) +# RSYNC WRITEBACK JOBS Appdata synced back to primary on handback +# +# ── DOCKER ESSENTIALS ────────────────────────────────────────────────────────────────────── +# DOCKER DAILY RESTART Containers restarted daily +# DOCKER WEEKLY RESTART Containers restarted weekly +# DOCKER WATCHDOG Continuous two-tier self-healing container monitoring +# DOCKER NETWORK CONNECT Connect containers to extra networks on array start +# +# ── UNRAID ESSENTIALS ────────────────────────────────────────────────────────────────────── +# REBOOT User warning delay before scheduled reboot +# MOVER Mover stop timeout +# SYSLOG FILTER Docker veth noise filter file path +# PHP-FPM PHP-FPM max children config +# CLEAR LOGS System log file paths +# WEBGUI WATCHDOG WebGUI nginx + emhttp monitoring and restart +# +# ── MEDIA ────────────────────────────────────────────────────────────────────────────────── +# MEDIA PERMISSIONS Share list, mode and owner for permissions script +# MEDIA CLEANER Anime and media folder lists and file patterns +# ARR CLEANUP Lidarr, Sonarr, Radarr orphan file cleanup +# ARR FAILED/STALLED RECOVERY Auto blocklist + re-search failed imports and stalled downloads +# +# ── TRANSCODES ───────────────────────────────────────────────────────────────────────────── +# TRANSCODE MANAGER Ramdisk and SSD fallback transcode management +# TRANSCODE SERVER ARRAY Multi-server session monitoring (Emby, Jellyfin, Plex) +# +# ── MONITORS ─────────────────────────────────────────────────────────────────────────────── +# CERTIFICATE MONITOR SSL certificate expiry monitoring +# BACKUP VERIFY Random sample checksum verification against remote +# SMART HEALTH Drive SMART attribute monitoring +# ZFS MEMORY SNAPSHOT Weekly ZFS health and memory diagnostic report +# BANDWIDTH MONITOR Daily rsync transfer logging and weekly summary +# HEALTH DIGEST Aggregated system health digest — always/smart/weekly +# EMBY SESSION REPORT Weekly Emby usage statistics via API +# +# ── SYSTEM WATCHDOG ──────────────────────────────────────────────────────────────────────── +# SYSTEM WATCHDOG Continuous system health monitoring — last line of defense +# +# ============================================================================================== -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# ============================================================================================== +# ── HOST CONFIGURATION ──────────────────────────────────────────────────────────────────────── +# ============================================================================================== -source "$SCRIPT_DIR/../Master.conf" -source "$SCRIPT_DIR/../common.sh" +# ━━━ Hosts ━━━ +# Hostnames must match exact Docker/unRAID hostnames — case sensitive. +# Used by detect_hosts() in common.sh to determine which server is local and which is remote. +# Both servers run identical scripts — host detection makes them bidirectional. + HOST1="unRAID-Gmer4Lfe" + HOST2="unRAID-Jayred365" -parse_args "$@" +# Data directory — persistent script state and statistics files. +# Array share — survives reboots, no flash drive wear. +# Created automatically if it doesn't exist. +# Only truly critical files (failover state, watchdog reboot log) stay on /boot/config. + DATA_DIR="/mnt/user/appdata/unraid_scripts/data" -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_GEAR Setup ━━━ -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_GEAR Setup ━━━" +# SSH keys for server-to-server rsync and failover container operations. +# Both keys must be in /root/.ssh/ and authorised in the remote server's authorized_keys. + HOST1_SSH_KEY="/root/.ssh/Gmer4Lfe-rsync-key" + HOST2_SSH_KEY="/root/.ssh/Jayred365-rsync-key" -if [[ "$EUID" -ne 0 ]]; then - error "Must be run as root" - exit 1 -fi +# ━━━ Emby ━━━ +# Defined once here — referenced by transcode_manager.sh, emby_session_report.sh, +# emby_database_repair.sh, weekly_sync_maintenance.sh, and TRANSCODE_SERVERS array. +# API key: Emby Dashboard → API Keys → + New Key + HOST1_EMBY_CONTAINER="Emby" + HOST1_EMBY_URL="http://localhost:8096" + HOST1_EMBY_API_KEY="0c27448d93a7431f9ac63569f7655829" -success "Running as root" + HOST2_EMBY_CONTAINER="Emby-Jayred365" + HOST2_EMBY_URL="http://localhost:8096" # same port — different server, different key + HOST2_EMBY_API_KEY="your-host2-emby-api-key" -if ! command -v docker &>/dev/null; then - error "Docker not found" - exit 1 -fi +# ============================================================================================== +# ── LOGGING ─────────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== -success "Docker found" +# Controls verbose [LOG] output across all scripts. +# true = show detailed [LOG] lines — useful for debugging or first-time setup +# false = show only user-facing output — cleaner for scheduled runs + ENABLE_LOGGING=true -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Status ━━━ -# ----------------------------------------------------------------------------------------------- -if [[ "$SHOW_STATUS" == true ]]; then - echo "" - echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" - echo "$ICON_CONTAINERS Containers: ${NETWORK_CONNECT_CONTAINERS[*]}" - echo "$ICON_DOCKER_NET Networks: ${NETWORK_CONNECT_NETWORKS[*]}" - echo "$ICON_GEAR Dry Run: $DRY_RUN" - echo "" - echo "━━━ Current Connections ━━━" - for container in "${NETWORK_CONNECT_CONTAINERS[@]}"; do - [[ -z "$container" ]] && continue - echo "$ICON_CONTAINERS $container:" - docker inspect "$container" \ - --format '{{range $k, $v := .NetworkSettings.Networks}} {{$k}}{{"\n"}}{{end}}' \ - 2>/dev/null || echo " not found" - done - echo "━━━━━━━━━━━━━━━━━━━━━━━" - exit 0 -fi +# ============================================================================================== +# ── NOTIFICATIONS ───────────────────────────────────────────────────────────────────────────── +# ============================================================================================== -[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no network connections will be made" +# unRAID native notification system — integrates with the bell icon in the WebGUI. +# normal = job completed successfully / warning = something failed or needs attention + NOTIFY_UNRAID=true -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_DOCKER_NET Network Connect ━━━ -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_DOCKER_NET Network Connect — $(date '+%Y-%m-%d %H:%M:%S') ━━━" -echo "$ICON_CONTAINERS Containers: ${NETWORK_CONNECT_CONTAINERS[*]}" -echo "$ICON_DOCKER_NET Networks: ${NETWORK_CONNECT_NETWORKS[*]}" -echo "" +# Discord webhook URL — leave blank to disable + DISCORD_WEBHOOK="" -START=$(date +%s) -CONNECTED=() -SKIPPED=() -FAILED=() +# ============================================================================================== +# ── GIT / REPO ──────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== -for container in "${NETWORK_CONNECT_CONTAINERS[@]}"; do - [[ -z "$container" ]] && continue +# Gitea self-hosted repository — used by git_pull_execute.sh. +# Detects Gitea container location at runtime — works through failover automatically. +# Falls back to GITEA_DOMAIN if local and Tailscale both fail. + GITEA_CONTAINER="Gitea" # exact Docker container name + GITEA_REPO_PATH="FailedProxy/Unraid_Scripts.git" # repo path on Gitea server + GITEA_DOMAIN="" # e.g. git.gmer4lfe.com — requires NPM + DNS setup + TARGET_DIR="/mnt/user/appdata/unraid_scripts" # where scripts are cloned to + GITEA_SSH_KEY="/root/.ssh/unraid_gitea" # SSH key for authenticating to Gitea + SSH_PORT=221 # Gitea SSH port (default 22, Gitea often uses 222/221) - echo "━━━ $ICON_CONTAINERS $container ━━━" +# ============================================================================================== +# ── ORCHESTRATORS ────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# All orchestrator job lists live here — edit arrays to add/remove scripts. +# No changes to orchestrator scripts needed when adding or removing jobs. - # Verify container exists - if ! docker inspect "$container" &>/dev/null; then - warn "$container not found — skipping" - continue - echo "" - continue - fi +# ━━━ Array Start ━━━ +# Scripts launched by array_start.sh when the array comes online. +# Launched in order — each as a background process. +# One-shot scripts (ramdisk, syslog, fpm, network) run and exit naturally. +# Continuous scripts (watchdogs, failover) run until array stops. - for network in "${NETWORK_CONNECT_NETWORKS[@]}"; do - [[ -z "$network" ]] && continue +ARRAY_START_SCRIPTS=( + "Transcodes/ramdisk_setup.sh" # creates ramdisk + symlink before Emby starts + "unRAID_Essentials/docker_syslog_filter.sh" # suppress veth noise before logs fill + "unRAID_Essentials/php_fpm_max_children.sh" # WebGUI performance tuning + "Docker_Essentials/docker_network_connect.sh" # ensure networks exist + connect containers + "unRAID_Essentials/system_watchdog.sh" # system health monitor — continuous loop + "Docker_Essentials/docker_watchdog.sh" # container health monitor — continuous loop +# "Failover/failover.sh" # mutual failover — continuous loop +) - # Verify network exists - if ! docker network inspect "$network" &>/dev/null; then - warn "Network $network not found — skipping" - continue - continue - fi +# ━━━ Daily Sync Maintenance ━━━ +# daily_sync_maintenance.sh runs the media share sync built into the script first, +# then iterates DAILY_MAINTENANCE_SCRIPTS for additional jobs. +# Schedule: 0 1 * * * (1am daily) - # Check if already connected - if docker network inspect "$network" \ - --format '{{range .Containers}}{{.Name}} {{end}}' 2>/dev/null \ - | grep -qw "$container"; then - info "$ICON_DOCKER_NET $container already connected to $network — skipping" - SKIPPED+=("$container→$network") - continue - fi +DAILY_MAINTENANCE_SCRIPTS=( + "git_pull_execute.sh" # pull latest scripts — always runs first + "Docker_Essentials/docker_daily_restart.sh" # daily container restarts +) - # Connect - if [[ "$DRY_RUN" == true ]]; then - warn "DRY RUN — would connect $container to $network" - continue - fi +# Media shares synced daily by daily_sync_maintenance.sh. +# Each server syncs only the shares it owns (source of truth) — direction is automatic. +# HOST1 pushes its truth shares to HOST2. HOST2 pushes its truth shares to HOST1. +# Never both pushing the same share — one server is always the truth holder. +# These shares use DEFAULT_RSYNC_OPTS — no profile entry needed. +# For shares needing custom options or container stops — create a profile in the RSYNC section. - info "$ICON_DOCKER_NET Connecting $container to $network..." - if docker network connect "$network" "$container" 2>/dev/null; then - success "$ICON_DOCKER_NET $container connected to $network" - CONNECTED+=("$container→$network") - else - error "Failed to connect $container to $network" - FAILED+=("$container:$network") - fi - done +HOST1_DAILY_SYNC_SHARES=( + /mnt/user/Books + /mnt/user/Intros + /mnt/user/Kids_Movies + /mnt/user/Kids_Tv_Shows + /mnt/user/Movies + /mnt/user/Music + /mnt/user/Music_Videos + /mnt/user/Nextcloud + /mnt/user/stand-up_comedy + /mnt/user/Sports + /mnt/user/Tv_Shows + /mnt/user/Anime_Shows-Old + /mnt/user/Anime_Movies-Old +) - echo "" -done +HOST2_DAILY_SYNC_SHARES=( + /mnt/user/Anime_Movies + /mnt/user/Anime_Shows +) -END=$(date +%s) +# Personal encrypted shares — synced for offsite backup, independent of media shares. +# ZFS encrypted at dataset level — remote receives encrypted blocks, cannot read content. +# See README-Rsync_Setup.md for ZFS encryption setup before uncommenting. +HOST1_PERSONAL_SHARES=( + # /mnt/user/Gmer4Lfe-Personal # uncomment after creating encrypted dataset +) -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Summary ━━━ -# ----------------------------------------------------------------------------------------------- -echo "━━━━━ $ICON_SUMMARY NETWORK CONNECT SUMMARY ━━━━━" -echo "$ICON_TIME Duration: $(format_duration $((END - START)))" -[[ ${#CONNECTED[@]} -gt 0 ]] && echo "$ICON_DOCKER_NET Connected: ${CONNECTED[*]}" -[[ ${#SKIPPED[@]} -gt 0 ]] && echo "$ICON_RUNNING Skipped: ${SKIPPED[*]}" -[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}" -echo "" +HOST2_PERSONAL_SHARES=( + # /mnt/user/Jayred365-Personal # uncomment after creating encrypted dataset +) -if [[ "$DRY_RUN" == true ]]; then - echo "$ICON_WARN Status: DRY RUN — no changes made" -elif [[ ${#FAILED[@]} -gt 0 ]]; then - echo "$ICON_ERROR Status: $ICON_ERROR SOME CONNECTIONS FAILED" - notify "Docker network connect failed on $(hostname) — ${FAILED[*]}" "Network Connect" "warning" -else - echo "$ICON_DONE Status: $ICON_SUCCESS DONE" - if [[ ${#CONNECTED[@]} -gt 0 ]]; then - notify "Docker networks connected on $(hostname) — ${CONNECTED[*]}" "Network Connect" "normal" - fi -fi -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +# ━━━ Weekly Sync Maintenance ━━━ +# weekly_sync_maintenance.sh handles the critical sync built into the script first: +# stop containers both sides → pull updates → sync Emby + Critical-Data → restart +# Then iterates WEEKLY_MAINTENANCE_SCRIPTS for additional jobs. +# Schedule: 30 2 * * 0 (Sunday 2:30am) -[[ ${#FAILED[@]} -gt 0 ]] && exit 1 -exit 0 \ No newline at end of file +WEEKLY_MAINTENANCE_SCRIPTS=( + "Docker_Essentials/docker_weekly_restart.sh" # weekly container restarts after sync +) + +# Shares synced by weekly_sync_maintenance.sh during the maintenance window. +# Containers are stopped both sides before these sync — full clean state guaranteed. +# Profiles drive container stops, excludes, and options — configure in RSYNC section. +# Order matters — Emby first, then auth stack. +WEEKLY_SYNC_JOBS=( + "/mnt/user/Media_Server/Emby" # emby profile — full clean mirror + "/mnt/user/appdata-Failover/Critical-Data" # critical-data profile — auth stack +) + +# Container update toggles for the weekly sync window. +# Containers are already stopped for the sync — updates pull at no extra downtime. +# Both false → sync only, no updates. +# Toggle false temporarily to skip updates without changing the schedule. + CRITICAL_SYNC_UPDATES=true # pull container updates locally + CRITICAL_SYNC_UPDATES_REMOTE=true # pull container updates on remote via SSH + +# ━━━ Media Management ━━━ +# Job list run directly by daily_sync_maintenance.sh after the media share sync. +# Runs sequentially — permissions first, then cleaners, then arr cleanup. +# Comment out any job to disable without removing it. +# Each individual script can still be run manually for one-off maintenance. + +MEDIA_MANAGEMENT_JOBS=( + "Media/media_shares_permissions.sh" # apply permissions — runs first + "Media/media_cleaner.sh anime" # remove junk from anime shares + "Media/media_cleaner.sh media" # remove junk from media shares +# "Media/lidarr_cleanup.sh" # remove orphaned music files +# "Media/sonarr_cleanup.sh" # remove orphaned TV files +# "Media/radarr_cleanup.sh" # remove orphaned movie files +) + +# ============================================================================================== +# ── RSYNC ───────────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +# ━━━ Rsync Defaults ━━━ +# Global fallback values used when no profile match is found. +# Media shares in HOST*_DAILY_SYNC_SHARES always use these globals — no profile needed. +# Appdata shares match profiles by directory basename (lowercased). +# If a profile key exists it overrides the global. If missing the global is used. + + BW_LIMIT=12500 # KB/s — 12500 ≈ 100Mbit — network transfer speed cap + RETRY_COUNT=3 # retry attempts if rsync fails before giving up + SLEEP=300 # seconds between retry attempts + CRITICAL_CONTAINER_NAMES=() # containers to stop on REMOTE before rsync — profiles override + DELAYED_CONTAINERS=() # containers needing delay before starting — profiles override + CONTAINER_DELAY=5 # seconds to wait before starting delayed containers + EXCLUDE_DIRS=() # directories to exclude from transfer — profiles override + +# --delete removes files on remote that no longer exist on source (mirror behaviour) +# --inplace writes directly to destination — better for large files, avoids temp copies +# --no-whole-file forces delta transfer even on fast connections — sends only changed blocks + DEFAULT_RSYNC_OPTS=(-av --info=progress2 --human-readable --bwlimit="$BW_LIMIT" --delete --inplace --no-whole-file) + +# ━━━ Remote Health Checks ━━━ +# Pre-flight check — aborts if remote rootfs (/) usage is at or above this percentage. +# When remote array is down, rsync writes land on rootfs — fills fast and crashes the server. + ROOTFS_WARN=75 + +# ━━━ Rsync Profile System ━━━ +# Profiles allow per-share rsync behaviour without touching script logic. +# Profile key matched by basename of directory passed to rsync.sh (lowercased). +# Override with --profile=name flag. +# +# IMPORTANT: PROFILE_RSYNC_OPTS does NOT inherit DEFAULT_RSYNC_OPTS. +# List ALL desired options explicitly when defining a profile. +# +# Current profiles: +# arrs_stack — arr databases — lower bandwidth, containers stopped for consistency +# critical-data — auth stack — containers stopped both sides, Authelia delayed start +# gmer4lfe — server-specific appdata — no container stops needed +# important-data — NextCloud + Postgres — NextCloud delayed start after Postgres +# emby — weekly clean sync — both Emby stopped, full mirror, minimal excludes +# called by weekly_sync_maintenance.sh only — do NOT schedule separately +# emby-failover — frequent dirty sync — Emby stays running, WAL excluded, critical data only +# also used for failover writeback on handback + +declare -A PROFILE_RSYNC_OPTS=( + [arrs_stack]="-av --info=progress2 --human-readable --bwlimit=$BW_LIMIT --delete --inplace" + [critical-data]="-av --human-readable --bwlimit=$BW_LIMIT --delete" + [gmer4lfe]="-av --info=progress2 --bwlimit=$BW_LIMIT" + [important-data]="-av --human-readable --bwlimit=$BW_LIMIT" + [emby]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file" + [emby-failover]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file" +) + +# Per-profile bandwidth limits in KB/s — overrides global BW_LIMIT for that profile only +# Lower for shares running alongside other jobs, higher for time-sensitive critical data +declare -A PROFILE_BW_LIMIT=( + [arrs_stack]=5000 # lower — runs alongside other syncs, avoids saturating link + [critical-data]=9500 # high — small dataset, get it synced fast and clean + [gmer4lfe]=8000 + [important-data]=9500 # high — database sync needs to be fast + [emby]=8000 # medium — large full mirror, steady transfer + [emby-failover]=9500 # high — small critical dataset, sync as fast as possible +) + +# Retry attempts per profile — how many times to retry before giving up on a failed sync +declare -A PROFILE_RETRY_COUNT=( + [arrs_stack]=3 + [critical-data]=3 + [gmer4lfe]=3 + [important-data]=3 + [emby]=3 + [emby-failover]=3 +) + +# Seconds to wait between retry attempts +# emby-failover shorter — frequent sync, faster retry on transient failures +declare -A PROFILE_SLEEP=( + [arrs_stack]=300 + [critical-data]=300 + [gmer4lfe]=300 + [important-data]=300 + [emby]=300 + [emby-failover]=120 # shorter — frequent dirty sync, retry faster +) + +# Containers stopped on BOTH LOCAL and REMOTE servers before rsync. +# Local stops first — flushes databases cleanly before pushing data out. +# Remote stops next — prevents writes to destination while receiving. +# Only containers that were running get restarted — stopped containers stay stopped. +# Same container names on both servers — consistent naming is required by this ecosystem. +# If a container is not found on a server it is skipped gracefully, not errored. +# SPACE-SEPARATED STRINGS — converted to array at runtime +declare -A PROFILE_CRITICAL_CONTAINER_NAMES=( + [arrs_stack]="Sonarr Lidarr Readarr Radarr Prowlarr Bazarr Pinchflat" + [critical-data]="Mariadb-Authelia Mariadb-Authelia-Secondary Redis-Authelia Redis-Authelia-Secondary Lldap-Gmer4Lfe NginxProxyManager Authelia Authelia-Secondary" + [gmer4lfe]="Organizrv2-Gmer4Lfe UptimeKuma-Gmer4Lfe VaultWarden-Gmer4Lfe" + [important-data]="Postgres-NextCloud NextCloud" + [emby]="Emby" # weekly clean sync — both Emby instances stopped, WAL checkpointed + [emby-failover]="" # dirty sync — Emby stays running both sides, WAL excluded from sync +) + +# Containers that need a delay before starting after rsync completes. +# Database containers must be accepting connections before dependent apps start. +# Authelia waits for Mariadb + Redis. NextCloud waits for Postgres. +# SPACE-SEPARATED STRINGS — converted to array at runtime +declare -A PROFILE_DELAYED_CONTAINERS=( + [arrs_stack]="" + [critical-data]="Authelia Authelia-Secondary" # wait for Mariadb + Redis to be ready + [gmer4lfe]="" + [important-data]="NextCloud" # wait for Postgres to accept connections + [emby]="" + [emby-failover]="" +) + +# Seconds to wait before starting delayed containers +# 15s gives Mariadb, Redis, and LLDAP time to accept connections before Authelia starts +declare -A PROFILE_CONTAINER_DELAY=( + [arrs_stack]=5 + [critical-data]=15 # Mariadb + Redis need time to accept connections + [gmer4lfe]=5 + [important-data]=10 # Postgres needs time before NextCloud + [emby]=5 + [emby-failover]=5 +) + +# Directories excluded from rsync transfer per profile +# emby-failover excludes WAL files — safe to sync while Emby is running +# emby clean sync only excludes logs, transcodes, cache — full metadata mirror +# SPACE-SEPARATED STRINGS — converted to array at runtime +declare -A PROFILE_EXCLUDE_DIRS=( + [arrs_stack]="logs *.tmp" + [gmer4lfe]="logs *.tmp" + [important-data]="logs *.tmp" + [critical-data]="logs *.tmp *.log nginx/temp nginx/cache __pycache__ notification.txt" + [emby]="logs transcodes cache crash*" + # emby-failover: Emby running, WAL excluded — only safe critical data synced + # users.db, library.db, authentication.db, config/ — everything else excluded + [emby-failover]="logs transcodes cache metadata *.db-wal *.db-shm crash* plugins root" +) + +# Skip per-disk space check for these profiles — appdata syncs go to cache/appdata +# not to array disks, so disk space check is irrelevant and just slows things down +declare -A PROFILE_SKIP_DISK_CHECK=( + [arrs_stack]=true + [critical-data]=true + [gmer4lfe]=true + [important-data]=true + [emby]=true + [emby-failover]=true +) + +# ============================================================================================== +# ── FAILOVER ────────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# Mutual container failover between two unRAID servers. +# Each server runs Failover/failover.sh independently via array_start.sh. +# All decisions based on two pings: remote reachable + internet reachable. +# +# States: NORMAL | FAILOVER | NO_INTERNET | DARK +# +# DDNS rules — absolute: +# Internet loss → stop own DDNS immediately +# Failover → start remote DDNS first (Tier 1) +# Handback → stop remote DDNS → rsync → start containers → start local DDNS last +# +# Tiers: +# Tier 1 — Immediate — vital services + Live TV +# Tier 2 — configurable delay — productivity services +# Tier 3 — configurable delay — secondary services +# Tier 4 — configurable delay — arrs + downloaders + + EXTERNAL_IP="8.8.8.8" + FAILOVER_CHECK_INTERVAL=120 + FAILOVER_HANDBACK_STRIKES=2 + FAILOVER_STATE_FILE="/boot/config/failover_state.db" + +# ━━━ Failover Test ━━━ + FAILOVER_TEST_BLOCK_WAIT=150 + FAILOVER_TEST_HANDBACK_WAIT=360 + +# ━━━ DDNS ━━━ +HOST1_DDNS_CONTAINERS=( + "Gmer4Lfe.com" +) + +HOST2_DDNS_CONTAINERS=( + "Gmer4Lfe.us" +) + +# ━━━ Internet Loss ━━━ +FAILOVER_HOST1_STOP_ON_NO_NET=( + "Gmer4Lfe.com" +) + +FAILOVER_HOST2_STOP_ON_NO_NET=( + "Gmer4Lfe.us" +) + +# ━━━ Tiered Container Lists ━━━ + +# HOST1 runs for HOST2 +FAILOVER_HOST1_RUNS_FOR_HOST2_IMMEDIATE=( + "Gmer4Lfe.us" + "VaultWarden-Jayred365" + # "container-placeholder" +) + +FAILOVER_HOST1_RUNS_FOR_HOST2_2HR=( + # "container-placeholder" +) + +FAILOVER_HOST1_RUNS_FOR_HOST2_6HR=( + # "container-placeholder" +) + +FAILOVER_HOST1_RUNS_FOR_HOST2_18HR=( + # "container-placeholder" +) + +# HOST2 runs for HOST1 +FAILOVER_HOST2_RUNS_FOR_HOST1_IMMEDIATE=( + "Gmer4Lfe.com" + "Emby" + "VaultWarden-Gmer4Lfe" + "Dispatcharr" + "Dispatcharr-Basic" + "Dispatcharr-Iptv-Users" + "ErsatzTV-Emby" +) + +FAILOVER_HOST2_RUNS_FOR_HOST1_2HR=( + "Postgres-NextCloud" + "NextCloud" + "PostgreSQL_Immich" + "Immich-Gmer4Lfe" + # "container-placeholder" +) + +FAILOVER_HOST2_RUNS_FOR_HOST1_6HR=( + "Gitea" + # "container-placeholder" +) + +FAILOVER_HOST2_RUNS_FOR_HOST1_18HR=( + "Sonarr" + "Radarr" + "Lidarr" + "Readarr" + "Prowlarr" + "Bazarr" + "SABnzbd-Gmer4Lfe" + "Qbittorrent-Gmer4Lfe" + "LidaTube" + "Pinchflat" + "ChannelTube" + # "container-placeholder" +) + +# ━━━ Tier Delay Settings ━━━ +# How long the primary server must be down before each tier activates — in minutes. +# Tier 1 is always immediate — Live TV and media can't wait. +# Set independently per host — adjust based on hardware and what's worth starting. +# Longer delays = less resource usage on covering server but slower recovery. +# +# HOST1's containers running on HOST2 (HOST1 is down): +HOST1_TIER2_DELAY=240 # 4 hours — NextCloud, Immich — can wait +HOST1_TIER3_DELAY=720 # 12 hours — secondary services — Gitea etc. +HOST1_TIER4_DELAY=1440 # 24 hours — full workflow — arrs and downloaders + +# HOST2's containers running on HOST1 (HOST2 is down): +HOST2_TIER2_DELAY=240 +HOST2_TIER3_DELAY=720 +HOST2_TIER4_DELAY=1440 + +# ━━━ Rsync Writeback Jobs ━━━ +# Syncs critical appdata BACK to primary server during handback after failover. +# Containers are stopped before writeback runs — clean source, no competing writes. +# Purpose: primary comes back online with the state that built up during its outage +# (watch states, auth changes, library updates that happened on HOST2) +# +# HOST*_TIER1_WRITEBACK_DELAY: +# Short outages skip Tier 1 writeback — primary state is more reliable than dirty sync data +# Only writeback if outage lasted longer than this many minutes +# 60 minutes = if HOST1 was down less than 1hr, don't bother writing back Emby +# +# Tier 4 writeback automatically syncs HOST*_DAILY_SYNC_SHARES back — no need to list those here +# Only add paths that are NOT in DAILY_SYNC_SHARES and need writeback after extended outage + +HOST1_TIER1_WRITEBACK_DELAY=60 # minutes — skip Emby writeback if outage under 1hr +HOST2_TIER1_WRITEBACK_DELAY=60 + +# HOST1 writeback — run by HOST2 during HOST1 handback +FAILOVER_HOST1_WRITEBACK_TIER1=( + "/mnt/user/Media_Server/Emby" # watch states, playstates built up during outage +) + +FAILOVER_HOST1_WRITEBACK_TIER2=( + "/mnt/user/appdata-Failover/Important-Data" # NextCloud + Postgres — files added during outage +) + +FAILOVER_HOST1_WRITEBACK_TIER3=( + # "location-placeholder" +) + +FAILOVER_HOST1_WRITEBACK_TIER4=( + # Edge cases outside HOST1_DAILY_SYNC_SHARES + "/mnt/user/appdata-Failover/Arrs_Stack" # arr databases — downloads queued during outage +) + +# HOST2 writeback — run by HOST1 during HOST2 handback +FAILOVER_HOST2_WRITEBACK_TIER1=( + # "/mnt/user/appdata-Failover/Jayred365-Emby" +) + +FAILOVER_HOST2_WRITEBACK_TIER2=( + # "/mnt/user/appdata-Failover/Jayred365-Important" +) + +FAILOVER_HOST2_WRITEBACK_TIER3=( + # "location-placeholder" +) + +FAILOVER_HOST2_WRITEBACK_TIER4=( + # Edge cases outside HOST2_DAILY_SYNC_SHARES + "/mnt/user/appdata-Failover/Arrs_Stack" +) + +# ============================================================================================== +# ── DOCKER ESSENTIALS ───────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +# ━━━ Docker Daily Restart ━━━ +# Containers restarted every day by docker_daily_restart.sh via daily_sync_maintenance.sh. +# These containers run better with a daily restart — not just "keeping things fresh". +# Dispatcharr specifically degrades over time without restart — daily is intentional. +# Schedule is set in daily_sync_maintenance.sh — runs at 1am as part of daily window. +# Case-sensitive — must match exact Docker container names. +DAILY_RESTART_CONTAINERS=( + "NginxProxyManager" + "Authelia" + "Dispatcharr-Iptv-Users" + "Dispatcharr" # Live TV scheduler — degrades without daily restart + "Dispatcharr-Basic" + "ErsatzTV-Emby" +) + +# ━━━ Docker Weekly Restart ━━━ +# Less critical services restarted weekly by docker_weekly_restart.sh. +# Called by weekly_sync_maintenance.sh Sunday 2:30am — containers already stopped +# for the weekly sync window so restart adds zero extra downtime. +# Weekly restarts also catch any pending image updates not applied during weekly sync. +WEEKLY_RESTART_CONTAINERS=( + "NextCloud" + "Organizrv2-Gmer4Lfe" + "AdGuard-Home" + "Immich-Gmer4Lfe" +) + +# ━━━ Docker Watchdog ━━━ +# Continuous two-tier self-healing container monitoring. +# Started by array_start.sh — runs until array stops. +# Re-sources Master.conf each cycle — add/remove containers without restarting watchdog. +# Silent when all healthy — only logs when something needs attention. +# Heartbeat fires periodically as proof of life even when everything is healthy. +# +# Tier 1 — strict monitoring of explicitly configured containers: +# Memory hard limits — immediate restart if container exceeds limit +# CPU thresholds — strike system, restart after CPU_FAIL_LIMIT sustained strikes +# HTTP responsiveness — strike system, restart after RESP_FAIL_LIMIT failed checks +# Required containers — must always be running, strike + skip list with auto-clear +# +# Tier 2 — global health scan of ALL running containers: +# Unhealthy status — Docker HEALTHCHECK unhealthy → restart +# OOM killed — kernel killed container → restart + notify +# Crash loop detection — RestartCount climbing → notify, critical above limit +# Dead containers — remove and restart +# Unexpected exits — non-zero exit code → restart + +# Memory hard limits in MB — immediate restart if exceeded +# Container restarted the moment it crosses this line — no strike system +# 20GB=20480 16GB=16384 12GB=12288 10GB=10240 +# 8GB=8192 6GB=6144 4GB=4096 2GB=2048 1GB=1024 +declare -A WATCHDOG_CONTAINERS=( + ["Emby"]=16384 + ["LidaTube"]=6144 + ["Tdarr"]=6144 + ["Code-Server"]=1024 +) + +# HTTP health check URLs — checked every cycle, strike system before restart +# Container must respond with HTTP 200 within CURL_TIMEOUT seconds +# Per-host — HOST1 and HOST2 may run different containers on different ports +declare -A HOST1_WATCHDOG_CONTAINER_URLS=( + ["Emby"]="http://localhost:8096" +) + +declare -A HOST2_WATCHDOG_CONTAINER_URLS=( + ["Emby"]="http://localhost:8096" +) + +# Required containers — must always be running +# Strike system: SYS_WATCHDOG_STRIKE_LIMIT strikes before restart attempt +# Persistent skip list: added after WATCHDOG_CONTAINER_RESTART_LIMIT restarts in window +# Skip list auto-clears when container recovers — no manual intervention needed +# Per-host — each server has different critical containers +HOST1_WATCHDOG_REQUIRED_CONTAINERS=( + "NginxProxyManager" + "Lldap-Gmer4Lfe" + "Authelia" + "Mariadb-Authelia" + "Redis-Authelia" + "Authelia-Secondary" + "Redis-Authelia-Secondary" +) + +HOST2_WATCHDOG_REQUIRED_CONTAINERS=( + "NginxProxyManager" + # add HOST2 required containers here +) + +# Strike state file — /tmp resets on reboot which is correct +# Fresh start after reboot means no stale strikes carrying over + WATCHDOG_STATE_FILE="/tmp/container_watchdog_state.db" + +# CPU thresholds — normalised against total core count automatically at runtime +# SOFT = warn only, HARD = strike toward restart +# CPU_FAIL_LIMIT = consecutive HARD strikes before restart + SOFT_CPU_THRESHOLD=80 # warn at this % of total system CPU + HARD_CPU_THRESHOLD=85 # strike at this % of total system CPU + CPU_FAIL_LIMIT=2 # consecutive hard CPU strikes before container restart + +# Memory soft threshold — warn when container reaches this % of its WATCHDOG_CONTAINERS hard limit +# Does not trigger restart — informational only + SOFT_MEM_THRESHOLD=80 + +# HTTP responsiveness — consecutive failed checks before restart +# CURL_TIMEOUT = seconds before curl gives up on a single check + RESP_FAIL_LIMIT=2 # consecutive failed checks before restart + CURL_TIMEOUT=5 # seconds per check before timeout + +# How often the watchdog runs its checks +# 900 = 15 minutes — long enough to not be noisy, short enough to catch issues quickly +# Containers have this long to recover before next check + DOCKER_WATCHDOG_INTERVAL=900 # seconds between watchdog cycles + +# Heartbeat — proof of life logged periodically even when everything is healthy +# Useful to confirm the watchdog is still running without flooding logs + DOCKER_WATCHDOG_HEARTBEAT=true # true = log heartbeat / false = completely silent + DOCKER_WATCHDOG_HEARTBEAT_HOURS=1 # how often to log heartbeat (hours) + +# Tier 2 master toggle — set false to disable global container scanning entirely +# When false only WATCHDOG_CONTAINERS and required containers are monitored + WATCHDOG_SCAN_ALL=true + +# Containers to skip in Tier 2 scan entirely +# Useful for containers that legitimately exit/restart frequently +WATCHDOG_SCAN_IGNORE=( + "DashGate" + "PIA-WG-Config-Generator" + "Aperture" + "Aperture-Kids" + "pgvector-18-Apeture-Kids" + "Pgvector18-Aperture" +) + +# Individual Tier 2 check toggles — disable specific checks without disabling Tier 2 + WATCHDOG_RESTART_UNHEALTHY=true # restart containers with Docker HEALTHCHECK = unhealthy + WATCHDOG_RESTART_DEAD=true # restart containers in dead state + WATCHDOG_RESTART_CRASHED=true # restart containers that exited with non-zero code + WATCHDOG_NOTIFY_OOM=true # notify + restart OOM killed containers + WATCHDOG_NOTIFY_CRASHLOOP=true # notify when Docker RestartCount keeps climbing + +# Crash loop threshold — notify critical if Docker has restarted this many times total +# Above this number the notification escalates to critical — manual intervention needed + WATCHDOG_CRASH_LIMIT=5 + +# Startup grace period — skip restarts while system is still booting after array start +# Prevents watchdog from restarting containers that are legitimately still initializing + WATCHDOG_STARTUP_GRACE=600 # seconds after boot before watchdog acts on failures + +# Restart loop protection — stops hammering a broken container +# If watchdog restarts a container more than LIMIT times in WINDOW hours → skip list +# Skip list auto-clears when container recovers healthy + WATCHDOG_CONTAINER_RESTART_LIMIT=3 # max watchdog restarts allowed in window + WATCHDOG_CONTAINER_RESTART_WINDOW=1 # rolling window in hours + WATCHDOG_CONTAINER_RESTART_LOG="$DATA_DIR/container_restart_history.db" + # rolling restart history for loop detection + +# Notification batching — one clean summary per cycle instead of one ping per event +# true = batch all events into one notification at end of cycle +# false = send one notification per event (noisy on busy systems) + WATCHDOG_BATCH_NOTIFY=true + +# Dependency ordering — skip restarting a container if its dependency is also down +# Prevents restarting Authelia before its database is ready +# Space-separated list of dependencies per container +declare -A WATCHDOG_DEPENDENCIES=( + ["Authelia"]="Mariadb-Authelia Redis-Authelia" + ["Authelia-Secondary"]="Mariadb-Authelia Redis-Authelia-Secondary" + ["NextCloud"]="Postgres-NextCloud" +) + +# ━━━ Docker Network Connect ━━━ +# Ensures custom Docker networks exist then connects specified containers to them. +# Two operations in one pass — no separate ensure script needed. +# +# Step 1 — network ensure: +# Checks each network in NETWORK_CONNECT_NETWORKS +# Missing → creates it (bridge driver, Docker assigns subnet automatically) +# Exists → skips creation +# unRAID updates occasionally wipe custom networks — this self-heals on next array start +# +# Step 2 — container connect: +# Connects each container in NETWORK_CONNECT_CONTAINERS to each network +# Container not found → warn and skip (not an error — may not be running on this host) +# Already connected → skip cleanly +# +# Use case: +# high-availability is the main custom bridge — most containers run here +# Containers with their own networks (NextCloud AIO, etc.) can be listed in +# NETWORK_CONNECT_CONTAINERS to gain access to high-availability +# without touching their primary network — they simply join both +NETWORK_CONNECT_CONTAINERS=( +# "memcached" +# "Npm-CrowdSec" +) + +NETWORK_CONNECT_NETWORKS=( + "high-availability" # must exist before array start — create in Docker settings +) + +# ============================================================================================== +# ── UNRAID ESSENTIALS ───────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +# ━━━ Reboot ━━━ +# Seconds of warning broadcast to logged-in users before server_reboot.sh reboots. +# Gives users time to save work — 300s = 5 minutes + REBOOT_SLEEP=300 + +# ━━━ Mover ━━━ +# Seconds to wait before mover_stop.sh sends SIGTERM to the mover process. +# Gives mover time to finish current file transfer before being interrupted. + MOVER_STOP_TIMEOUT=300 + +# ━━━ Syslog Filter ━━━ +# Path for the rsyslog filter file that suppresses Docker veth interface noise. +# Docker creates a new veth interface for each container — generates hundreds of +# log lines per hour that have no diagnostic value. Filter removes them at source. + FILTER_FILE="/etc/rsyslog.d/ignore-docker-veth.conf" + +# ━━━ PHP-FPM ━━━ +# Higher max_children allows more concurrent PHP requests to the unRAID WebGUI. +# Default is very low — increasing it prevents WebGUI slowdowns under load. +# 250 is safe for servers with 32GB+ RAM. + PHP_CONF="/etc/php-fpm.d/www.conf" + PHP_MAX_CHILDREN=250 + +# ━━━ Clear Logs ━━━ +# System log files cleared weekly to prevent rootfs fill over time. +# These grow continuously — without clearing they eventually consume all rootfs space. + LOG_FILES=(/var/log/syslog /var/log/messages /var/log/dmesg) + +# ━━━ WebGUI Watchdog ━━━ +# Monitors unRAID WebGUI responsiveness — escalates through nginx restart → emhttp restart. +# Separate from docker_watchdog — this monitors the unRAID UI itself, not containers. +# WEBGUI_NGINX_WAIT = seconds after nginx restart before rechecking +# WEBGUI_EMHTTP_WAIT = seconds after emhttp restart before rechecking + WEBGUI_URL="http://localhost" + WEBGUI_TIMEOUT=5 # seconds before curl gives up on WebGUI check + WEBGUI_NGINX_WAIT=15 # seconds after nginx restart before rechecking + WEBGUI_EMHTTP_WAIT=30 # seconds after emhttp restart before rechecking + +# ============================================================================================== +# ── MEDIA ───────────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +# ━━━ Media Permissions ━━━ +# Applied recursively to all shares in MEDIA_PERMISSION_SHARES by media_shares_permissions.sh. +# Runs first in MEDIA_MANAGEMENT_JOBS — arr cleanup scripts depend on correct ownership. +# 777 mode = read/write/execute for all users — standard for unRAID media shares +# nobody:users = standard unRAID media share ownership + PERMISSIONS_MODE="777" + PERMISSIONS_OWNER="nobody:users" + +MEDIA_PERMISSION_SHARES=( + /mnt/user/Anime_Movies + /mnt/user/Anime_Movies-Old + /mnt/user/Anime_Shows + /mnt/user/Anime_Shows-Old + /mnt/user/appcache + /mnt/user/Books + /mnt/user/Downloads + /mnt/user/Games + /mnt/user/Intros + /mnt/user/Kids_Movies + /mnt/user/Kids_Tv_Shows + /mnt/user/Movie_Recordings + /mnt/user/Movies + /mnt/user/Music + /mnt/user/Music_Videos + /mnt/user/Photo + /mnt/user/Sports + /mnt/user/stand-up_comedy + /mnt/user/Temp_Storage + /mnt/user/Tv_Recordings + /mnt/user/Tv_Shows + /mnt/user/YouTube +) + +# ━━━ Media Cleaner ━━━ +# Removes junk files from media shares — two profiles: anime and media. +# Called via MEDIA_MANAGEMENT_JOBS. Run manually: Media/media_cleaner.sh anime|media + +ANIME_CLEAN_FOLDERS=( + /mnt/user/Anime_Movies + /mnt/user/Anime_Movies-Old + /mnt/user/Anime_Shows + /mnt/user/Anime_Shows-Old +) + +MEDIA_CLEAN_FOLDERS=( + /mnt/user/Kids_Movies + /mnt/user/Kids_Tv_Shows + /mnt/user/Movies + /mnt/user/Music + /mnt/user/Sports + /mnt/user/stand-up_comedy + /mnt/user/Tv_Shows +) + +ANIME_FILE_PATTERNS=( + '*.sfv' '*.md5' '*.sha1' '*.txt' '*.url' '*.lnk' + '*.rar' '*.zip' '*.info' '*.torrent' '*.sample*' '*.proof*' + '*sync-conflict*' '*.scr' '*.srr' '*.exe' '*.webp' + '*.log' '*.json' +) + +MEDIA_FILE_PATTERNS=( + '*.sfv' '*.md5' '*.sha1' '*.txt' '*.url' '*.lnk' + '*.rar' '*.zip' '*.info' '*.torrent' '*.sample*' '*.proof*' + '*sync-conflict*' '*.scr' '*.srr' '*.exe' '*.webp' + '*.log' '*.json' '*.iso' '*.lrc' +) + +# ━━━ Arr Cleanup ━━━ +# Orphan file cleanup via Lidarr, Sonarr, and Radarr APIs. +# Compares tracked file paths from API against disk — deletes untracked files older than ORPHAN_AGE. +# detect_hosts() selects correct URL, API key, and root path at runtime. +# +# Protected patterns are NEVER deleted — cover art, metadata, subtitles generated by the arr +# are not included in the tracked file API response but must not be deleted. +# +# API versions and endpoint patterns: +# Sonarr v4 → /api/v3/series (get IDs) → /api/v3/episodefile?seriesId=X per series +# Radarr v5 → /api/v3/movie (get IDs) → /api/v3/moviefile?movieId=X per movie +# Lidarr v3 → /api/v1/artist (get IDs) → /api/v1/trackFile?artistId=X per artist +# All require per-ID loops — bulk endpoints removed in newer versions +# +# Version checking — scripts verify the arr major version matches before running +# If the arr updates and breaks the API the script exits safely before touching files +# Update the MAJOR version here when the script is updated to support a new version +# MINOR = 0 means any minor version within that major is accepted + + SONARR_VERSION_MAJOR=4 # tested major version — script exits if major differs + RADARR_VERSION_MAJOR=6 # tested major version — script exits if major differs + LIDARR_VERSION_MAJOR=3 # tested major version — script exits if major differs + +# ── Lidarr ──────────────────────────────────────────────────────────────────────────────────── +HOST1_LIDARR_URL="http://192.168.50.2:8686" +HOST1_LIDARR_API_KEY="b2977e71ef074bc0a0529d9fcce3b2dc" +HOST1_LIDARR_MUSIC_ROOT="/mnt/user/Music-New" +LIDARR_LOCK_WARN_AGE=3600 # 1hr — large libraries take time, not stuck + +# Container path → host path translation +# Lidarr stores file paths using container paths — script scans host paths +# Add one entry per root folder configured in Lidarr Settings → Media Management → Root Folders +declare -A HOST1_LIDARR_PATH_MAP=( + ["/ext-music"]="/mnt/user/Music-New" +) +declare -A HOST2_LIDARR_PATH_MAP=( + # HOST2 does not run Lidarr — fill in if that changes + # ["/ext-music"]="/mnt/user/Music-New" +) + +LIDARR_ORPHAN_AGE=7 # days — files must be older than this before eligible for deletion + # protects files that may still be mid-import or recently downloaded +LIDARR_EXTENSIONS=("flac" "mp3" "m4a" "wav" "aac" "ogg" "opus" "wma") +LIDARR_PROTECTED_PATTERNS=( + # Metadata + "*.nfo" "*.tbn" + # Images — album art, artist images, Emby artwork + "*.jpg" "*.jpeg" "*.png" "*.webp" "*.svg" + "poster.*" "fanart.*" "backdrop.*" "clearlogo.*" + "banner.*" "thumb.*" "landscape.*" + "folder.*" "cover.*" "album.*" "artist.*" "disc.*" + # Lyrics + "*.lrc" +) + # NEVER deleted — cover art, metadata, lyrics + # Lidarr generates these but doesn't include them in trackFile API + # Without this protection cleanup would delete all your artwork +LIDARR_MAX_DELETE_GB=1 # require --i-know-what-im-doing if deletion exceeds this +LIDARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run + # protects against API returning partial data on a bad day +LIDARR_TRACKED_COUNT_FILE="$DATA_DIR/lidarr_tracked.count" + # persists last known tracked count for percentage comparison + +# ── Sonarr ──────────────────────────────────────────────────────────────────────────────────── +HOST1_SONARR_URL="http://192.168.50.2:8989" +HOST1_SONARR_API_KEY="130decd3db5b4c25afad64864cd03f9f" +HOST1_SONARR_TV_ROOT="/mnt/user/Tv_Shows" + +# Container path → host path translation +# Add one entry per root folder configured in Sonarr Settings → Media Management → Root Folders +# Note: stand-up_comedy in both Sonarr + Radarr — TV specials and movie specials, one folder +declare -A HOST1_SONARR_PATH_MAP=( + ["/tv"]="/mnt/user/Tv_Shows" + ["/ext-standup-comedy"]="/mnt/user/stand-up_comedy" + ["/kids tv"]="/mnt/user/Kids_Tv_Shows" + ["/ext-anime-shows"]="/mnt/user/Anime_Shows-Old" +) + +HOST2_SONARR_URL="http://localhost:8989" +HOST2_SONARR_API_KEY="your-host2-sonarr-api-key" +HOST2_SONARR_TV_ROOT="/mnt/user/Anime_Shows" + +declare -A HOST2_SONARR_PATH_MAP=( + # Fill in when HOST2 is back online + # ["/tv"]="/mnt/user/Anime_Shows" +) + +SONARR_ORPHAN_AGE=7 # days — files must be older than this before eligible for deletion +SONARR_MAX_DELETE_GB=1 # require --i-know-what-im-doing if deletion exceeds this +SONARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "ts" "wmv" "mov") +SONARR_PROTECTED_PATTERNS=( + # Subtitles + "*.srt" "*.sub" "*.ass" "*.ssa" "*.idx" "*.vtt" + # Metadata + "*.nfo" "*.tbn" + # Images — cover art, posters, fanart, Emby artwork + "*.jpg" "*.jpeg" "*.png" "*.webp" "*.svg" + "poster.*" "fanart.*" "backdrop.*" "clearlogo.*" + "banner.*" "thumb.*" "landscape.*" + # Kodi/Emby extras — not tracked by Sonarr API + "*-trailer.*" "*-featurette.*" "*-behindthescenes.*" + "*-interview.*" "*-scene.*" "*-short.*" "*-deleted.*" + "*-clip.*" "*-other.*" + # Theme songs — stored in show folder, not tracked + "theme.mp3" "theme.flac" "theme.wav" "theme.m4a" "theme.mka" +) + +# ── Radarr ──────────────────────────────────────────────────────────────────────────────────── +HOST1_RADARR_URL="http://192.168.50.2:7878" +HOST1_RADARR_API_KEY="d43a3ec6cf1549edb4af0cc63f98b2a9" +HOST1_RADARR_MOVIES_ROOT="/mnt/user/Movies" + +# Container path → host path translation +# Add one entry per root folder configured in Radarr Settings → Media Management → Root Folders +# Note: stand-up_comedy in both Radarr + Sonarr — movie specials and TV specials, one folder +declare -A HOST1_RADARR_PATH_MAP=( + ["/movies"]="/mnt/user/Movies" + ["/kids movies"]="/mnt/user/Kids_Movies" + ["/ext-stand-up-comedy"]="/mnt/user/stand-up_comedy" + ["/anime-movies"]="/mnt/user/Anime_Movies-Old" +) + +HOST2_RADARR_URL="http://localhost:7878" +HOST2_RADARR_API_KEY="your-host2-radarr-api-key" +HOST2_RADARR_MOVIES_ROOT="/mnt/user/Anime_Movies" + +declare -A HOST2_RADARR_PATH_MAP=( + # Fill in when HOST2 is back online + # ["/anime-movies"]="/mnt/user/Anime_Movies" +) + +RADARR_ORPHAN_AGE=7 # days — files must be older than this before eligible for deletion +RADARR_MAX_DELETE_GB=1 # require --i-know-what-im-doing if deletion exceeds this +RADARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "wmv" "mov") +RADARR_PROTECTED_PATTERNS=( + # Subtitles + "*.srt" "*.sub" "*.ass" "*.ssa" "*.idx" "*.vtt" + # Metadata + "*.nfo" "*.tbn" + # Images — cover art, posters, fanart, Emby artwork + "*.jpg" "*.jpeg" "*.png" "*.webp" "*.svg" + "poster.*" "fanart.*" "backdrop.*" "clearlogo.*" + "banner.*" "thumb.*" "landscape.*" + # Kodi/Emby extras — not tracked by Radarr API + "*-trailer.*" "*-featurette.*" "*-behindthescenes.*" + "*-interview.*" "*-scene.*" "*-short.*" "*-deleted.*" + "*-clip.*" "*-other.*" + # Theme songs — stored in movie folder, not tracked + "theme.mp3" "theme.flac" "theme.wav" "theme.m4a" "theme.mka" +) + +# ━━━ Arr Failed/Stalled Recovery ━━━ +# Auto blocklist + re-search failed imports and stalled downloads. +# Runs every 6 hours — schedule: 0 */6 * * * +# +# Targets four problem types: +# importFailed — downloaded but arr couldn't import +# importPending — downloaded, stuck waiting to import (won't self-resolve) +# error status — serious failure not covered above +# stalled — download stuck with no connections or progress +# +# Items newer than ARR_IMPORT_RECOVERY_AGE are skipped — gives arr time to retry first. +# API versions: Sonarr /api/v3/ — Radarr /api/v3/ — Lidarr /api/v1/ +# Lidarr runs on HOST1 only — exits cleanly on HOST2. + +ARR_IMPORT_RECOVERY_AGE=6 # hours — skip items newer than this + # gives the arr time to retry on its own before we intervene + # matches cron interval — items are eligible after one missed cycle + +# Per-arr enable/disable toggles — set false to temporarily disable without removing from cron +# Useful if an arr is having issues and you want to skip it for a few runs +HOST1_SONARR_RECOVERY=true # Tv_Shows import recovery +HOST1_RADARR_RECOVERY=true # Movies import recovery +HOST1_LIDARR_RECOVERY=true # Music import recovery — HOST1 only, exits cleanly on HOST2 +HOST2_SONARR_RECOVERY=true # Anime_Shows import recovery +HOST2_RADARR_RECOVERY=true # Anime_Movies import recovery + +# ============================================================================================== +# ── TRANSCODES ──────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# Session-based storage allocator using filesystem symlink indirection. +# ffmpeg resolves the symlink ONCE at session start — existing sessions are never affected. +# +# How it works: +# ramdisk_setup.sh — creates tmpfs and symlink at array start via array_start.sh +# transcode_management.sh — every 3min, runs cleanup then manager in correct order +# transcode_cleanup.sh — removes old inactive files +# transcode_manager.sh — manages symlink direction based on usage thresholds +# +# ⚠️ Docker mount — must use shared propagation: +# --mount type=bind,source=/mnt/ram-transcode,target=/ext-ram-transcode,bind-propagation=shared +# Standard rprivate bind mounts lock the inode — sessions drift to SSD permanently. + +# ━━━ Transcode Manager ━━━ +# tmpfs mount point — created at array start by ramdisk_setup.sh +# Must exist before Emby starts so the symlink resolves correctly + RAMDISK_PATH="/mnt/ramdisk_transcodes" + +# Ramdisk size ceiling — tmpfs only uses RAM actually needed, not the full size upfront +# Set this to a comfortable limit based on your typical concurrent stream count +# Real-world: 9 streams peaked at ~5.5GB — 8G gives comfortable headroom + RAMDISK_SIZE="8G" + +# Symlink that Emby points at — this path NEVER changes regardless of ramdisk/SSD state +# Emby resolves the symlink once per session at start — symlink flips are transparent +# Must match the container path configured in Emby's Extra Parameters + TRANSCODE_LINK="/mnt/ram-transcode" + +# SSD fallback location — where transcodes land when ramdisk is too full +# Must have enough free space to handle peak session load + TRANSCODE_SSD="/mnt/cache/Temp_Storage/Emby/Transcodes/" + +# Usage thresholds in GB — hysteresis gap between WARN and LOW prevents flip-flop +# RAMDISK_WARN_GB: flip symlink to SSD when ramdisk usage reaches this +# RAMDISK_LOW_GB: flip symlink back to ramdisk when usage drops to this +# Gap (6.8 - 5.5 = 1.3GB) means ramdisk must drop 1.3GB before flipping back +# Without hysteresis a session right at the threshold causes rapid flipping + RAMDISK_WARN_GB=6.8 + RAMDISK_LOW_GB=5.5 + +# Minimum free GB on SSD before allowing a flip to SSD +# Prevents flipping to SSD when it's almost full — that would be worse than a full ramdisk + RAMDISK_SSD_MIN_GB=20 + +# File age thresholds in minutes before cleanup eligibility +# TRANSCODE_MAX_AGE: HLS segment files older than this with no active session = clean up +# TRANSCODE_ORPHAN_AGE: files with no matching session at all = clean up + TRANSCODE_MAX_AGE=20 + TRANSCODE_ORPHAN_AGE=30 + +# Notify if symlink flips this many times in one hour +# Frequent flips indicate the ramdisk is too small or thresholds need adjustment + TRANSCODE_FLIP_WARN=3 + +# Permissions applied to ramdisk and SSD transcode directories + TRANSCODE_OWNER="nobody:users" + TRANSCODE_CHMOD="755" + +# Operating mode — controls symlink direction behaviour +# smart — auto-flips between ramdisk and SSD based on RAMDISK_WARN_GB / RAMDISK_LOW_GB +# hysteresis gap prevents flip-flop — default for production +# ramdisk — always uses ramdisk, never flips to SSD +# warns if RAMDISK_WARN_GB exceeded but holds position +# use during SSD maintenance or when SSD space is low +# ssd — always uses SSD, never flips to ramdisk +# use during ramdisk maintenance or after a ramdisk issue + TRANSCODE_MANAGER_MODE="smart" # smart | ramdisk | ssd + +# Daily statistics log — read by weekly_health_digest.sh for transcode summary +# Tracks peak usage, flip count, session ratio, files cleaned per day +# Bounded to TRANSCODE_LOG_RETENTION days — auto-purges old entries on each write + TRANSCODE_DAILY_LOG="$DATA_DIR/transcode_daily.db" + TRANSCODE_LOG_RETENTION=90 # days before old entries are purged + +# ━━━ Transcode Server Array ━━━ +# All media servers sharing the ramdisk transcode space. +# Format: "ContainerName|URL|APIKey|Type" — Type: emby | jellyfin | plex +# Entries with placeholder API keys are skipped automatically. +# ⚠️ Tdarr does NOT belong here — keep Tdarr on SSD. + +TRANSCODE_SERVERS=( + "${HOST1_EMBY_CONTAINER}|${HOST1_EMBY_URL}|${HOST1_EMBY_API_KEY}|emby" + # "${HOST2_EMBY_CONTAINER}|${HOST2_EMBY_URL}|${HOST2_EMBY_API_KEY}|emby" + # "Jellyfin|http://localhost:8097|jellyfin-api-key|jellyfin" + # "Plex|http://localhost:32400|plex-token|plex" +) + + TRANSCODE_CHECK_EMBY=true + +# ============================================================================================== +# ── MONITORS ────────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +# ━━━ Certificate Monitor ━━━ +# Checks SSL certificate expiry via direct openssl connection — no NPM dependency. +# Checks the actual certificate served by each domain, not what NPM thinks it has. +# CERT_WARN_DAYS = notify this many days before expiry +# CERT_CRIT_DAYS = escalate to critical this many days before expiry +# CERT_TIMEOUT = seconds before giving up on the openssl connection +CERT_MONITOR_DOMAINS=( + "Gmer4Lfe.com" + "Gmer4Lfe.us" +) + CERT_WARN_DAYS=30 # warn when cert expires within this many days + CERT_CRIT_DAYS=7 # critical alert within this many days + CERT_TIMEOUT=10 # seconds per domain check + +# ━━━ Backup Verify ━━━ +# Verifies rsync mirror health by comparing random file checksums between servers. +# Catches silent corruption or incomplete syncs that rsync itself wouldn't detect. +# Leave BACKUP_VERIFY_SHARES empty to use HOST*_DAILY_SYNC_SHARES automatically. +# BACKUP_VERIFY_SAMPLE = number of random files to checksum per share +# BACKUP_VERIFY_MIN_SIZE = skip files smaller than this (small files are rarely corrupted) +BACKUP_VERIFY_SHARES=( + # leave empty to use HOST*_DAILY_SYNC_SHARES automatically +) + BACKUP_VERIFY_SAMPLE=10 # random files to check per share + BACKUP_VERIFY_MIN_SIZE=1M # minimum file size to include in sample + +# ━━━ SMART Health ━━━ +# Monitors drive SMART attributes — discovers all drives automatically via /dev/sd* and /dev/nvme*. +# Reads live SMART data — no persistent writes. +# SMART_IGNORE_DRIVES = drives to skip (boot USB, drives without meaningful SMART data) + SMART_TEMP_WARN=45 # Celsius — warn above this temperature + SMART_TEMP_CRIT=55 # Celsius — critical above this temperature +SMART_IGNORE_DRIVES=( + "sda" # boot USB — SMART not meaningful on flash drives +) + +# ━━━ ZFS Memory Snapshot ━━━ +# Weekly ZFS pool health and memory diagnostic report — informational only, no action taken. +# ZFS_REPORT_ARC_WARN_PCT = warn if ARC is using more than this % of its max +# ZFS_REPORT_FREE_WARN_GB = warn if less than this GB free RAM +# ZFS_REPORT_AVAIL_WARN_GB = warn if less than this GB available on ZFS pool +# ZFS_REPORT_DOCKER_TOP = how many top Docker containers to show by memory usage +# ZFS_REPORT_IGNORE_POOLS = individual disk pools to skip (unRAID array disks as ZFS) + ZFS_REPORT_LOG="/var/log/zfs-weekly-health.log" + ZFS_REPORT_ARC_WARN_PCT=90 + ZFS_REPORT_FREE_WARN_GB=10 + ZFS_REPORT_AVAIL_WARN_GB=20 + ZFS_REPORT_DOCKER_TOP=10 +ZFS_REPORT_IGNORE_POOLS=( + "disk10" + "disk9" + "disk8" + "disk6" + "disk5" +) + +# ━━━ Bandwidth Monitor ━━━ +# Called automatically by rsync.sh after each sync — one bounded write per run. +# Tracks transfer size, duration and profile per sync for weekly summary reporting. +# BANDWIDTH_LOG_RETENTION = days to keep entries before auto-purging old records +# BANDWIDTH_WARN_GB = flag in weekly summary if a single sync exceeded this size + BANDWIDTH_LOG="$DATA_DIR/bandwidth_history.db" + BANDWIDTH_LOG_RETENTION=90 # days before old entries are purged + BANDWIDTH_WARN_GB=50 # flag syncs larger than this in weekly report + +# Stats files — written by cleanup and recovery scripts, read by coffee report +# All in DATA_DIR — array always running when these are written + ARR_CLEANUP_STATS="$DATA_DIR/arr_cleanup_stats.db" # lidarr/sonarr/radarr orphan stats + ARR_RECOVERY_STATS="$DATA_DIR/arr_recovery_stats.db" # blocklist + re-search stats + +# ━━━ Health Digest ━━━ +# Aggregated system health summary — reads existing state files, no new writes. +# Three profiles control when the digest email is sent: +# always — sends every run regardless of findings +# smart — sends only when DIGEST_SMART_ON_* conditions are found +# weekly — sends once per week on DIGEST_DAY only +# Smart profile triggers — set true to send digest when finding is detected: + DIGEST_PROFILE="weekly" # always | smart | weekly + DIGEST_DAY="Sunday" # day of week for weekly profile + DIGEST_SMART_ON_WATCHDOG=true # send if any watchdog strikes are active + DIGEST_SMART_ON_FAILOVER=true # send if failover state is not NORMAL + DIGEST_SMART_ON_CERT_WARN=true # send if any cert is under CERT_WARN_DAYS + DIGEST_SMART_ON_BANDWIDTH=true # send if any transfer exceeded BANDWIDTH_WARN_GB + +# ━━━ Emby Session Report ━━━ +# Weekly Emby usage statistics via API — no persistent writes, queries fresh each run. +# Shows top content, most active users, session counts over the report period. +# URL and API key pulled from HOST1/HOST2_EMBY_URL and HOST1/HOST2_EMBY_API_KEY +# defined in Host Configuration at the top of this file — no duplication needed. + EMBY_REPORT_DAYS=7 # days to include in the report period + EMBY_REPORT_TOP_N=10 # number of top content items to show + +# ============================================================================================== +# ── SYSTEM WATCHDOG ─────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# Continuous system health monitoring — last line of defense before a crash. +# Started by array_start.sh — runs until array stops. +# Re-sources Master.conf each cycle — config changes take effect on next cycle. +# Strike system: sustained threshold hits trigger reboot — single spikes ignored. +# Reboot loop protection: shuts down instead if reboot limit hit in rolling window. +# Silent when healthy — logs only when a threshold is triggered. + +# ━━━ State Files ━━━ + SYS_WATCHDOG_STATE_FILE="/tmp/system_watchdog_state.db" # /tmp resets on reboot ✅ + SYS_WATCHDOG_FAILED_FILE="/boot/config/system_watchdog_failed.db" # survives reboots + SYS_WATCHDOG_REBOOT_LOG="/boot/config/system_watchdog_reboots.db" # reboot loop detection + +# ━━━ Strike and Reboot Loop Settings ━━━ +# Strike system: a check must fail this many consecutive cycles before action is taken +# Single spikes (one bad reading) are ignored — sustained problems trigger reboot + SYS_WATCHDOG_STRIKE_LIMIT=2 # consecutive failures before reboot trigger + +# How often checks run — 300s = 5 minutes +# At STRIKE_LIMIT=2 and INTERVAL=300: problem must persist 10min before reboot + SYSTEM_WATCHDOG_INTERVAL=300 # seconds between watchdog cycles + +# Reboot loop protection — if system keeps rebooting something is seriously wrong +# After REBOOT_LIMIT reboots in REBOOT_WINDOW_HRS hours → shutdown instead of reboot +# Prevents infinite reboot loops when the underlying problem can't be fixed by rebooting + SYS_WATCHDOG_REBOOT_LIMIT=3 # max reboots before shutdown instead + SYS_WATCHDOG_REBOOT_WINDOW_HRS=12 # rolling window in hours + +# Heartbeat — proof of life logged periodically even when everything is healthy + SYSTEM_WATCHDOG_HEARTBEAT=true # true = log heartbeat / false = completely silent + SYSTEM_WATCHDOG_HEARTBEAT_HOURS=1 # how often to log heartbeat (hours) + +# ━━━ Thresholds ━━━ +# Set at "about to become unstable" levels — not "things are a bit high" +# These should be high enough that normal operation never triggers them + +# rootfs (/) usage percentage — when array is down rsync writes land on rootfs +# fills rapidly and can crash the server — 95% is almost too late, act fast + SYS_WATCHDOG_ROOTFS_PCT=95 + +# /var/log usage percentage — log spam can fill rootfs, indicates something broken + SYS_WATCHDOG_LOG_PCT=95 + +# Free RAM in GB — below this is critically low, OOM or swap imminent +# Your server has 128GB — 4GB free means something is consuming everything + SYS_WATCHDOG_MEM_GB=4 + +# ZFS ARC pinned percentage — ARC not releasing after reclaim = memory stuck +# SYS_WATCHDOG_ARC_RELEASE_PCT = after reclaim attempt, if still above this → trigger + SYS_WATCHDOG_ARC_PINNED_PCT=98 + SYS_WATCHDOG_ARC_RELEASE_PCT=95 + +# Load average multiplier — threshold = MULTIPLIER × CPU core count +# MULTIPLIER=3 on 16-core = load average of 48 before triggering +# Set high — transcoding causes legitimate high load spikes + SYS_WATCHDOG_LOAD_MULTIPLIER=3 + +# Zombie process count — large numbers indicate serious process management failure +# A few zombies are normal — 50 means something is very wrong + SYS_WATCHDOG_ZOMBIE_LIMIT=50 + +# CPU temperature in Celsius — sustained high temp causes throttling or kernel panic +# 95°C is close to tjmax on most CPUs — triggers before thermal shutdown + SYS_WATCHDOG_CPU_TEMP_MAX=95 + +# ━━━ Check Toggles ━━━ +# Disable individual checks without disabling the whole watchdog +# All enabled by default except load — transcoding causes legitimate load spikes + SYS_WATCHDOG_CHECK_ROOTFS=true + SYS_WATCHDOG_CHECK_LOG=true + SYS_WATCHDOG_CHECK_RAM=true + SYS_WATCHDOG_CHECK_ARC=true + SYS_WATCHDOG_CHECK_CPU_TEMP=true + SYS_WATCHDOG_CHECK_LOAD=false # disabled — load spikes during transcoding are normal + SYS_WATCHDOG_CHECK_ZOMBIES=true + SYS_WATCHDOG_CHECK_CONTAINERS=true # checks docker_watchdog persistent skip list + SYS_WATCHDOG_CHECK_DOCKER_DAEMON=true # checks if Docker daemon is responding + +# ━━━ Abort Toggles ━━━ +# Conditions that prevent reboot even when a threshold is hit +# true = abort reboot if this condition is active (conservative — avoid data loss) +# false = reboot anyway (aggressive — a clean reboot beats a hard crash) +# Philosophy: aborting is safer for data, rebooting is safer for stability + SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY=true # rebooting with bad pool risks data loss + SYS_WATCHDOG_ABORT_ON_PARITY=false # aborting parity is better than crashing mid-check + SYS_WATCHDOG_ABORT_ON_MOVER=false # aborting move is better than crashing mid-move + +# ============================================================================================== +# ──────────────────────── End Of User Variables ─────────────────────────────────────────────── +# ============================================================================================== \ No newline at end of file diff --git a/Orchestrators/array_start.sh b/Orchestrators/array_start.sh index 812ebdc..a2c3713 100644 --- a/Orchestrators/array_start.sh +++ b/Orchestrators/array_start.sh @@ -9,7 +9,7 @@ # Transcodes/ramdisk_setup.sh — creates tmpfs + symlink before Emby starts (one-shot) # unRAID_Essentials/docker_syslog_filter.sh — suppress veth log noise (one-shot) # unRAID_Essentials/php_fpm_max_children.sh — WebGUI performance tuning (one-shot) -# Docker_Essentials/docker_network_connect.sh — connect containers to extra networks (one-shot) +# Docker_Essentials/docker_network_connect.sh — ensure networks exist + connect containers (one-shot) # unRAID_Essentials/system_watchdog.sh — system health monitor (continuous loop) # Docker_Essentials/docker_watchdog.sh — container health monitor (continuous loop) # Failover/failover.sh — mutual failover monitor (continuous loop)