audit echo vs log across all scripts — outcomes always visible, verbose for per-item loops
This commit is contained in:
Gmer4Lfe
@@ -0,0 +1,494 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ========================== HOSTN CONFIGURATION — (hostname) ==================================
|
||||
# ==============================================================================================
|
||||
# HOSTN-specific variables — credentials, container names, share paths, failover lists.
|
||||
# Sourced after master.conf — values here extend shared profile arrays and add HOSTN-specific
|
||||
# identity, credentials, and container configuration.
|
||||
#
|
||||
# Sparse checkout (git) ensures other hosts never receive this file.
|
||||
#
|
||||
# DO NOT put shared config here — thresholds, toggles, profiles belong in master.conf.
|
||||
# DO NOT put other hosts' variables here — they belong in their own host*.conf files.
|
||||
#
|
||||
# ── HOW TO USE THIS TEMPLATE ──────────────────────────────────────────────────────────────────
|
||||
# This file was generated by the Varaverk first-run wizard.
|
||||
# Fill in the sections that apply to your setup — leave unused sections empty.
|
||||
# All scripts self-guard against empty values — safe to leave sections blank until needed.
|
||||
#
|
||||
# ── INDEX ─────────────────────────────────────────────────────────────────────────────────────
|
||||
#
|
||||
# ── IDENTITY & CONNECTIVITY ────────────────────────────────────────────────────────────────
|
||||
# IDENTITY hostname, SSH key, Unraid API key
|
||||
# EMBY container name, URL, API key
|
||||
# JELLYFIN container name, URL, API key
|
||||
# GITEA API token for SSH key registration
|
||||
# NOTIFICATIONS Discord webhook
|
||||
#
|
||||
# ── PARTNERSHIP ────────────────────────────────────────────────────────────────────────────
|
||||
# PARTNERSHIP auth containers, backup paths, emby provisioning
|
||||
#
|
||||
# ── RSYNC ──────────────────────────────────────────────────────────────────────────────────
|
||||
# DAILY SYNC SHARES media shares this host owns and pushes
|
||||
# PERSONAL SHARES private encrypted shares for offsite backup
|
||||
# WEEKLY SYNC SHARES appdata shares synced weekly
|
||||
# INTERMEDIATE SYNC mid-day appdata propagation
|
||||
# CRITICAL SYNC SHARES appdata shares synced every 30 minutes
|
||||
# BACKUP VERIFY shares for checksum verification against remote
|
||||
# HOSTN RSYNC PROFILE host-specific appdata sync profile
|
||||
#
|
||||
# ── FALLBACK ───────────────────────────────────────────────────────────────────────────────
|
||||
# DDNS DDNS containers managed by this host
|
||||
# INTERNET LOSS containers stopped when internet is lost
|
||||
# FALLBACK TIERS what this host runs for the remote per tier
|
||||
# TIER DELAYS delays before each tier activates
|
||||
# RSYNC WRITEBACK appdata synced back on handback
|
||||
#
|
||||
# ── DOCKER ─────────────────────────────────────────────────────────────────────────────────
|
||||
# DOCKER DAILY RESTART containers restarted daily
|
||||
# DOCKER WEEKLY RESTART containers restarted weekly
|
||||
# DOCKER WATCHDOG memory limits, health URLs, required containers
|
||||
# NETWORK WATCHDOG DDNS domain, NPM URL for connectivity checks
|
||||
# DOCKER NETWORK CONNECT networks and containers for array start
|
||||
#
|
||||
# ── MEDIA ──────────────────────────────────────────────────────────────────────────────────
|
||||
# MEDIA PERMISSIONS share list for permissions script
|
||||
# MEDIA CLEANER folder lists for media_cleaner.sh
|
||||
#
|
||||
# ── ARR STACK ──────────────────────────────────────────────────────────────────────────────
|
||||
# DOWNLOADERS slskd, SABnzbd, qBittorrent credentials and URLs
|
||||
# LIDARR / SONARR / RADARR URL, API key, path map
|
||||
# ARR RECOVERY per-arr recovery toggles
|
||||
#
|
||||
# ── TRANSCODES ─────────────────────────────────────────────────────────────────────────────
|
||||
# TRANSCODES ramdisk size, thresholds, SSD path, server array
|
||||
#
|
||||
# ── MONITORS ───────────────────────────────────────────────────────────────────────────────
|
||||
# CERTIFICATE MONITOR domains checked for SSL expiry
|
||||
# SMART HEALTH drives to skip in SMART monitoring
|
||||
# ZFS REPORT pools to exclude from ZFS health report
|
||||
#
|
||||
# ── RESOURCE MANAGER ───────────────────────────────────────────────────────────────────────
|
||||
# RESOURCE MANAGER containers paused/stopped under memory pressure
|
||||
#
|
||||
# ── SYSTEM WATCHDOG ────────────────────────────────────────────────────────────────────────
|
||||
# SYSTEM WATCHDOG per-host check toggles and NIC configuration
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
# ==============================================================================================
|
||||
# ── STORAGE MODE ──────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Storage mode ━━━
|
||||
# Controls where Varaverk stores scripts, conf, and state files.
|
||||
# true = internal NVMe/SSD — /boot/config/plugins/varaverk (write-safe, git-direct)
|
||||
# false = USB flash boot — /mnt/user/appdata/Varaverk (preserves flash lifetime)
|
||||
# Auto-detected from boot device transport on first setup.
|
||||
# To change: Settings → Storage → Migrate.
|
||||
HOSTN_STORAGE_MODE_INTERNAL=true
|
||||
|
||||
# ==============================================================================================
|
||||
# ── IDENTITY & CONNECTIVITY ───────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Identity ━━━
|
||||
# HOSTN hostname lives in master.conf (not a credential — safe for all servers).
|
||||
# SSH key used for all server-to-server operations — rsync, failover, conf sync.
|
||||
# Convention: /root/.ssh/<hostname-lowercase-no-unraid-prefix>_rsync_automation
|
||||
# Must be in /root/.ssh/ and authorised in the partner's /root/.ssh/authorized_keys.
|
||||
# Run Partnership/ssh_setup.sh to generate the key and copy it to the partner.
|
||||
HOSTN_SSH_KEY="" # e.g. /root/.ssh/myserver_rsync_automation
|
||||
HOSTN_OWNER="" # short identifier for this server (e.g. myserver)
|
||||
HOSTN_OWNER_EMAIL=""
|
||||
|
||||
# ━━━ Unraid API ━━━
|
||||
# Used by the Varaverk plugin to query this server's Unraid GraphQL API.
|
||||
# Generate in Unraid: Settings → Management Access → API Keys → + New Key
|
||||
HOSTN_UNRAID_API_KEY=""
|
||||
|
||||
# ━━━ Emby ━━━
|
||||
HOSTN_EMBY_CONTAINER="Emby"
|
||||
HOSTN_EMBY_URL="http://localhost:8096"
|
||||
HOSTN_EMBY_API_KEY="" # Emby Dashboard → API Keys → + New Key
|
||||
|
||||
# ━━━ Jellyfin ━━━
|
||||
HOSTN_JELLYFIN_CONTAINER="Jellyfin"
|
||||
HOSTN_JELLYFIN_URL="http://localhost:8095"
|
||||
HOSTN_JELLYFIN_API_KEY="" # Jellyfin Dashboard → Administration → API Keys
|
||||
|
||||
# ━━━ Gitea ━━━
|
||||
# Personal access token for gitea_ssh_setup.sh.
|
||||
# Create in Gitea: Settings → Applications → Generate Token → scope: write:user
|
||||
HOSTN_GITEA_API_TOKEN=""
|
||||
|
||||
# ━━━ Notifications ━━━
|
||||
# Discord webhook — leave blank to disable.
|
||||
HOSTN_DISCORD_WEBHOOK=""
|
||||
|
||||
# ==============================================================================================
|
||||
# ── PARTNERSHIP ───────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# Auth containers reconfigured on onboard/offboard.
|
||||
# Format: "ContainerName|WebUIPort"
|
||||
HOSTN_PARTNERSHIP_AUTH_WEBUIS=(
|
||||
# "NginxProxyManager|81"
|
||||
# "Authelia|9091"
|
||||
)
|
||||
|
||||
# XML templates pushed to mirror during onboard — auth stack.
|
||||
# Dependencies (databases) must come before apps that depend on them.
|
||||
HOSTN_PARTNERSHIP_AUTH_STACK=(
|
||||
# "my-Authelia.xml"
|
||||
# "my-NginxProxyManager.xml"
|
||||
)
|
||||
|
||||
# XML templates pushed to mirror during onboard — arr stack.
|
||||
HOSTN_PARTNERSHIP_ARR_STACK=(
|
||||
# "my-Sonarr.xml"
|
||||
# "my-Radarr.xml"
|
||||
)
|
||||
|
||||
# Paths the partner should collect during the grace window after offboard.
|
||||
HOSTN_PARTNERSHIP_MIRROR_BACKUPS=(
|
||||
# "/mnt/user/appdata-Fallback/Partner-Emby"
|
||||
)
|
||||
|
||||
# Containers parked on this server when partnership is active.
|
||||
HOSTN_PARTNERSHIP_OWN_CONTAINERS=(
|
||||
# "Emby"
|
||||
)
|
||||
|
||||
# Emby admin provisioning — owner controls whether Emby is shared.
|
||||
HOSTN_PARTNERSHIP_PROVISION_EMBY_ADMIN=false
|
||||
HOSTN_PARTNERSHIP_EMBY_PORT=8096
|
||||
HOSTN_PARTNERSHIP_EMBY_ADMIN_USER=""
|
||||
HOSTN_PARTNERSHIP_EMBY_ADMIN_PASS=""
|
||||
|
||||
# ==============================================================================================
|
||||
# ── RSYNC ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Daily Sync Shares ━━━
|
||||
# Media shares this host pushes to all other nodes every night.
|
||||
# Uses DEFAULT_RSYNC_OPTS from master.conf — no profile needed.
|
||||
HOSTN_DAILY_SYNC_SHARES=(
|
||||
# /mnt/user/Movies
|
||||
# /mnt/user/Tv_Shows
|
||||
# /mnt/user/Music
|
||||
)
|
||||
|
||||
# ━━━ Personal Shares ━━━
|
||||
# Private encrypted shares synced for offsite backup, independent of media shares.
|
||||
HOSTN_PERSONAL_SHARES=(
|
||||
# /mnt/user/Personal # e.g. ZFS-encrypted dataset
|
||||
)
|
||||
|
||||
# ━━━ Weekly Sync Shares ━━━
|
||||
# Appdata shares synced during the weekly maintenance window.
|
||||
# Profiles (emby, critical-data) drive container stops — define in master.conf.
|
||||
HOSTN_WEEKLY_SYNC_SHARES=(
|
||||
# "/mnt/user/Media_Server/Emby" # emby profile
|
||||
# "/mnt/user/appdata-Fallback/Critical-Data" # critical-data profile
|
||||
)
|
||||
|
||||
# ━━━ Intermediate Sync Shares ━━━
|
||||
# Shares synced every 4 hours. Leave empty to skip mid-day rsync.
|
||||
HOSTN_INTERMEDIATE_SYNC_SHARES=(
|
||||
# Add shares here to enable mid-day rsync
|
||||
)
|
||||
|
||||
# ━━━ Critical Sync Shares ━━━
|
||||
# Appdata shares synced every 30 minutes.
|
||||
# Format: "/path/to/share" or "/path/to/share|profile-name"
|
||||
HOSTN_CRITICAL_SYNC_SHARES=(
|
||||
# "/mnt/user/appdata-Fallback/Critical-Data|critical-fallback"
|
||||
# "/mnt/user/Media_Server/Emby|emby-fallback"
|
||||
)
|
||||
|
||||
# ━━━ Backup Verify ━━━
|
||||
# Leave empty to use HOSTN_DAILY_SYNC_SHARES automatically.
|
||||
HOSTN_BACKUP_VERIFY_SHARES=(
|
||||
# leave empty to use HOSTN_DAILY_SYNC_SHARES automatically
|
||||
)
|
||||
|
||||
# ━━━ HOSTN Rsync Profile — hostn-appdata ━━━
|
||||
# Host-specific appdata sync profile.
|
||||
PROFILE_RSYNC_OPTS[hostn-appdata]="-av --info=progress2 --bwlimit=${PROFILE_BW_LIMIT[hostn-appdata]:-8000}"
|
||||
PROFILE_BW_LIMIT[hostn-appdata]=8000
|
||||
PROFILE_RETRY_COUNT[hostn-appdata]=3
|
||||
PROFILE_SLEEP[hostn-appdata]=300
|
||||
PROFILE_CRITICAL_CONTAINER_NAMES[hostn-appdata]=""
|
||||
PROFILE_DELAYED_CONTAINERS[hostn-appdata]=""
|
||||
PROFILE_CONTAINER_DELAY[hostn-appdata]=5
|
||||
PROFILE_EXCLUDE_DIRS[hostn-appdata]="logs *.tmp"
|
||||
|
||||
# ==============================================================================================
|
||||
# ── FALLBACK ──────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ DDNS ━━━
|
||||
# DDNS containers this host manages.
|
||||
HOSTN_DDNS_CONTAINERS=(
|
||||
# "MyServer.com"
|
||||
)
|
||||
|
||||
# ━━━ Internet Loss ━━━
|
||||
# Containers stopped immediately when internet is lost.
|
||||
FALLBACK_HOSTN_STOP_ON_NO_NET=(
|
||||
# "MyServer.com"
|
||||
)
|
||||
|
||||
# ━━━ Fallback Tiers — HOSTN Runs for Partner ━━━
|
||||
# Containers this host starts when the partner goes down.
|
||||
# Replace REMOTE_ID below with the actual remote host ID (HOST1, HOST2, etc.)
|
||||
FALLBACK_HOSTN_COVERS_REMOTE_ID_TIER1=(
|
||||
# "Partner-DDNS-Container"
|
||||
)
|
||||
|
||||
FALLBACK_HOSTN_COVERS_REMOTE_ID_TIER2=(
|
||||
# "container-placeholder"
|
||||
)
|
||||
|
||||
FALLBACK_HOSTN_COVERS_REMOTE_ID_TIER3=(
|
||||
# "container-placeholder"
|
||||
)
|
||||
|
||||
FALLBACK_HOSTN_COVERS_REMOTE_ID_TIER4=(
|
||||
# "container-placeholder"
|
||||
)
|
||||
|
||||
# ━━━ Tier Delays — Partner's Containers on this Host ━━━
|
||||
# How long the partner must be down before each tier activates here — in minutes.
|
||||
# Replace REMOTE_ID with the actual remote host ID (HOST1, HOST2, etc.)
|
||||
REMOTE_ID_TIER2_DELAY=240 # 4 hours
|
||||
REMOTE_ID_TIER3_DELAY=720 # 12 hours
|
||||
REMOTE_ID_TIER4_DELAY=1440 # 24 hours
|
||||
|
||||
# ━━━ Rsync Writeback ━━━
|
||||
HOSTN_TIER1_WRITEBACK_DELAY=60 # skip Emby writeback if outage under 1hr
|
||||
|
||||
FALLBACK_HOSTN_WRITEBACK_TIER1=(
|
||||
# "/mnt/user/Media_Server/Emby"
|
||||
)
|
||||
|
||||
FALLBACK_HOSTN_WRITEBACK_TIER2=(
|
||||
# "/mnt/user/appdata-Fallback/Important-Data"
|
||||
)
|
||||
|
||||
FALLBACK_HOSTN_WRITEBACK_TIER3=(
|
||||
# "location-placeholder"
|
||||
)
|
||||
|
||||
FALLBACK_HOSTN_WRITEBACK_TIER4=(
|
||||
# "/mnt/user/appdata-Fallback/Arrs_Stack"
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── DOCKER ────────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Docker Daily Restart ━━━
|
||||
HOSTN_DAILY_RESTART_CONTAINERS=(
|
||||
# "NginxProxyManager"
|
||||
# "Authelia"
|
||||
)
|
||||
|
||||
# ━━━ Docker Weekly Restart ━━━
|
||||
HOSTN_WEEKLY_RESTART_CONTAINERS=(
|
||||
# "NextCloud"
|
||||
# "AdGuard-Home"
|
||||
)
|
||||
|
||||
# ━━━ Docker Watchdog ━━━
|
||||
|
||||
# Memory hard limits in MB — immediate restart if exceeded.
|
||||
# 20GB=20480 16GB=16384 8GB=8192 4GB=4096 2GB=2048 1GB=1024
|
||||
declare -A HOSTN_WATCHDOG_CONTAINERS=(
|
||||
# ["Emby"]=18432
|
||||
)
|
||||
|
||||
# HTTP health check URLs — checked every cycle.
|
||||
declare -A HOSTN_WATCHDOG_CONTAINER_URLS=(
|
||||
# ["Emby"]="http://localhost:8096"
|
||||
)
|
||||
|
||||
# Required containers — must always be running.
|
||||
HOSTN_WATCHDOG_REQUIRED_CONTAINERS=(
|
||||
# "NginxProxyManager"
|
||||
# "Authelia"
|
||||
)
|
||||
|
||||
# Containers to skip in Tier 2 global scan.
|
||||
HOSTN_WATCHDOG_SCAN_IGNORE=(
|
||||
# "my-occasional-container"
|
||||
)
|
||||
|
||||
# Dependency ordering — skip restarting a container if its dependency is also down.
|
||||
declare -A HOSTN_WATCHDOG_DEPENDENCIES=(
|
||||
# ["Authelia"]="Mariadb Redis-Authelia"
|
||||
)
|
||||
|
||||
# Per-container appdata growth suppress ceilings in MB.
|
||||
declare -A HOSTN_WATCHDOG_APPDATA_SIZES=(
|
||||
# ["Tdarr"]="25600"
|
||||
)
|
||||
|
||||
# ━━━ Network Watchdog ━━━
|
||||
HOSTN_NETWORK_WATCHDOG_DDNS_DOMAIN="" # e.g. myserver.com
|
||||
HOSTN_NETWORK_WATCHDOG_DDNS_CONTAINER="" # e.g. MyServer.com
|
||||
HOSTN_NETWORK_WATCHDOG_NPM_URL="" # e.g. https://myserver.com
|
||||
|
||||
# ━━━ Docker Network Connect ━━━
|
||||
HOSTN_NETWORK_CONNECT_CONTAINERS=(
|
||||
# "memcached"
|
||||
)
|
||||
|
||||
HOSTN_NETWORK_CONNECT_NETWORKS=(
|
||||
# "high-availability"
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── MEDIA ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Media Permissions ━━━
|
||||
HOSTN_MEDIA_PERMISSION_SHARES=(
|
||||
# /mnt/user/Movies
|
||||
# /mnt/user/Tv_Shows
|
||||
# /mnt/user/Music
|
||||
# /mnt/user/Downloads
|
||||
)
|
||||
|
||||
# ━━━ Media Cleaner ━━━
|
||||
HOSTN_ANIME_CLEAN_FOLDERS=(
|
||||
# /mnt/user/Anime_Movies
|
||||
# /mnt/user/Anime_Shows
|
||||
)
|
||||
|
||||
HOSTN_MEDIA_CLEAN_FOLDERS=(
|
||||
# /mnt/user/Movies
|
||||
# /mnt/user/Tv_Shows
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── ARR STACK ─────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Downloaders ━━━
|
||||
HOSTN_SLSKD_URL="http://localhost:8980"
|
||||
HOSTN_SLSKD_API_KEY=""
|
||||
HOSTN_SLSKD_FAILED_IMPORTS_DIR=""
|
||||
|
||||
HOSTN_SABNZBD_URL="http://localhost:8180"
|
||||
HOSTN_SABNZBD_API_KEY=""
|
||||
|
||||
HOSTN_QBIT_URL="http://localhost:8080"
|
||||
HOSTN_QBIT_USERNAME="admin"
|
||||
HOSTN_QBIT_PASSWORD=""
|
||||
|
||||
# ━━━ Lidarr ━━━
|
||||
HOSTN_LIDARR_URL="http://localhost:8686"
|
||||
HOSTN_LIDARR_API_KEY=""
|
||||
HOSTN_LIDARR_MUSIC_ROOT="/mnt/user/Music"
|
||||
HOSTN_FANART_API_KEY=""
|
||||
HOSTN_LASTFM_API_KEY=""
|
||||
|
||||
declare -A HOSTN_LIDARR_PATH_MAP=(
|
||||
# ["/music"]="/mnt/user/Music"
|
||||
)
|
||||
|
||||
# ━━━ Sonarr ━━━
|
||||
HOSTN_SONARR_URL="http://localhost:8989"
|
||||
HOSTN_SONARR_API_KEY=""
|
||||
HOSTN_SONARR_TV_ROOT="/mnt/user/Tv_Shows"
|
||||
|
||||
declare -A HOSTN_SONARR_PATH_MAP=(
|
||||
# ["/tv"]="/mnt/user/Tv_Shows"
|
||||
)
|
||||
|
||||
# ━━━ Radarr ━━━
|
||||
HOSTN_RADARR_URL="http://localhost:7878"
|
||||
HOSTN_RADARR_API_KEY=""
|
||||
HOSTN_TMDB_API_KEY=""
|
||||
HOSTN_RADARR_MOVIE_ROOT="/mnt/user/Movies"
|
||||
|
||||
declare -A HOSTN_RADARR_PATH_MAP=(
|
||||
# ["/movies"]="/mnt/user/Movies"
|
||||
)
|
||||
|
||||
# ━━━ Arr Recovery Toggles ━━━
|
||||
HOSTN_LIDARR_RECOVERY=false
|
||||
HOSTN_SONARR_RECOVERY=true
|
||||
HOSTN_RADARR_RECOVERY=true
|
||||
|
||||
# ==============================================================================================
|
||||
# ── TRANSCODES ────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
HOSTN_RAMDISK_SIZE="10G"
|
||||
HOSTN_RAMDISK_WARN_GB=8.5
|
||||
HOSTN_RAMDISK_LOW_GB=7
|
||||
HOSTN_TRANSCODE_SSD="/mnt/cache/Temp_Storage/Emby/Transcodes/"
|
||||
|
||||
HOSTN_TRANSCODE_SERVERS=(
|
||||
"${HOSTN_EMBY_CONTAINER}|${HOSTN_EMBY_URL}|${HOSTN_EMBY_API_KEY}|emby"
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── MONITORS ──────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Certificate Monitor ━━━
|
||||
HOSTN_CERT_MONITOR_DOMAINS=(
|
||||
# "myserver.com"
|
||||
)
|
||||
|
||||
# ━━━ SMART Health ━━━
|
||||
HOSTN_SMART_IGNORE_DRIVES=(
|
||||
"sda" # boot USB — SMART not meaningful on flash drives
|
||||
)
|
||||
|
||||
# ━━━ ZFS Report ━━━
|
||||
HOSTN_ZFS_REPORT_IGNORE_POOLS=(
|
||||
# "disk5"
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── RESOURCE MANAGER ──────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
HOSTN_RW_PAUSE_CONTAINERS=(
|
||||
# "Tdarr"
|
||||
# "LidaTube"
|
||||
)
|
||||
|
||||
HOSTN_RW_STOP_CONTAINERS=(
|
||||
# "Tdarr"
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── SYSTEM WATCHDOG ───────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
HOSTN_SYS_WATCHDOG_NIC="" # e.g. eth0 — for network monitoring
|
||||
|
||||
HOSTN_SYS_WATCHDOG_CHECK_DOCKER=true
|
||||
HOSTN_SYS_WATCHDOG_CHECK_ROOTFS=true
|
||||
HOSTN_SYS_WATCHDOG_CHECK_KERNEL_OOPS=true
|
||||
HOSTN_SYS_WATCHDOG_CHECK_FD=true
|
||||
HOSTN_SYS_WATCHDOG_CHECK_BOOT=true
|
||||
HOSTN_SYS_WATCHDOG_CHECK_OOM=true
|
||||
HOSTN_SYS_WATCHDOG_CHECK_RAM=true
|
||||
HOSTN_SYS_WATCHDOG_CHECK_LOG=true
|
||||
HOSTN_SYS_WATCHDOG_CHECK_ARC=true
|
||||
HOSTN_SYS_WATCHDOG_CHECK_TEMP=true
|
||||
HOSTN_SYS_WATCHDOG_CHECK_LOAD=true
|
||||
HOSTN_SYS_WATCHDOG_CHECK_ZOMBIES=true
|
||||
HOSTN_SYS_WATCHDOG_CHECK_CONTAINERS=true
|
||||
HOSTN_SYS_WATCHDOG_CHECK_TMP=true
|
||||
HOSTN_SYS_WATCHDOG_CHECK_MDSTAT=true
|
||||
HOSTN_SYS_WATCHDOG_CHECK_NETWORK=true
|
||||
HOSTN_SYS_WATCHDOG_CHECK_SSHD=true
|
||||
HOSTN_SYS_WATCHDOG_CHECK_RUNAWAY=false
|
||||
@@ -0,0 +1,505 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ========================== HOSTN CONFIGURATION — (hostname) ==================================
|
||||
# ==============================================================================================
|
||||
# HOSTN-specific variables — credentials, container names, share paths, failover lists.
|
||||
# Sourced after master.conf — values here extend shared profile arrays and add HOSTN-specific
|
||||
# identity, credentials, and container configuration.
|
||||
#
|
||||
# Sparse checkout (git) ensures other hosts never receive this file.
|
||||
#
|
||||
# DO NOT put shared config here — thresholds, toggles, profiles belong in master.conf.
|
||||
# DO NOT put other hosts' variables here — they belong in their own host*.conf files.
|
||||
#
|
||||
# ── HOW TO USE THIS TEMPLATE ──────────────────────────────────────────────────────────────────
|
||||
# This file was generated by the Varaverk first-run wizard.
|
||||
# Fill in the sections that apply to your setup — leave unused sections empty.
|
||||
# All scripts self-guard against empty values — safe to leave sections blank until needed.
|
||||
#
|
||||
# ── INDEX ─────────────────────────────────────────────────────────────────────────────────────
|
||||
#
|
||||
# ── IDENTITY & CONNECTIVITY ────────────────────────────────────────────────────────────────
|
||||
# IDENTITY hostname, SSH key, Unraid API key
|
||||
# EMBY container name, URL, API key
|
||||
# JELLYFIN container name, URL, API key
|
||||
# GITEA API token for SSH key registration
|
||||
# NOTIFICATIONS Discord webhook
|
||||
#
|
||||
# ── PARTNERSHIP ────────────────────────────────────────────────────────────────────────────
|
||||
# PARTNERSHIP auth containers, backup paths, emby provisioning
|
||||
#
|
||||
# ── RSYNC ──────────────────────────────────────────────────────────────────────────────────
|
||||
# DAILY SYNC SHARES media shares this host owns and pushes
|
||||
# PERSONAL SHARES private encrypted shares for offsite backup
|
||||
# WEEKLY SYNC SHARES appdata shares synced weekly
|
||||
# INTERMEDIATE SYNC mid-day appdata propagation
|
||||
# CRITICAL SYNC SHARES appdata shares synced every 30 minutes
|
||||
# BACKUP VERIFY shares for checksum verification against remote
|
||||
# HOSTN RSYNC PROFILE host-specific appdata sync profile
|
||||
#
|
||||
# ── FALLBACK ───────────────────────────────────────────────────────────────────────────────
|
||||
# DDNS DDNS containers managed by this host
|
||||
# INTERNET LOSS containers stopped when internet is lost
|
||||
# FALLBACK TIERS what this host runs for the remote per tier
|
||||
# TIER DELAYS delays before each tier activates
|
||||
# RSYNC WRITEBACK appdata synced back on handback
|
||||
#
|
||||
# ── DOCKER ─────────────────────────────────────────────────────────────────────────────────
|
||||
# DOCKER DAILY RESTART containers restarted daily
|
||||
# DOCKER WEEKLY RESTART containers restarted weekly
|
||||
# DOCKER WATCHDOG memory limits, health URLs, required containers
|
||||
# NETWORK WATCHDOG DDNS domain, NPM URL for connectivity checks
|
||||
# DOCKER NETWORK CONNECT networks and containers for array start
|
||||
#
|
||||
# ── MEDIA ──────────────────────────────────────────────────────────────────────────────────
|
||||
# MEDIA PERMISSIONS share list for permissions script
|
||||
# MEDIA CLEANER folder lists for media_cleaner.sh
|
||||
#
|
||||
# ── ARR STACK ──────────────────────────────────────────────────────────────────────────────
|
||||
# DOWNLOADERS slskd, SABnzbd, qBittorrent credentials and URLs
|
||||
# LIDARR / SONARR / RADARR URL, API key, path map
|
||||
# ARR RECOVERY per-arr recovery toggles
|
||||
#
|
||||
# ── TRANSCODES ─────────────────────────────────────────────────────────────────────────────
|
||||
# TRANSCODES ramdisk size, thresholds, SSD path, server array
|
||||
#
|
||||
# ── MONITORS ───────────────────────────────────────────────────────────────────────────────
|
||||
# CERTIFICATE MONITOR domains checked for SSL expiry
|
||||
# SMART HEALTH drives to skip in SMART monitoring
|
||||
# ZFS REPORT pools to exclude from ZFS health report
|
||||
#
|
||||
# ── RESOURCE MANAGER ───────────────────────────────────────────────────────────────────────
|
||||
# RESOURCE MANAGER containers paused/stopped under memory pressure
|
||||
#
|
||||
# ── SYSTEM WATCHDOG ────────────────────────────────────────────────────────────────────────
|
||||
# SYSTEM WATCHDOG per-host check toggles and NIC configuration
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
# ==============================================================================================
|
||||
# ── STORAGE MODE ──────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Storage mode ━━━
|
||||
# Controls where Varaverk stores scripts, conf, and state files.
|
||||
# true = internal NVMe/SSD — /boot/config/plugins/varaverk (write-safe, git-direct)
|
||||
# false = USB flash boot — /mnt/user/appdata/Varaverk (preserves flash lifetime)
|
||||
# Auto-detected from boot device transport on first setup.
|
||||
# To change: Settings → Storage → Migrate.
|
||||
HOSTN_STORAGE_MODE_INTERNAL=true
|
||||
|
||||
# ==============================================================================================
|
||||
# ── IDENTITY & CONNECTIVITY ───────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Identity ━━━
|
||||
# HOSTN hostname lives in master.conf (not a credential — safe for all servers).
|
||||
# SSH key used for all server-to-server operations — rsync, failover, conf sync.
|
||||
# Convention: /root/.ssh/<hostname-lowercase-no-unraid-prefix>_rsync_automation
|
||||
# Must be in /root/.ssh/ and authorised in the partner's /root/.ssh/authorized_keys.
|
||||
# Run Partnership/ssh_setup.sh to generate the key and copy it to the partner.
|
||||
HOSTN_SSH_KEY="" # e.g. /root/.ssh/myserver_rsync_automation
|
||||
HOSTN_OWNER="" # short identifier for this server (e.g. myserver)
|
||||
HOSTN_OWNER_EMAIL=""
|
||||
|
||||
# ━━━ Unraid API ━━━
|
||||
# Used by the Varaverk plugin to query this server's Unraid GraphQL API.
|
||||
# Generate in Unraid: Settings → Management Access → API Keys → + New Key
|
||||
HOSTN_UNRAID_API_KEY=""
|
||||
|
||||
# ━━━ Emby ━━━
|
||||
HOSTN_EMBY_CONTAINER="Emby"
|
||||
HOSTN_EMBY_URL="http://localhost:8096"
|
||||
HOSTN_EMBY_API_KEY="" # Emby Dashboard → API Keys → + New Key
|
||||
|
||||
# ━━━ Jellyfin ━━━
|
||||
HOSTN_JELLYFIN_CONTAINER="Jellyfin"
|
||||
HOSTN_JELLYFIN_URL="http://localhost:8095"
|
||||
HOSTN_JELLYFIN_API_KEY="" # Jellyfin Dashboard → Administration → API Keys
|
||||
|
||||
# ━━━ Gitea ━━━
|
||||
# Personal access token for gitea_ssh_setup.sh.
|
||||
# Create in Gitea: Settings → Applications → Generate Token → scope: write:user
|
||||
HOSTN_GITEA_API_TOKEN=""
|
||||
|
||||
# ━━━ Notifications ━━━
|
||||
# Discord webhook — leave blank to disable.
|
||||
HOSTN_DISCORD_WEBHOOK=""
|
||||
|
||||
# ==============================================================================================
|
||||
# ── PARTNERSHIP ───────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# Auth containers reconfigured on onboard/offboard.
|
||||
# Format: "ContainerName|WebUIPort"
|
||||
HOSTN_PARTNERSHIP_AUTH_WEBUIS=(
|
||||
# "NginxProxyManager|81"
|
||||
# "Authelia|9091"
|
||||
)
|
||||
|
||||
# XML templates pushed to mirror during onboard — auth stack.
|
||||
# Dependencies (databases) must come before apps that depend on them.
|
||||
HOSTN_PARTNERSHIP_AUTH_STACK=(
|
||||
# "my-Authelia.xml"
|
||||
# "my-NginxProxyManager.xml"
|
||||
)
|
||||
|
||||
# XML templates pushed to mirror during onboard — arr stack.
|
||||
HOSTN_PARTNERSHIP_ARR_STACK=(
|
||||
# "my-Sonarr.xml"
|
||||
# "my-Radarr.xml"
|
||||
)
|
||||
|
||||
# Paths the partner should collect during the grace window after offboard.
|
||||
HOSTN_PARTNERSHIP_MIRROR_BACKUPS=(
|
||||
# "/mnt/user/appdata-Fallback/Partner-Emby"
|
||||
)
|
||||
|
||||
# Containers parked on this server when partnership is active.
|
||||
HOSTN_PARTNERSHIP_OWN_CONTAINERS=(
|
||||
# "Emby"
|
||||
)
|
||||
|
||||
# Containers stopped on THIS server before deploying the mirror's stack on onboard.
|
||||
HOSTN_PARTNERSHIP_REPLACE_CONTAINERS=(
|
||||
)
|
||||
|
||||
# Arr containers stopped on this server when mirror's arr stack is deployed.
|
||||
HOSTN_PARTNERSHIP_ARR_REPLACE_CONTAINERS=(
|
||||
)
|
||||
|
||||
# Emby admin provisioning — owner controls whether Emby is shared.
|
||||
HOSTN_PARTNERSHIP_PROVISION_EMBY_ADMIN=false
|
||||
HOSTN_PARTNERSHIP_EMBY_PORT=8096
|
||||
HOSTN_PARTNERSHIP_EMBY_ADMIN_USER=""
|
||||
HOSTN_PARTNERSHIP_EMBY_ADMIN_PASS=""
|
||||
|
||||
# ==============================================================================================
|
||||
# ── RSYNC ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Daily Sync Shares ━━━
|
||||
# Media shares this host pushes to all other nodes every night.
|
||||
# Uses DEFAULT_RSYNC_OPTS from master.conf — no profile needed.
|
||||
HOSTN_DAILY_SYNC_SHARES=(
|
||||
# /mnt/user/Movies
|
||||
# /mnt/user/Tv_Shows
|
||||
# /mnt/user/Music
|
||||
)
|
||||
|
||||
# ━━━ Personal Shares ━━━
|
||||
# Private encrypted shares synced for offsite backup, independent of media shares.
|
||||
HOSTN_PERSONAL_SHARES=(
|
||||
# /mnt/user/Personal # e.g. ZFS-encrypted dataset
|
||||
)
|
||||
|
||||
# ━━━ Weekly Sync Shares ━━━
|
||||
# Appdata shares synced during the weekly maintenance window.
|
||||
# Profiles (emby, critical-data) drive container stops — define in master.conf.
|
||||
HOSTN_WEEKLY_SYNC_SHARES=(
|
||||
# "/mnt/user/Media_Server/Emby" # emby profile
|
||||
# "/mnt/user/appdata-Fallback/Critical-Data" # critical-data profile
|
||||
)
|
||||
|
||||
# ━━━ Intermediate Sync Shares ━━━
|
||||
# Shares synced every 4 hours. Leave empty to skip mid-day rsync.
|
||||
HOSTN_INTERMEDIATE_SYNC_SHARES=(
|
||||
# Add shares here to enable mid-day rsync
|
||||
)
|
||||
|
||||
# ━━━ Critical Sync Shares ━━━
|
||||
# Appdata shares synced every 30 minutes.
|
||||
# Format: "/path/to/share" or "/path/to/share|profile-name"
|
||||
HOSTN_CRITICAL_SYNC_SHARES=(
|
||||
# "/mnt/user/appdata-Fallback/Critical-Data|critical-fallback"
|
||||
# "/mnt/user/Media_Server/Emby|emby-fallback"
|
||||
)
|
||||
|
||||
# ━━━ Backup Verify ━━━
|
||||
# Leave empty to use HOSTN_DAILY_SYNC_SHARES automatically.
|
||||
HOSTN_BACKUP_VERIFY_SHARES=(
|
||||
# leave empty to use HOSTN_DAILY_SYNC_SHARES automatically
|
||||
)
|
||||
|
||||
# ━━━ HOSTN Rsync Profile — hostn-appdata ━━━
|
||||
# Host-specific appdata sync profile.
|
||||
PROFILE_RSYNC_OPTS[hostn-appdata]="-av --info=progress2 --bwlimit=${PROFILE_BW_LIMIT[hostn-appdata]:-8000}"
|
||||
PROFILE_BW_LIMIT[hostn-appdata]=8000
|
||||
PROFILE_RETRY_COUNT[hostn-appdata]=3
|
||||
PROFILE_SLEEP[hostn-appdata]=300
|
||||
PROFILE_CRITICAL_CONTAINER_NAMES[hostn-appdata]=""
|
||||
PROFILE_DELAYED_CONTAINERS[hostn-appdata]=""
|
||||
PROFILE_CONTAINER_DELAY[hostn-appdata]=5
|
||||
PROFILE_EXCLUDE_DIRS[hostn-appdata]="logs *.tmp"
|
||||
|
||||
# ==============================================================================================
|
||||
# ── FALLBACK ──────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ DDNS ━━━
|
||||
# DDNS containers this host manages.
|
||||
HOSTN_DDNS_CONTAINERS=(
|
||||
# "MyServer.com"
|
||||
)
|
||||
|
||||
# ━━━ Internet Loss ━━━
|
||||
# Containers stopped immediately when internet is lost.
|
||||
FALLBACK_HOSTN_STOP_ON_NO_NET=(
|
||||
# "MyServer.com"
|
||||
)
|
||||
|
||||
# ━━━ Fallback Tiers — HOSTN Runs for Partner ━━━
|
||||
# Containers this host starts when the partner goes down.
|
||||
# Replace REMOTE_ID below with the actual remote host ID (HOST1, HOST2, etc.)
|
||||
FALLBACK_HOSTN_COVERS_REMOTE_ID_TIER1=(
|
||||
# "Partner-DDNS-Container"
|
||||
)
|
||||
|
||||
FALLBACK_HOSTN_COVERS_REMOTE_ID_TIER2=(
|
||||
# "container-placeholder"
|
||||
)
|
||||
|
||||
FALLBACK_HOSTN_COVERS_REMOTE_ID_TIER3=(
|
||||
# "container-placeholder"
|
||||
)
|
||||
|
||||
FALLBACK_HOSTN_COVERS_REMOTE_ID_TIER4=(
|
||||
# "container-placeholder"
|
||||
)
|
||||
|
||||
# ━━━ Tier Delays — This Host's Outage Timers ━━━
|
||||
# How long THIS host must be down before each tier activates on the partner.
|
||||
HOSTN_TIER2_DELAY=240 # 4 hours
|
||||
HOSTN_TIER3_DELAY=720 # 12 hours
|
||||
HOSTN_TIER4_DELAY=1440 # 24 hours
|
||||
|
||||
# ━━━ Rsync Writeback ━━━
|
||||
HOSTN_TIER1_WRITEBACK_DELAY=60 # skip Emby writeback if outage under 1hr
|
||||
|
||||
FALLBACK_HOSTN_WRITEBACK_TIER1=(
|
||||
# "/mnt/user/Media_Server/Emby"
|
||||
)
|
||||
|
||||
FALLBACK_HOSTN_WRITEBACK_TIER2=(
|
||||
# "/mnt/user/appdata-Fallback/Important-Data"
|
||||
)
|
||||
|
||||
FALLBACK_HOSTN_WRITEBACK_TIER3=(
|
||||
# "location-placeholder"
|
||||
)
|
||||
|
||||
FALLBACK_HOSTN_WRITEBACK_TIER4=(
|
||||
# "/mnt/user/appdata-Fallback/Arrs_Stack"
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── DOCKER ────────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Docker Daily Restart ━━━
|
||||
HOSTN_DAILY_RESTART_CONTAINERS=(
|
||||
# "NginxProxyManager"
|
||||
# "Authelia"
|
||||
)
|
||||
|
||||
# ━━━ Docker Weekly Restart ━━━
|
||||
HOSTN_WEEKLY_RESTART_CONTAINERS=(
|
||||
# "NextCloud"
|
||||
# "AdGuard-Home"
|
||||
)
|
||||
|
||||
# ━━━ Docker Watchdog ━━━
|
||||
|
||||
# Memory hard limits in MB — immediate restart if exceeded.
|
||||
# 20GB=20480 16GB=16384 8GB=8192 4GB=4096 2GB=2048 1GB=1024
|
||||
declare -A HOSTN_WATCHDOG_CONTAINERS=(
|
||||
# ["Emby"]=18432
|
||||
)
|
||||
|
||||
# HTTP health check URLs — checked every cycle.
|
||||
declare -A HOSTN_WATCHDOG_CONTAINER_URLS=(
|
||||
# ["Emby"]="http://localhost:8096"
|
||||
)
|
||||
|
||||
# API-level health checks. Format: ["ContainerName"]="url|expected_json_key|expected_value"
|
||||
declare -A HOSTN_WATCHDOG_CONTAINER_API_CHECKS=(
|
||||
)
|
||||
|
||||
# Required containers — must always be running.
|
||||
HOSTN_WATCHDOG_REQUIRED_CONTAINERS=(
|
||||
# "NginxProxyManager"
|
||||
# "Authelia"
|
||||
)
|
||||
|
||||
# Containers to skip in Tier 2 global scan.
|
||||
HOSTN_WATCHDOG_SCAN_IGNORE=(
|
||||
# "my-occasional-container"
|
||||
)
|
||||
|
||||
# Dependency ordering — skip restarting a container if its dependency is also down.
|
||||
declare -A HOSTN_WATCHDOG_DEPENDENCIES=(
|
||||
# ["Authelia"]="Mariadb Redis-Authelia"
|
||||
)
|
||||
|
||||
# Per-container appdata growth suppress ceilings in MB.
|
||||
declare -A HOSTN_WATCHDOG_APPDATA_SIZES=(
|
||||
# ["Tdarr"]="25600"
|
||||
)
|
||||
|
||||
# ━━━ Network Watchdog ━━━
|
||||
HOSTN_NETWORK_WATCHDOG_DDNS_DOMAIN="" # e.g. myserver.com
|
||||
HOSTN_NETWORK_WATCHDOG_DDNS_CONTAINER="" # e.g. MyServer.com
|
||||
HOSTN_NETWORK_WATCHDOG_NPM_URL="" # e.g. https://myserver.com
|
||||
|
||||
# ━━━ Docker Network Connect ━━━
|
||||
HOSTN_NETWORK_CONNECT_CONTAINERS=(
|
||||
# "memcached"
|
||||
)
|
||||
|
||||
HOSTN_NETWORK_CONNECT_NETWORKS=(
|
||||
# "high-availability"
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── MEDIA ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Media Permissions ━━━
|
||||
HOSTN_MEDIA_PERMISSION_SHARES=(
|
||||
# /mnt/user/Movies
|
||||
# /mnt/user/Tv_Shows
|
||||
# /mnt/user/Music
|
||||
# /mnt/user/Downloads
|
||||
)
|
||||
|
||||
# ━━━ Media Cleaner ━━━
|
||||
HOSTN_ANIME_CLEAN_FOLDERS=(
|
||||
# /mnt/user/Anime_Movies
|
||||
# /mnt/user/Anime_Shows
|
||||
)
|
||||
|
||||
HOSTN_MEDIA_CLEAN_FOLDERS=(
|
||||
# /mnt/user/Movies
|
||||
# /mnt/user/Tv_Shows
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── ARR STACK ─────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Downloaders ━━━
|
||||
HOSTN_SLSKD_URL="http://localhost:8980"
|
||||
HOSTN_SLSKD_API_KEY=""
|
||||
HOSTN_SLSKD_FAILED_IMPORTS_DIR=""
|
||||
|
||||
HOSTN_SABNZBD_URL="http://localhost:8180"
|
||||
HOSTN_SABNZBD_API_KEY=""
|
||||
|
||||
HOSTN_QBIT_URL="http://localhost:8080"
|
||||
HOSTN_QBIT_USERNAME="admin"
|
||||
HOSTN_QBIT_PASSWORD=""
|
||||
|
||||
# ━━━ Lidarr ━━━
|
||||
HOSTN_LIDARR_URL="http://localhost:8686"
|
||||
HOSTN_LIDARR_API_KEY=""
|
||||
HOSTN_LIDARR_MUSIC_ROOT="/mnt/user/Music"
|
||||
HOSTN_FANART_API_KEY=""
|
||||
HOSTN_LASTFM_API_KEY=""
|
||||
|
||||
declare -A HOSTN_LIDARR_PATH_MAP=(
|
||||
# ["/music"]="/mnt/user/Music"
|
||||
)
|
||||
|
||||
# ━━━ Sonarr ━━━
|
||||
HOSTN_SONARR_URL="http://localhost:8989"
|
||||
HOSTN_SONARR_API_KEY=""
|
||||
HOSTN_SONARR_TV_ROOT="/mnt/user/Tv_Shows"
|
||||
|
||||
declare -A HOSTN_SONARR_PATH_MAP=(
|
||||
# ["/tv"]="/mnt/user/Tv_Shows"
|
||||
)
|
||||
|
||||
# ━━━ Radarr ━━━
|
||||
HOSTN_RADARR_URL="http://localhost:7878"
|
||||
HOSTN_RADARR_API_KEY=""
|
||||
HOSTN_TMDB_API_KEY=""
|
||||
HOSTN_RADARR_MOVIES_ROOT="/mnt/user/Movies"
|
||||
|
||||
declare -A HOSTN_RADARR_PATH_MAP=(
|
||||
# ["/movies"]="/mnt/user/Movies"
|
||||
)
|
||||
|
||||
# ━━━ Arr Recovery Toggles ━━━
|
||||
HOSTN_LIDARR_RECOVERY=false
|
||||
HOSTN_SONARR_RECOVERY=true
|
||||
HOSTN_RADARR_RECOVERY=true
|
||||
|
||||
# ==============================================================================================
|
||||
# ── TRANSCODES ────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
HOSTN_RAMDISK_SIZE="10G"
|
||||
HOSTN_RAMDISK_WARN_GB=8.5
|
||||
HOSTN_RAMDISK_LOW_GB=7
|
||||
HOSTN_TRANSCODE_SSD="/mnt/cache/Temp_Storage/Emby/Transcodes/"
|
||||
|
||||
HOSTN_TRANSCODE_SERVERS=(
|
||||
"${HOSTN_EMBY_CONTAINER}|${HOSTN_EMBY_URL}|${HOSTN_EMBY_API_KEY}|emby"
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── MONITORS ──────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Certificate Monitor ━━━
|
||||
HOSTN_CERT_MONITOR_DOMAINS=(
|
||||
# "myserver.com"
|
||||
)
|
||||
|
||||
# ━━━ SMART Health ━━━
|
||||
HOSTN_SMART_IGNORE_DRIVES=(
|
||||
"sda" # boot USB — SMART not meaningful on flash drives
|
||||
)
|
||||
|
||||
# ━━━ ZFS Report ━━━
|
||||
HOSTN_ZFS_REPORT_IGNORE_POOLS=(
|
||||
# "disk5"
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── RESOURCE MANAGER ──────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
HOSTN_RW_PAUSE_CONTAINERS=(
|
||||
# "Tdarr"
|
||||
# "LidaTube"
|
||||
)
|
||||
|
||||
HOSTN_RW_STOP_CONTAINERS=(
|
||||
# "Tdarr"
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── SYSTEM WATCHDOG ───────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
HOSTN_SYS_WATCHDOG_NIC="" # e.g. eth0 — for network monitoring
|
||||
|
||||
HOSTN_SYS_WATCHDOG_CHECK_DOCKER_DAEMON=true
|
||||
HOSTN_SYS_WATCHDOG_CHECK_ROOTFS=true
|
||||
HOSTN_SYS_WATCHDOG_CHECK_KERNEL_OOPS=true
|
||||
HOSTN_SYS_WATCHDOG_CHECK_FD=true
|
||||
HOSTN_SYS_WATCHDOG_CHECK_BOOT=true
|
||||
HOSTN_SYS_WATCHDOG_CHECK_OOM=true
|
||||
HOSTN_SYS_WATCHDOG_CHECK_RAM=true
|
||||
HOSTN_SYS_WATCHDOG_CHECK_LOG=true
|
||||
HOSTN_SYS_WATCHDOG_CHECK_ARC=true
|
||||
HOSTN_SYS_WATCHDOG_CHECK_CPU_TEMP=true
|
||||
HOSTN_SYS_WATCHDOG_CHECK_LOAD=true
|
||||
HOSTN_SYS_WATCHDOG_CHECK_ZOMBIES=true
|
||||
HOSTN_SYS_WATCHDOG_CHECK_CONTAINERS=true
|
||||
HOSTN_SYS_WATCHDOG_CHECK_TMP=true
|
||||
HOSTN_SYS_WATCHDOG_CHECK_MDSTAT=true
|
||||
HOSTN_SYS_WATCHDOG_CHECK_NETWORK=true
|
||||
HOSTN_SYS_WATCHDOG_CHECK_SSHD=true
|
||||
HOSTN_SYS_WATCHDOG_CHECK_RUNAWAY=false
|
||||
@@ -0,0 +1,782 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ========================== HOST1 CONFIGURATION — unRAID-Gmer4Lfe ============================
|
||||
# ==============================================================================================
|
||||
# HOST1-specific variables — credentials, container names, share paths, failover lists.
|
||||
# Sourced after master.conf — values here extend shared profile arrays and add HOST1-specific
|
||||
# identity, credentials, and container configuration.
|
||||
#
|
||||
# Sparse checkout (git) ensures HOST2 never receives this file.
|
||||
# HOST2 never sees HOST1 credentials — clean separation at the file level.
|
||||
#
|
||||
# DO NOT put shared config here — thresholds, toggles, profiles belong in master.conf.
|
||||
# DO NOT put HOST2 variables here — they belong in host2.conf.
|
||||
#
|
||||
# ── INDEX ─────────────────────────────────────────────────────────────────────────────────────
|
||||
#
|
||||
# ── IDENTITY & CONNECTIVITY ────────────────────────────────────────────────────────────────
|
||||
# IDENTITY hostname, SSH key, Unraid API key
|
||||
# EMBY container name, URL, API key
|
||||
# JELLYFIN container name, URL, API key
|
||||
# GITEA API token for SSH key registration
|
||||
# NOTIFICATIONS Discord webhook
|
||||
#
|
||||
# ── PARTNERSHIP ────────────────────────────────────────────────────────────────────────────
|
||||
# PARTNERSHIP auth containers, backup paths, emby provisioning
|
||||
#
|
||||
# ── RSYNC ──────────────────────────────────────────────────────────────────────────────────
|
||||
# DAILY SYNC SHARES media shares HOST1 owns and pushes to HOST2
|
||||
# WEEKLY SYNC SHARES appdata shares synced weekly (Sunday 2:30am)
|
||||
# CRITICAL SYNC SHARES appdata shares synced every 30 minutes
|
||||
# BACKUP VERIFY shares for checksum verification against remote
|
||||
# HOST1 RSYNC PROFILE host1-appdata profile for HOST1-specific appdata syncs
|
||||
#
|
||||
# ── FALLBACK ───────────────────────────────────────────────────────────────────────────────
|
||||
# DDNS DDNS containers managed by HOST1
|
||||
# INTERNET LOSS containers stopped when internet is lost
|
||||
# FALLBACK TIERS what HOST1 runs for HOST2 per tier
|
||||
# TIER DELAYS how long HOST1 must be down before each tier activates on HOST2
|
||||
# RSYNC WRITEBACK HOST1 appdata synced back on handback
|
||||
#
|
||||
# ── DOCKER ─────────────────────────────────────────────────────────────────────────────────
|
||||
# DOCKER DAILY RESTART containers restarted daily
|
||||
# DOCKER WEEKLY RESTART containers restarted weekly
|
||||
# DOCKER WATCHDOG memory limits, health URLs, required containers, ignore list
|
||||
# NETWORK WATCHDOG DDNS domain, NPM URL for connectivity checks
|
||||
# DOCKER NETWORK CONNECT networks and containers for docker_network_connect.sh
|
||||
#
|
||||
# ── MEDIA ──────────────────────────────────────────────────────────────────────────────────
|
||||
# MEDIA PERMISSIONS share list for media_shares_permissions.sh
|
||||
# MEDIA CLEANER folder lists for media_cleaner.sh
|
||||
#
|
||||
# ── ARR STACK ──────────────────────────────────────────────────────────────────────────────
|
||||
# DOWNLOADERS slskd, SABnzbd, qBittorrent credentials and URLs
|
||||
# LIDARR URL, API key, path map
|
||||
# SONARR URL, API key, path map
|
||||
# RADARR URL, API key, path map
|
||||
# ARR RECOVERY per-arr recovery toggles
|
||||
#
|
||||
# ── TRANSCODES ─────────────────────────────────────────────────────────────────────────────
|
||||
# TRANSCODES ramdisk size, thresholds, SSD path, server array
|
||||
#
|
||||
# ── MONITORS ───────────────────────────────────────────────────────────────────────────────
|
||||
# CERTIFICATE MONITOR domains checked for SSL expiry
|
||||
# SMART HEALTH drives to skip in SMART monitoring
|
||||
# ZFS REPORT pools to exclude from ZFS health report
|
||||
#
|
||||
# ── RESOURCE MANAGER ───────────────────────────────────────────────────────────────────────
|
||||
# RESOURCE MANAGER containers paused/stopped under memory pressure
|
||||
#
|
||||
# ── SYSTEM WATCHDOG ────────────────────────────────────────────────────────────────────────
|
||||
# SYSTEM WATCHDOG per-host check toggles and NIC configuration
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
# ==============================================================================================
|
||||
# ── IDENTITY & CONNECTIVITY ───────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Identity ━━━
|
||||
# HOST1 hostname lives in master.conf (not a credential — safe for all servers).
|
||||
# SSH key used for all server-to-server operations — rsync, failover container commands.
|
||||
# Must be in /root/.ssh/ and authorised in HOST2's /root/.ssh/authorized_keys.
|
||||
HOST1_SSH_KEY="/root/.ssh/gmer4lfe_rsync_automation"
|
||||
HOST1_OWNER="gmer4lfe"
|
||||
HOST1_OWNER_EMAIL="gmer4lfe@gmail.com"
|
||||
|
||||
# ━━━ Unraid API ━━━
|
||||
# Used by the Varaverk plugin to query this server's Unraid GraphQL API.
|
||||
# Generate in Unraid: Settings → Management Access → API Keys → + New Key
|
||||
HOST1_UNRAID_API_KEY="1825c3a2e03ea5089974f4da2e171aa2d5907a1dea23cc479c33e492c8ff4dbb"
|
||||
|
||||
# ━━━ Emby ━━━
|
||||
# Referenced by transcode_manager.sh, emby_session_report.sh, emby_database_repair.sh,
|
||||
# weekly_sync_maintenance.sh, and HOST1_TRANSCODE_SERVERS below.
|
||||
# API key: Emby Dashboard → API Keys → + New Key
|
||||
HOST1_EMBY_CONTAINER="Emby"
|
||||
HOST1_EMBY_URL="http://localhost:8096"
|
||||
HOST1_EMBY_API_KEY="0c27448d93a7431f9ac63569f7655829"
|
||||
|
||||
# ━━━ Jellyfin ━━━
|
||||
# API key: Jellyfin Dashboard → Administration → API Keys → + New Key
|
||||
HOST1_JELLYFIN_CONTAINER="Jellyfin"
|
||||
HOST1_JELLYFIN_URL="http://localhost:8095"
|
||||
HOST1_JELLYFIN_API_KEY="4e820e7df74c4933acec212b1996314e"
|
||||
|
||||
# ━━━ Gitea ━━━
|
||||
# Personal access token for gitea_ssh_setup.sh — registers this server's SSH public key
|
||||
# with Gitea so git operations use key auth instead of passwords.
|
||||
# Create in Gitea: Settings → Applications → Generate Token → scope: write:user
|
||||
HOST1_GITEA_API_TOKEN=""
|
||||
|
||||
# ━━━ Notifications ━━━
|
||||
# Discord webhook — leave blank to disable.
|
||||
# Per-host so HOST1 and HOST2 can post to different channels or only one server notifies.
|
||||
HOST1_DISCORD_WEBHOOK=""
|
||||
|
||||
# ==============================================================================================
|
||||
# ── PARTNERSHIP ───────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# HOST1 is always the owner (source of truth) unless --transfer has been run.
|
||||
# See README-Partnership.md and master.conf PARTNERSHIP section for full lifecycle docs.
|
||||
|
||||
# Auth containers reconfigured on onboard/offboard.
|
||||
# Format: "ContainerName|WebUIPort"
|
||||
# On onboard → WebUI pointed at owner's Tailscale IP (mirror clicks NPM, gets owner's auth)
|
||||
# On offboard → WebUI pointed back at localhost
|
||||
HOST1_PARTNERSHIP_AUTH_WEBUIS=(
|
||||
"NginxProxyManager|81"
|
||||
"Lldap-Gmer4Lfe|17170"
|
||||
"Authelia|9091"
|
||||
"Authelia-Secondary|9092"
|
||||
)
|
||||
|
||||
# XML templates (from this server's templates-user/) pushed to mirror during onboard.
|
||||
# These become the mirror's active auth stack, backed by the rsync-synced appdata.
|
||||
# Update filename if Lldap is renamed to drop the host suffix.
|
||||
HOST1_PARTNERSHIP_AUTH_STACK=(
|
||||
# Dependencies first — Mariadb/Redis must be healthy before Authelia starts
|
||||
"my-Mariadb-Authelia.xml"
|
||||
"my-Mariadb-Authelia-Secondary.xml"
|
||||
"my-Redis-Authelia.xml"
|
||||
"my-Redis-Authelia-Secondary.xml"
|
||||
# Auth apps — deployed after their deps are confirmed healthy
|
||||
"my-Authelia.xml"
|
||||
"my-Authelia-Secondary.xml"
|
||||
"my-NginxProxyManager.xml"
|
||||
"my-Lldap-Gmer4Lfe.xml"
|
||||
# Source of truth — must be available on HOST2 independently of the auth stack
|
||||
"my-Gitea.xml"
|
||||
)
|
||||
|
||||
# XML templates pushed to mirror for the arr stack during onboard.
|
||||
# Deps (e.g. databases) first if any — same ordering rule as auth stack.
|
||||
HOST1_PARTNERSHIP_ARR_STACK=(
|
||||
# "my-Sonarr.xml"
|
||||
# "my-Radarr.xml"
|
||||
# "my-Lidarr.xml"
|
||||
# "my-Prowlarr.xml"
|
||||
# "my-Bazarr.xml"
|
||||
)
|
||||
|
||||
# Paths HOST2 should collect during the grace window after offboard.
|
||||
# Notified on offboard — no auto-deletion, HOST2 must collect manually within PARTNERSHIP_GRACE_HOURS.
|
||||
HOST1_PARTNERSHIP_MIRROR_BACKUPS=(
|
||||
# "/mnt/user/appdata-Fallback/Jayred365-Emby"
|
||||
)
|
||||
|
||||
# Containers parked on this server when partnership is active.
|
||||
# Stopped on onboard (owner deploys its stack instead), restarted on offboard.
|
||||
HOST1_PARTNERSHIP_OWN_CONTAINERS=(
|
||||
# "Emby"
|
||||
# "NginxProxyManager"
|
||||
)
|
||||
|
||||
# Emby admin provisioning — toggle is owner-only, credentials are per-host.
|
||||
# Owner enables/disables the feature. Each host sets the account they want on the shared Emby.
|
||||
# On onboard: owner reads mirror's HOST*_PARTNERSHIP_EMBY_ADMIN_* and creates that account.
|
||||
# On offboard: account is deleted. Username collision → onboard exits with error.
|
||||
HOST1_PARTNERSHIP_PROVISION_EMBY_ADMIN=false # owner controls whether Emby is shared
|
||||
HOST1_PARTNERSHIP_EMBY_PORT=8096
|
||||
HOST1_PARTNERSHIP_EMBY_ADMIN_USER="" # this server's desired Emby username
|
||||
HOST1_PARTNERSHIP_EMBY_ADMIN_PASS="" # this server's desired Emby password
|
||||
|
||||
# ==============================================================================================
|
||||
# ── RSYNC ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Daily Sync Shares ━━━
|
||||
# Shares HOST1 pushes to all other nodes every night (1am via daily_sync_maintenance.sh).
|
||||
# Mesh model: every node pushes every media share — no ownership, no mirrors.
|
||||
# arr_sync ensures all arr libraries converge (union). rsync spreads files (additive, no --delete).
|
||||
# arr_cleanup removes true orphans based on local arr state.
|
||||
# Any node can download content to any share — it propagates to all nodes on the next cycle.
|
||||
# Nextcloud is intentionally one-directional (HOST1→HOST2 offsite backup — not arr-managed).
|
||||
# Uses DEFAULT_RSYNC_OPTS from master.conf — no profile needed.
|
||||
# For shares needing container stops or custom options — add a profile in master.conf.
|
||||
HOST1_DAILY_SYNC_SHARES=(
|
||||
/mnt/user/Books
|
||||
/mnt/user/Intros
|
||||
/mnt/user/Kids_Movies
|
||||
/mnt/user/Kids_Tv_Shows
|
||||
/mnt/user/Movies
|
||||
/mnt/user/Music
|
||||
/mnt/user/Music_Videos
|
||||
/mnt/user/Nextcloud
|
||||
/mnt/user/stand-up_comedy
|
||||
/mnt/user/Sports
|
||||
# /mnt/user/Tv_Shows
|
||||
/mnt/user/Anime_Shows-Old
|
||||
/mnt/user/Anime_Movies-Old
|
||||
/mnt/user/Anime_Movies
|
||||
/mnt/user/Anime_Shows
|
||||
)
|
||||
|
||||
# Personal encrypted shares — synced for offsite backup, independent of media shares.
|
||||
# ZFS encrypted at dataset level — remote receives encrypted blocks, cannot read content.
|
||||
# See README-Rsync_Setup.md for ZFS encryption setup before uncommenting.
|
||||
HOST1_PERSONAL_SHARES=(
|
||||
# /mnt/user/HOST1-Personal # uncomment after creating encrypted dataset
|
||||
)
|
||||
|
||||
# ━━━ Weekly Sync Shares ━━━
|
||||
# Appdata shares synced during the weekly maintenance window (Sunday 2:30am).
|
||||
# Containers stopped both sides before sync — full clean state guaranteed.
|
||||
# Profiles drive container stops, excludes, and options — configured in master.conf RSYNC section.
|
||||
# Order matters — Emby first (larger transfer), then Critical-Data (auth stack).
|
||||
HOST1_WEEKLY_SYNC_SHARES=(
|
||||
"/mnt/user/Media_Server/Emby" # emby profile — full clean mirror
|
||||
"/mnt/user/appdata-Fallback/Critical-Data" # critical-data profile — auth stack
|
||||
)
|
||||
|
||||
# ━━━ Intermediate Sync Shares ━━━
|
||||
# Shares synced every 4 hours by intermediate_sync_maintenance.sh.
|
||||
# Uses DEFAULT_RSYNC_OPTS (no --delete) — for sub-daily propagation of metadata or watch state.
|
||||
# Full media share sync stays in the daily window. Leave empty to skip mid-day rsync entirely.
|
||||
HOST1_INTERMEDIATE_SYNC_SHARES=(
|
||||
# Add shares here to enable mid-day rsync
|
||||
# Example: "/mnt/user/Emby_Metadata"
|
||||
)
|
||||
|
||||
# ━━━ Critical Sync Shares ━━━
|
||||
# Appdata shares synced every 30 minutes by critical_sync_maintenance.sh.
|
||||
# Format: "/path/to/share" or "/path/to/share|profile-name"
|
||||
# Order matters — Critical-Data first (auth stack), then Emby dirty sync.
|
||||
HOST1_CRITICAL_SYNC_SHARES=(
|
||||
"/mnt/user/appdata-Fallback/Critical-Data|critical-fallback" # auth dirty sync — stays running
|
||||
"/mnt/user/Media_Server/Emby|emby-fallback" # Emby dirty sync — stays running
|
||||
)
|
||||
|
||||
# ━━━ Backup Verify ━━━
|
||||
# Shares verified by backup_verify.sh — random file checksum comparison against remote.
|
||||
# Leave empty to use HOST1_DAILY_SYNC_SHARES automatically.
|
||||
# Sample size and minimum file size defined in master.conf.
|
||||
HOST1_BACKUP_VERIFY_SHARES=(
|
||||
# leave empty to use HOST1_DAILY_SYNC_SHARES automatically
|
||||
)
|
||||
|
||||
# ━━━ HOST1 Rsync Profile — host1-appdata ━━━
|
||||
# HOST1-specific appdata sync profile — extends the shared PROFILE_* arrays in master.conf.
|
||||
# Use for appdata unique to HOST1 (Organizrv2, VaultWarden, UptimeKuma etc.)
|
||||
# Shared appdata (auth stack, Emby) use dedicated profiles defined in master.conf.
|
||||
# Run manually: bash Rsync/rsync.sh /mnt/user/appdata-Fallback/HOST1-Appdata --profile=host1-appdata
|
||||
PROFILE_RSYNC_OPTS[host1-appdata]="-av --info=progress2 --bwlimit=${PROFILE_BW_LIMIT[host1-appdata]:-8000}"
|
||||
PROFILE_BW_LIMIT[host1-appdata]=8000
|
||||
PROFILE_RETRY_COUNT[host1-appdata]=3
|
||||
PROFILE_SLEEP[host1-appdata]=300
|
||||
PROFILE_CRITICAL_CONTAINER_NAMES[host1-appdata]="Organizrv2-Gmer4Lfe UptimeKuma-Gmer4Lfe VaultWarden-Gmer4Lfe"
|
||||
PROFILE_DELAYED_CONTAINERS[host1-appdata]=""
|
||||
PROFILE_CONTAINER_DELAY[host1-appdata]=5
|
||||
PROFILE_EXCLUDE_DIRS[host1-appdata]="logs *.tmp"
|
||||
|
||||
# ==============================================================================================
|
||||
# ── FALLBACK ──────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ DDNS ━━━
|
||||
# DDNS containers HOST1 manages — started/stopped by fallback.sh per DDNS absolute rules:
|
||||
# Internet loss → stop immediately
|
||||
# Failover → HOST2 starts HOST1's DDNS as Tier 1 (before any other containers)
|
||||
# Handback → stop HOST1's DDNS on HOST2 → rsync → start containers → start local DDNS last
|
||||
HOST1_DDNS_CONTAINERS=(
|
||||
"Gmer4Lfe.com"
|
||||
)
|
||||
|
||||
# ━━━ Internet Loss ━━━
|
||||
# Containers stopped immediately on HOST1 when internet connection is lost.
|
||||
# Prevents external-facing services from operating without connectivity.
|
||||
FALLBACK_HOST1_STOP_ON_NO_NET=(
|
||||
"Gmer4Lfe.com"
|
||||
)
|
||||
|
||||
# ━━━ Fallback Tiers — HOST1 Runs for HOST2 ━━━
|
||||
# Containers HOST1 starts when HOST2 goes down.
|
||||
# Tier 1 is always immediate — vital services cannot wait.
|
||||
# Higher tiers activate after HOST2_TIER*_DELAY minutes (set in host2.conf).
|
||||
FALLBACK_HOST1_COVERS_HOST2_TIER1=(
|
||||
"Gmer4Lfe.us"
|
||||
"VaultWarden-Jayred365"
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_COVERS_HOST2_TIER2=(
|
||||
# "container-placeholder"
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_COVERS_HOST2_TIER3=(
|
||||
# "container-placeholder"
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_COVERS_HOST2_TIER4=(
|
||||
# "container-placeholder"
|
||||
)
|
||||
|
||||
# ━━━ Tier Delays — HOST1's Containers on HOST2 ━━━
|
||||
# How long HOST1 must be down before each tier activates on HOST2 — in minutes.
|
||||
# Tier 1 is always immediate — no delay var needed.
|
||||
HOST1_TIER2_DELAY=240 # 4 hours — NextCloud, Immich
|
||||
HOST1_TIER3_DELAY=720 # 12 hours — secondary services
|
||||
HOST1_TIER4_DELAY=1440 # 24 hours — arrs + downloaders
|
||||
|
||||
# ━━━ Rsync Writeback — HOST1 Appdata Back on Handback ━━━
|
||||
# Syncs HOST1 appdata BACK to HOST1 when it comes back online after a failover.
|
||||
# Containers stopped before writeback — clean source, no competing writes.
|
||||
#
|
||||
# HOST1_TIER1_WRITEBACK_DELAY: short outages skip Tier 1 writeback — primary state
|
||||
# is more reliable than dirty sync data for brief outages.
|
||||
HOST1_TIER1_WRITEBACK_DELAY=60 # skip Emby writeback if outage under 1hr
|
||||
|
||||
# Tier 4 automatically syncs HOST1_DAILY_SYNC_SHARES — only list paths NOT in that array.
|
||||
FALLBACK_HOST1_WRITEBACK_TIER1=(
|
||||
"/mnt/user/Media_Server/Emby" # watch states built up during outage
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_WRITEBACK_TIER2=(
|
||||
"/mnt/user/appdata-Fallback/Important-Data" # NextCloud + Postgres — files added during outage
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_WRITEBACK_TIER3=(
|
||||
# "location-placeholder"
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_WRITEBACK_TIER4=(
|
||||
"/mnt/user/appdata-Fallback/Arrs_Stack" # arr databases — downloads queued during outage
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── DOCKER ────────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Docker Daily Restart ━━━
|
||||
# Containers restarted every day via DAILY_MAINTENANCE_SCRIPTS.
|
||||
# Dispatcharr degrades over time without restart — daily is intentional, not just housekeeping.
|
||||
# Order matters — auth stack first, then media services.
|
||||
HOST1_DAILY_RESTART_CONTAINERS=(
|
||||
"NginxProxyManager"
|
||||
"Lldap-Gmer4Lfe"
|
||||
"Authelia"
|
||||
"Authelia-Secondary"
|
||||
"Dispatcharr-Iptv-Users"
|
||||
"Dispatcharr" # Live TV scheduler — degrades without daily restart
|
||||
"Dispatcharr-Basic"
|
||||
"ErsatzTV-Emby"
|
||||
"slskd" # Soulseek connection drops after extended uptime; restart refreshes share index
|
||||
)
|
||||
|
||||
# ━━━ Docker Weekly Restart ━━━
|
||||
# Less critical services restarted weekly via WEEKLY_MAINTENANCE_SCRIPTS (Sunday 2:30am).
|
||||
# Containers already stopped for weekly sync — restart adds zero extra downtime.
|
||||
HOST1_WEEKLY_RESTART_CONTAINERS=(
|
||||
"NextCloud"
|
||||
"Organizrv2-Gmer4Lfe"
|
||||
"AdGuard-Home"
|
||||
"Immich-Gmer4Lfe"
|
||||
)
|
||||
|
||||
# ━━━ Docker Watchdog ━━━
|
||||
# Per-HOST1 container configuration for docker_watchdog.sh.
|
||||
# Shared thresholds and toggles live in master.conf.
|
||||
|
||||
# Memory hard limits in MB — immediate restart if exceeded.
|
||||
# Set at "container is clearly broken" not "container is busy".
|
||||
# 20GB=20480 18GB=18432 16GB=16384 12GB=12288 8GB=8192 4GB=4096 2GB=2048 1GB=1024
|
||||
declare -A HOST1_WATCHDOG_CONTAINERS=(
|
||||
["Emby"]=20480 # 20GB — large library + active transcodes
|
||||
["LidaTube"]=6144 # 6GB — memory leak over time
|
||||
["Tdarr"]=6144 # 6GB — encoding is memory intensive
|
||||
["Code-Server"]=1024 # 1GB — should never need more
|
||||
)
|
||||
|
||||
# HTTP health check URLs — checked every cycle, strike system before restart.
|
||||
# Only add containers with a meaningful web interface to check.
|
||||
declare -A HOST1_WATCHDOG_CONTAINER_URLS=(
|
||||
["Emby"]="http://localhost:8096"
|
||||
["NginxProxyManager"]="http://localhost:7818"
|
||||
["Authelia"]="http://localhost:9091/api/health"
|
||||
["Authelia-Secondary"]="http://localhost:9092/api/health"
|
||||
["Lldap-Gmer4Lfe"]="http://localhost:17170"
|
||||
)
|
||||
|
||||
# Required containers — must always be running on HOST1.
|
||||
# Strike system before restart — repeated failures go on skip list, auto-clears on recovery.
|
||||
# Listed in dependency order — dependencies before dependents.
|
||||
HOST1_WATCHDOG_REQUIRED_CONTAINERS=(
|
||||
"NginxProxyManager"
|
||||
"Lldap-Gmer4Lfe"
|
||||
"Mariadb-Authelia"
|
||||
"Mariadb-Authelia-Secondary"
|
||||
"Redis-Authelia"
|
||||
"Redis-Authelia-Secondary"
|
||||
"Authelia"
|
||||
"Authelia-Secondary"
|
||||
)
|
||||
|
||||
# Containers to skip in Tier 2 global scan — legitimately stopped or frequently restarting.
|
||||
# Watchdog leaves these alone entirely — no restart attempts, no crash loop tracking.
|
||||
HOST1_WATCHDOG_SCAN_IGNORE=(
|
||||
"DashGate"
|
||||
"PIA-WG-Config-Generator"
|
||||
"Aperture"
|
||||
"Aperture-Kids"
|
||||
"pgvector-18-Apeture-Kids"
|
||||
"Pgvector18-Aperture"
|
||||
"emby-test" # broken test container (exit 127 — bad image)
|
||||
)
|
||||
|
||||
# Dependency ordering — skip restarting a container if its dependency is also down.
|
||||
# Prevents watchdog from restarting Authelia before Mariadb is back up.
|
||||
# SPACE-SEPARATED STRINGS — converted to array at runtime.
|
||||
declare -A HOST1_WATCHDOG_DEPENDENCIES=(
|
||||
["Authelia"]="Mariadb-Authelia Redis-Authelia"
|
||||
["Authelia-Secondary"]="Mariadb-Authelia Redis-Authelia-Secondary"
|
||||
["NextCloud"]="Postgres-NextCloud"
|
||||
)
|
||||
|
||||
# Per-container appdata growth suppress ceilings in MB.
|
||||
# ONLY needed in specific cases — growth rate detection covers all containers automatically.
|
||||
# Use this when a container legitimately has large stable data and you want to guarantee
|
||||
# it never triggers a false-positive growth alert. Growth warnings are suppressed while the
|
||||
# container's dir stays below this ceiling; above it, warnings resume as normal.
|
||||
# 50GB=51200 25GB=25600 20GB=20480 15GB=15360 10GB=10240 5GB=5120
|
||||
declare -A HOST1_WATCHDOG_APPDATA_SIZES=(
|
||||
["Tdarr"]="25600" # 25GB — transcode cache grows legitimately during active jobs
|
||||
["7dtd"]="20480" # 20GB — game server world data, expected to be large
|
||||
)
|
||||
|
||||
# ━━━ Network Watchdog ━━━
|
||||
# Host-specific connectivity config for Watchdogs/System/network_watchdog.sh.
|
||||
HOST1_NETWORK_WATCHDOG_DDNS_DOMAIN="gmer4lfe.com"
|
||||
HOST1_NETWORK_WATCHDOG_DDNS_CONTAINER="Gmer4Lfe.com"
|
||||
HOST1_NETWORK_WATCHDOG_NPM_URL="https://gmer4lfe.com"
|
||||
|
||||
# ━━━ Docker Network Connect ━━━
|
||||
# Containers connected to custom networks at array start by docker_network_connect.sh.
|
||||
# Networks created if they don't exist — idempotent, safe to re-run.
|
||||
HOST1_NETWORK_CONNECT_CONTAINERS=(
|
||||
"memcached"
|
||||
"Npm-CrowdSec"
|
||||
)
|
||||
|
||||
HOST1_NETWORK_CONNECT_NETWORKS=(
|
||||
"high-availability"
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── MEDIA ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Media Permissions ━━━
|
||||
# Shares that media_shares_permissions.sh applies PERMISSIONS_MODE and PERMISSIONS_OWNER to.
|
||||
# Runs first in DAILY_MAINTENANCE_SCRIPTS — arr cleanup depends on correct ownership.
|
||||
HOST1_MEDIA_PERMISSION_SHARES=(
|
||||
/mnt/user/Anime_Movies
|
||||
/mnt/user/Anime_Movies-Old
|
||||
/mnt/user/Anime_Shows
|
||||
/mnt/user/Anime_Shows-Old
|
||||
/mnt/user/appcache
|
||||
/mnt/user/Books
|
||||
/mnt/user/Downloads
|
||||
/mnt/user/Games
|
||||
/mnt/user/Intros
|
||||
/mnt/user/Kids_Movies
|
||||
/mnt/user/Kids_Tv_Shows
|
||||
/mnt/user/Movie_Recordings
|
||||
/mnt/user/Movies
|
||||
/mnt/user/Music
|
||||
/mnt/user/Music_Videos
|
||||
/mnt/user/Photo
|
||||
/mnt/user/Sports
|
||||
/mnt/user/stand-up_comedy
|
||||
/mnt/user/Tv_Recordings
|
||||
/mnt/user/Tv_Shows
|
||||
/mnt/user/YouTube
|
||||
)
|
||||
|
||||
# ━━━ Media Cleaner ━━━
|
||||
# Folder lists for media_cleaner.sh — two profiles: anime and media.
|
||||
# File patterns shared across all servers — defined in master.conf.
|
||||
# Called via DAILY_MAINTENANCE_SCRIPTS. Run manually: Media/media_cleaner.sh anime|media
|
||||
HOST1_ANIME_CLEAN_FOLDERS=(
|
||||
/mnt/user/Anime_Movies
|
||||
/mnt/user/Anime_Movies-Old
|
||||
/mnt/user/Anime_Shows
|
||||
/mnt/user/Anime_Shows-Old
|
||||
)
|
||||
|
||||
HOST1_MEDIA_CLEAN_FOLDERS=(
|
||||
/mnt/user/Kids_Movies
|
||||
/mnt/user/Kids_Tv_Shows
|
||||
/mnt/user/Movies
|
||||
/mnt/user/Music
|
||||
/mnt/user/Sports
|
||||
/mnt/user/stand-up_comedy
|
||||
/mnt/user/Tv_Shows
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── ARR STACK ─────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Used by arr cleanup scripts and arrs_failed_stalled_recovery.sh.
|
||||
# detect_hosts() selects HOST1 vars when running on HOST1.
|
||||
#
|
||||
# PATH MAPS — container path → host path translation.
|
||||
# Arr stores file paths using container-internal paths — scripts need host paths to scan.
|
||||
# Add one entry per root folder in arr Settings → Media Management → Root Folders.
|
||||
|
||||
# ━━━ Downloaders ━━━
|
||||
# Used by downloaders_reset.sh — runs every 30min via CRITICAL_MAINTENANCE_SCRIPTS.
|
||||
# Clears stuck states, purges old history, prepares each client for a clean cycle.
|
||||
|
||||
# slskd — clears stuck searches, dead transfers, purges expired failed imports.
|
||||
# SLSKD_FAILED_IMPORTS_DIR: where Soularr moves albums Lidarr rejected.
|
||||
HOST1_SLSKD_URL="http://localhost:8980"
|
||||
HOST1_SLSKD_API_KEY="4bF9kL2mNpQrT7vWxYz1A3dEgHjKoRsU"
|
||||
HOST1_SLSKD_FAILED_IMPORTS_DIR="/mnt/user/Temp_Storage/Slskd/completed/failed_imports"
|
||||
|
||||
# SABnzbd
|
||||
HOST1_SABNZBD_URL="http://localhost:8180"
|
||||
HOST1_SABNZBD_API_KEY="8bfefe41d83b4d50883e32859b55ca9a"
|
||||
|
||||
# qBittorrent — deleteFiles=false removes torrent from qBit but leaves files on disk.
|
||||
# Radarr/Sonarr manage actual files independently.
|
||||
HOST1_QBIT_URL="http://localhost:8080"
|
||||
HOST1_QBIT_USERNAME="root"
|
||||
HOST1_QBIT_PASSWORD="Stay0utD!ck"
|
||||
|
||||
# ━━━ Lidarr — HOST1 only ━━━
|
||||
# HOST2 does not run Lidarr — HOST1_LIDARR_RECOVERY flag handles the exit cleanly.
|
||||
HOST1_LIDARR_URL="http://localhost:8686"
|
||||
HOST1_LIDARR_API_KEY="b2977e71ef074bc0a0529d9fcce3b2dc"
|
||||
HOST1_LIDARR_MUSIC_ROOT="/mnt/user/Music-New"
|
||||
HOST1_FANART_API_KEY="Yd7147a43b692df0b364b94dc47efb81"
|
||||
HOST1_LASTFM_API_KEY="be6dc169c33ae263e690c30d18b7491d"
|
||||
|
||||
declare -A HOST1_LIDARR_PATH_MAP=(
|
||||
["/ext-music"]="/mnt/user/Music-New"
|
||||
)
|
||||
|
||||
# ━━━ Sonarr ━━━
|
||||
HOST1_SONARR_URL="http://localhost:8989"
|
||||
HOST1_SONARR_API_KEY="130decd3db5b4c25afad64864cd03f9f"
|
||||
HOST1_SONARR_TV_ROOT="/mnt/user/Tv_Shows"
|
||||
|
||||
# Note: stand-up_comedy in both Sonarr + Radarr — TV specials and movie specials, one folder
|
||||
declare -A HOST1_SONARR_PATH_MAP=(
|
||||
["/tv"]="/mnt/user/Tv_Shows"
|
||||
["/ext-standup-comedy"]="/mnt/user/stand-up_comedy/series"
|
||||
["/kids tv"]="/mnt/user/Kids_Tv_Shows"
|
||||
["/ext-anime-shows"]="/mnt/user/Anime_Shows-Old"
|
||||
)
|
||||
|
||||
# ━━━ Radarr ━━━
|
||||
HOST1_RADARR_URL="http://localhost:7878"
|
||||
HOST1_RADARR_API_KEY="d43a3ec6cf1549edb4af0cc63f98b2a9"
|
||||
HOST1_TMDB_API_KEY="3dac5e2e49b5540472d2eafec4f01260"
|
||||
HOST1_RADARR_MOVIES_ROOT="/mnt/user/Movies"
|
||||
|
||||
# Note: stand-up_comedy in both Radarr + Sonarr — movie specials and TV specials, one folder
|
||||
declare -A HOST1_RADARR_PATH_MAP=(
|
||||
["/movies"]="/mnt/user/Movies"
|
||||
["/kids movies"]="/mnt/user/Kids_Movies"
|
||||
["/ext-stand-up-comedy"]="/mnt/user/stand-up_comedy/specials"
|
||||
["/anime-movies"]="/mnt/user/Anime_Movies-Old"
|
||||
["/ext-anime-movies"]="/mnt/user/Anime_Movies-Old"
|
||||
)
|
||||
|
||||
# ━━━ Arr Recovery Toggles ━━━
|
||||
# false = skip that arr on this host — exits cleanly without error
|
||||
HOST1_SONARR_RECOVERY=true
|
||||
HOST1_RADARR_RECOVERY=true
|
||||
HOST1_LIDARR_RECOVERY=true # HOST1 only — exits cleanly on HOST2
|
||||
|
||||
# ==============================================================================================
|
||||
# ── TRANSCODES ────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# Ramdisk size ceiling — tmpfs only uses RAM actually needed, not the full size upfront.
|
||||
# Real-world: 9 streams peaked at ~5.5GB — 10G gives generous headroom on 128GB RAM.
|
||||
HOST1_RAMDISK_SIZE="10G"
|
||||
|
||||
# Usage thresholds — coupled to HOST1_RAMDISK_SIZE, adjust all three together if size changes.
|
||||
# Hysteresis gap (8.5 - 7 = 1.5GB) prevents flip-flop between ramdisk and SSD.
|
||||
HOST1_RAMDISK_WARN_GB=8.5 # flip to SSD when ramdisk usage reaches this
|
||||
HOST1_RAMDISK_LOW_GB=7 # flip back to ramdisk when usage drops to this
|
||||
|
||||
# SSD fallback path — where transcodes land when ramdisk exceeds HOST1_RAMDISK_WARN_GB.
|
||||
# Must be on cache pool — array disks too slow for active transcode writes.
|
||||
HOST1_TRANSCODE_SSD="/mnt/cache/Temp_Storage/Emby/Transcodes/"
|
||||
|
||||
# Media servers sharing the ramdisk transcode space on HOST1.
|
||||
# Format: "ContainerName|URL|APIKey|Type" — Type: emby | jellyfin | plex
|
||||
# Entries with placeholder API keys are skipped automatically.
|
||||
# ⚠️ Tdarr does NOT belong here — keep Tdarr on SSD, not ramdisk.
|
||||
HOST1_TRANSCODE_SERVERS=(
|
||||
"${HOST1_EMBY_CONTAINER}|${HOST1_EMBY_URL}|${HOST1_EMBY_API_KEY}|emby"
|
||||
"${HOST1_JELLYFIN_CONTAINER}|${HOST1_JELLYFIN_URL}|${HOST1_JELLYFIN_API_KEY}|jellyfin"
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── MONITORS ──────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Certificate Monitor ━━━
|
||||
# Domains checked via direct openssl connection — not relying on NPM's certificate state.
|
||||
# Checks the actual certificate served, not what NPM thinks it has.
|
||||
# Thresholds (CERT_WARN_DAYS, CERT_CRIT_DAYS) defined in master.conf.
|
||||
HOST1_CERT_MONITOR_DOMAINS=(
|
||||
"Gmer4Lfe.com"
|
||||
"Gmer4Lfe.us"
|
||||
)
|
||||
|
||||
# ━━━ SMART Health ━━━
|
||||
# Drives skipped in SMART attribute monitoring — hardware is server-specific.
|
||||
# Thresholds read from dynamix.cfg at runtime — fallbacks in master.conf.
|
||||
HOST1_SMART_IGNORE_DRIVES=(
|
||||
"sda" # boot USB — SMART not meaningful on flash drives
|
||||
)
|
||||
|
||||
# ━━━ ZFS Report ━━━
|
||||
# Pools excluded from the weekly ZFS health report — reduces noise from single-disk array pools.
|
||||
# These are individual array disks formatted as ZFS — converting to XFS over time via unBalance.
|
||||
# Pool health thresholds defined in master.conf.
|
||||
HOST1_ZFS_REPORT_IGNORE_POOLS=(
|
||||
"disk5"
|
||||
"disk6"
|
||||
"disk8"
|
||||
"disk9"
|
||||
"disk10"
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── RESOURCE MANAGER ──────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Containers to manage under pressure — see master.conf RW_CRITICAL_CONTAINERS for exclusions.
|
||||
|
||||
# docker pause at medium pressure (RAM < RW_RAM_MEDIUM_GB or load > medium threshold)
|
||||
# Suspended in-place — instant to pause/unpause, no state lost, no restart delay.
|
||||
HOST1_RW_PAUSE_CONTAINERS=(
|
||||
"Huntarr" # arr search automation — safe to suspend
|
||||
"Cleanuparr" # download cleanup — safe to suspend
|
||||
"Healarr" # arr health checks — safe to suspend
|
||||
"Soularr" # Slskd automation — background only
|
||||
"ChannelTube" # YouTube archiver — background only
|
||||
"Pinchflat" # YouTube archiver — background only
|
||||
)
|
||||
|
||||
# docker stop at hard pressure (RAM < RW_RAM_HARD_GB)
|
||||
# Full stop — these are optional/heavy services that free significant RAM when stopped.
|
||||
# resource_watchdog.sh restarts them when pressure fully clears (RAM >= RW_RAM_RECOVER_GB).
|
||||
HOST1_RW_STOP_CONTAINERS=(
|
||||
"LocalAI" # GPU/CPU heavy — largest RAM consumer when idle
|
||||
"7DaysToDie" # game server — optional
|
||||
"V-Rising" # game server — optional
|
||||
"Code-Server" # IDE — not needed during pressure events
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── SYSTEM WATCHDOG ───────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Per-host check toggles and NIC config for system_watchdog.sh.
|
||||
# Aliased by detect_hosts() — script uses unprefixed SYS_WATCHDOG_* names.
|
||||
# HOST1: TR1950X 128GB — full media server, active transcoding, ZFS cache pools.
|
||||
#
|
||||
# Three-tier response — all critical checks enabled by default on HOST1:
|
||||
# Tier 1 (bypass strikes, reboot now): docker daemon, rootfs full, kernel oops, FD, /boot
|
||||
# Tier 2 (bypass strikes with OOM): RAM critical + OOM kills in cycle
|
||||
# Tier 3 (standard strike system): everything else
|
||||
#
|
||||
# RAM tiers, OOM limits, and reboot loop settings in master.conf System Watchdog section.
|
||||
|
||||
# ━━━ Primary NIC ━━━
|
||||
# Network interface for NIC state check — verify with: ip link show | grep "^[0-9]"
|
||||
# Common values: eth0, bond0, br0, eno1
|
||||
HOST1_SYS_WATCHDOG_NIC="eth0"
|
||||
|
||||
# ━━━ Tier 1 — Critical Checks ━━━
|
||||
# These bypass the strike system — a single hit triggers immediate reboot.
|
||||
# Disabling any of these is not recommended — they protect against acute system failure.
|
||||
|
||||
# Docker daemon unresponsive → try restart, reboot if restart fails.
|
||||
# Without a working daemon docker_watchdog.sh is blind and containers cannot be managed.
|
||||
HOST1_SYS_WATCHDOG_CHECK_DOCKER_DAEMON=true
|
||||
|
||||
# rootfs at critical threshold (ROOTFS_CRITICAL_PCT=99) → reboot immediately.
|
||||
# At 99% rootfs writes fail silently — logs stop, Docker errors out, SSH may stop working.
|
||||
# Standard 95% threshold still uses strike system — only 99%+ is critical tier.
|
||||
HOST1_SYS_WATCHDOG_CHECK_ROOTFS=true
|
||||
|
||||
# Kernel BUG/Oops in dmesg delta since last cycle → reboot immediately.
|
||||
# A kernel oops means the kernel ran with a corrupted state — stability is not guaranteed.
|
||||
HOST1_SYS_WATCHDOG_CHECK_KERNEL_OOPS=true
|
||||
|
||||
# File descriptor exhaustion at FD_CRITICAL_PCT (95%) → reboot immediately.
|
||||
# At 95% FD: new connections fail, Docker can't spawn processes, SSH drops.
|
||||
HOST1_SYS_WATCHDOG_CHECK_FD=true
|
||||
|
||||
# /boot read-only detected → reboot immediately.
|
||||
# Unexpected read-only /boot means state files and config writes are silently failing.
|
||||
# Fallback state, watchdog reboot log, and lock files all go stale silently.
|
||||
HOST1_SYS_WATCHDOG_CHECK_BOOT=true
|
||||
|
||||
# ━━━ Tier 2 — Urgent OOM Check ━━━
|
||||
# Bypass strikes when RAM is critically low AND OOM kill rate confirms active crisis.
|
||||
# Both must be enabled for Tier 2 bypass to function — disable either to always use strikes.
|
||||
|
||||
# Track kernel OOM kills each cycle via /proc/vmstat oom_kill delta.
|
||||
# Also provides diagnostic context in reboot messages (which processes were killed).
|
||||
HOST1_SYS_WATCHDOG_CHECK_OOM=true
|
||||
|
||||
# Free RAM check — required for both Tier 2 bypass and RAM tier logic.
|
||||
# Tiers: MEM_WARN_GB(10) → notify | MEM_SHUTDOWN_GB(6) → stop containers | MEM_GB(4) → strikes
|
||||
HOST1_SYS_WATCHDOG_CHECK_RAM=true
|
||||
|
||||
# ━━━ Tier 3 — Standard Checks (strike system) ━━━
|
||||
# Each check must fail SYS_WATCHDOG_STRIKE_LIMIT consecutive cycles before action is taken.
|
||||
# Single spikes are ignored — sustained problems trigger reboot.
|
||||
|
||||
# /var/log filesystem usage above SYS_WATCHDOG_LOG_PCT.
|
||||
# Log spam (Docker log storms, syslog loops) fills rootfs — indicates something broken.
|
||||
HOST1_SYS_WATCHDOG_CHECK_LOG=true
|
||||
|
||||
# ZFS ARC memory pinned above SYS_WATCHDOG_ARC_PINNED_PCT after cache drop.
|
||||
# Enabled on HOST1 — ZFS cache pools actively used. Disable on hosts without ZFS.
|
||||
HOST1_SYS_WATCHDOG_CHECK_ARC=true
|
||||
|
||||
# CPU temperature above SYS_WATCHDOG_CPU_TEMP_MAX (95°C).
|
||||
# Sustained high temp causes kernel throttling or panic. Requires lm-sensors.
|
||||
HOST1_SYS_WATCHDOG_CHECK_CPU_TEMP=true
|
||||
|
||||
# Load average above SYS_WATCHDOG_LOAD_MULTIPLIER × core count.
|
||||
# DISABLED on HOST1 — Tdarr and Emby cause legitimate sustained load spikes during encoding.
|
||||
# Enable on idle servers or adjust SYS_WATCHDOG_LOAD_MULTIPLIER if load is always high.
|
||||
HOST1_SYS_WATCHDOG_CHECK_LOAD=false
|
||||
|
||||
# Zombie process count above SYS_WATCHDOG_ZOMBIE_LIMIT (50).
|
||||
# Large zombie counts indicate serious process management failure — something is stuck.
|
||||
HOST1_SYS_WATCHDOG_CHECK_ZOMBIES=true
|
||||
|
||||
# Check docker_watchdog.sh persistent skip list — required containers on skip list.
|
||||
# Cross-watchdog coordination: if docker_watchdog gave up, system_watchdog escalates.
|
||||
# ENABLED — HOST1 fully built and operational, skip list is meaningful.
|
||||
HOST1_SYS_WATCHDOG_CHECK_CONTAINERS=true
|
||||
|
||||
# /tmp filesystem usage above SYS_WATCHDOG_TMP_PCT with auto-clear attempt.
|
||||
# Script tries to clear aged /tmp files first — only strikes if clear fails.
|
||||
# Lock files, rsync temp files, and Docker ops use /tmp — 100% means lock failures.
|
||||
HOST1_SYS_WATCHDOG_CHECK_TMP=true
|
||||
|
||||
# Array disk error count delta in /proc/mdstat — accumulating errors = disk failing now.
|
||||
# Triggers on SYS_WATCHDOG_MDSTAT_ERROR_LIMIT new errors in one cycle.
|
||||
HOST1_SYS_WATCHDOG_CHECK_MDSTAT=true
|
||||
|
||||
# Primary NIC operstate — detects NIC going down (physical or driver failure).
|
||||
# Uses HOST1_SYS_WATCHDOG_NIC above. Strike system — brief flaps don't trigger reboot.
|
||||
HOST1_SYS_WATCHDOG_CHECK_NETWORK=true
|
||||
|
||||
# sshd running check — attempts restart before escalating.
|
||||
# sshd down = no remote access. Script tries rc.sshd start, notifies, strikes on failure.
|
||||
HOST1_SYS_WATCHDOG_CHECK_SSHD=true
|
||||
|
||||
# Runaway process detection — single process above SYS_WATCHDOG_RUNAWAY_CPU_PCT sustained.
|
||||
# DISABLED — Tdarr encoding and Emby transcoding legitimately peg CPU for extended periods.
|
||||
# Enable only if HOST1 has no CPU-intensive workloads.
|
||||
HOST1_SYS_WATCHDOG_CHECK_RUNAWAY=false
|
||||
@@ -0,0 +1,806 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ========================== HOST1 CONFIGURATION — unRAID-Gmer4Lfe ============================
|
||||
# ==============================================================================================
|
||||
# HOST1-specific variables — credentials, container names, share paths, failover lists.
|
||||
# Sourced after master.conf — values here extend shared profile arrays and add HOST1-specific
|
||||
# identity, credentials, and container configuration.
|
||||
#
|
||||
# Sparse checkout (git) ensures HOST2 never receives this file.
|
||||
# HOST2 never sees HOST1 credentials — clean separation at the file level.
|
||||
#
|
||||
# DO NOT put shared config here — thresholds, toggles, profiles belong in master.conf.
|
||||
# DO NOT put HOST2 variables here — they belong in host2.conf.
|
||||
#
|
||||
# ── INDEX ─────────────────────────────────────────────────────────────────────────────────────
|
||||
#
|
||||
# ── IDENTITY & CONNECTIVITY ────────────────────────────────────────────────────────────────
|
||||
# IDENTITY hostname, SSH key, Unraid API key
|
||||
# EMBY container name, URL, API key
|
||||
# JELLYFIN container name, URL, API key
|
||||
# GITEA API token for SSH key registration
|
||||
# NOTIFICATIONS Discord webhook
|
||||
#
|
||||
# ── PARTNERSHIP ────────────────────────────────────────────────────────────────────────────
|
||||
# PARTNERSHIP auth containers, backup paths, emby provisioning
|
||||
#
|
||||
# ── RSYNC ──────────────────────────────────────────────────────────────────────────────────
|
||||
# DAILY SYNC SHARES media shares HOST1 owns and pushes to HOST2
|
||||
# WEEKLY SYNC SHARES appdata shares synced weekly (Sunday 2:30am)
|
||||
# CRITICAL SYNC SHARES appdata shares synced every 30 minutes
|
||||
# BACKUP VERIFY shares for checksum verification against remote
|
||||
# HOST1 RSYNC PROFILE host1-appdata profile for HOST1-specific appdata syncs
|
||||
#
|
||||
# ── FALLBACK ───────────────────────────────────────────────────────────────────────────────
|
||||
# DDNS DDNS containers managed by HOST1
|
||||
# INTERNET LOSS containers stopped when internet is lost
|
||||
# FALLBACK TIERS what HOST1 runs for HOST2 per tier
|
||||
# TIER DELAYS how long HOST1 must be down before each tier activates on HOST2
|
||||
# RSYNC WRITEBACK HOST1 appdata synced back on handback
|
||||
#
|
||||
# ── DOCKER ─────────────────────────────────────────────────────────────────────────────────
|
||||
# DOCKER DAILY RESTART containers restarted daily
|
||||
# DOCKER WEEKLY RESTART containers restarted weekly
|
||||
# DOCKER WATCHDOG memory limits, health URLs, required containers, ignore list
|
||||
# NETWORK WATCHDOG DDNS domain, NPM URL for connectivity checks
|
||||
# DOCKER NETWORK CONNECT networks and containers for docker_network_connect.sh
|
||||
#
|
||||
# ── MEDIA ──────────────────────────────────────────────────────────────────────────────────
|
||||
# MEDIA PERMISSIONS share list for media_shares_permissions.sh
|
||||
# MEDIA CLEANER folder lists for media_cleaner.sh
|
||||
#
|
||||
# ── ARR STACK ──────────────────────────────────────────────────────────────────────────────
|
||||
# DOWNLOADERS slskd, SABnzbd, qBittorrent credentials and URLs
|
||||
# LIDARR URL, API key, path map
|
||||
# SONARR URL, API key, path map
|
||||
# RADARR URL, API key, path map
|
||||
# ARR RECOVERY per-arr recovery toggles
|
||||
#
|
||||
# ── TRANSCODES ─────────────────────────────────────────────────────────────────────────────
|
||||
# TRANSCODES ramdisk size, thresholds, SSD path, server array
|
||||
#
|
||||
# ── MONITORS ───────────────────────────────────────────────────────────────────────────────
|
||||
# CERTIFICATE MONITOR domains checked for SSL expiry
|
||||
# SMART HEALTH drives to skip in SMART monitoring
|
||||
# ZFS REPORT pools to exclude from ZFS health report
|
||||
#
|
||||
# ── RESOURCE MANAGER ───────────────────────────────────────────────────────────────────────
|
||||
# RESOURCE MANAGER containers paused/stopped under memory pressure
|
||||
#
|
||||
# ── SYSTEM WATCHDOG ────────────────────────────────────────────────────────────────────────
|
||||
# SYSTEM WATCHDOG per-host check toggles and NIC configuration
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
# ==============================================================================================
|
||||
# ── STORAGE MODE ──────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Storage mode ━━━
|
||||
# Controls where Varaverk stores scripts, conf, and state files.
|
||||
# true = internal NVMe/SSD — /boot/config/plugins/varaverk (write-safe, git-direct)
|
||||
# false = USB flash boot — /mnt/user/appdata/Varaverk (preserves flash lifetime)
|
||||
HOST1_STORAGE_MODE_INTERNAL=true
|
||||
|
||||
# ==============================================================================================
|
||||
# ── IDENTITY & CONNECTIVITY ───────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Identity ━━━
|
||||
# HOST1 hostname lives in master.conf (not a credential — safe for all servers).
|
||||
# SSH key used for all server-to-server operations — rsync, failover container commands.
|
||||
# Must be in /root/.ssh/ and authorised in HOST2's /root/.ssh/authorized_keys.
|
||||
HOST1_SSH_KEY="/root/.ssh/gmer4lfe_rsync_automation"
|
||||
HOST1_OWNER="gmer4lfe"
|
||||
HOST1_OWNER_EMAIL="gmer4lfe@gmail.com"
|
||||
|
||||
# ━━━ Unraid API ━━━
|
||||
# Used by the Varaverk plugin to query this server's Unraid GraphQL API.
|
||||
# Generate in Unraid: Settings → Management Access → API Keys → + New Key
|
||||
HOST1_UNRAID_API_KEY="1825c3a2e03ea5089974f4da2e171aa2d5907a1dea23cc479c33e492c8ff4dbb"
|
||||
|
||||
# ━━━ Emby ━━━
|
||||
# Referenced by transcode_manager.sh, emby_session_report.sh, emby_database_repair.sh,
|
||||
# weekly_sync_maintenance.sh, and HOST1_TRANSCODE_SERVERS below.
|
||||
# API key: Emby Dashboard → API Keys → + New Key
|
||||
HOST1_EMBY_CONTAINER="Emby"
|
||||
HOST1_EMBY_URL="http://localhost:8096"
|
||||
HOST1_EMBY_API_KEY="0c27448d93a7431f9ac63569f7655829"
|
||||
|
||||
# ━━━ Jellyfin ━━━
|
||||
# API key: Jellyfin Dashboard → Administration → API Keys → + New Key
|
||||
HOST1_JELLYFIN_CONTAINER="Jellyfin"
|
||||
HOST1_JELLYFIN_URL="http://localhost:8095"
|
||||
HOST1_JELLYFIN_API_KEY="4e820e7df74c4933acec212b1996314e"
|
||||
|
||||
# ━━━ Gitea ━━━
|
||||
# Personal access token for gitea_ssh_setup.sh — registers this server's SSH public key
|
||||
# with Gitea so git operations use key auth instead of passwords.
|
||||
# Create in Gitea: Settings → Applications → Generate Token → scope: write:user
|
||||
HOST1_GITEA_API_TOKEN=""
|
||||
|
||||
# ━━━ Notifications ━━━
|
||||
# Discord webhook — leave blank to disable.
|
||||
# Per-host so HOST1 and HOST2 can post to different channels or only one server notifies.
|
||||
HOST1_DISCORD_WEBHOOK=""
|
||||
|
||||
# ==============================================================================================
|
||||
# ── PARTNERSHIP ───────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# HOST1 is always the owner (source of truth) unless --transfer has been run.
|
||||
# See README-Partnership.md and master.conf PARTNERSHIP section for full lifecycle docs.
|
||||
|
||||
# Auth containers reconfigured on onboard/offboard.
|
||||
# Format: "ContainerName|WebUIPort"
|
||||
# On onboard → WebUI pointed at owner's Tailscale IP (mirror clicks NPM, gets owner's auth)
|
||||
# On offboard → WebUI pointed back at localhost
|
||||
HOST1_PARTNERSHIP_AUTH_WEBUIS=(
|
||||
"NginxProxyManager|81"
|
||||
"Lldap-Gmer4Lfe|17170"
|
||||
"Authelia|9091"
|
||||
"Authelia-Secondary|9092"
|
||||
)
|
||||
|
||||
# XML templates (from this server's templates-user/) pushed to mirror during onboard.
|
||||
# These become the mirror's active auth stack, backed by the rsync-synced appdata.
|
||||
# Update filename if Lldap is renamed to drop the host suffix.
|
||||
HOST1_PARTNERSHIP_AUTH_STACK=(
|
||||
# Dependencies first — Mariadb/Redis must be healthy before Authelia starts
|
||||
"my-Mariadb-Authelia.xml"
|
||||
"my-Mariadb-Authelia-Secondary.xml"
|
||||
"my-Redis-Authelia.xml"
|
||||
"my-Redis-Authelia-Secondary.xml"
|
||||
# Auth apps — deployed after their deps are confirmed healthy
|
||||
"my-Authelia.xml"
|
||||
"my-Authelia-Secondary.xml"
|
||||
"my-NginxProxyManager.xml"
|
||||
"my-Lldap-Gmer4Lfe.xml"
|
||||
# Source of truth — must be available on HOST2 independently of the auth stack
|
||||
"my-Gitea.xml"
|
||||
)
|
||||
|
||||
# XML templates pushed to mirror for the arr stack during onboard.
|
||||
# Deps (e.g. databases) first if any — same ordering rule as auth stack.
|
||||
HOST1_PARTNERSHIP_ARR_STACK=(
|
||||
# "my-Sonarr.xml"
|
||||
# "my-Radarr.xml"
|
||||
# "my-Lidarr.xml"
|
||||
# "my-Prowlarr.xml"
|
||||
# "my-Bazarr.xml"
|
||||
)
|
||||
|
||||
# Containers stopped on THIS server before deploying the mirror's stack on onboard.
|
||||
# Only needed when this server parks its own stack to make room for the mirror's.
|
||||
HOST1_PARTNERSHIP_REPLACE_CONTAINERS=(
|
||||
)
|
||||
|
||||
# Arr containers stopped on this server when mirror's arr stack is deployed.
|
||||
HOST1_PARTNERSHIP_ARR_REPLACE_CONTAINERS=(
|
||||
)
|
||||
|
||||
# Paths HOST2 should collect during the grace window after offboard.
|
||||
# Notified on offboard — no auto-deletion, HOST2 must collect manually within PARTNERSHIP_GRACE_HOURS.
|
||||
HOST1_PARTNERSHIP_MIRROR_BACKUPS=(
|
||||
# "/mnt/user/appdata-Fallback/Jayred365-Emby"
|
||||
)
|
||||
|
||||
# Containers parked on this server when partnership is active.
|
||||
# Stopped on onboard (owner deploys its stack instead), restarted on offboard.
|
||||
HOST1_PARTNERSHIP_OWN_CONTAINERS=(
|
||||
# "Emby"
|
||||
# "NginxProxyManager"
|
||||
)
|
||||
|
||||
# Emby admin provisioning — toggle is owner-only, credentials are per-host.
|
||||
# Owner enables/disables the feature. Each host sets the account they want on the shared Emby.
|
||||
# On onboard: owner reads mirror's HOST*_PARTNERSHIP_EMBY_ADMIN_* and creates that account.
|
||||
# On offboard: account is deleted. Username collision → onboard exits with error.
|
||||
HOST1_PARTNERSHIP_PROVISION_EMBY_ADMIN=false # owner controls whether Emby is shared
|
||||
HOST1_PARTNERSHIP_EMBY_PORT=8096
|
||||
HOST1_PARTNERSHIP_EMBY_ADMIN_USER="" # this server's desired Emby username
|
||||
HOST1_PARTNERSHIP_EMBY_ADMIN_PASS="" # this server's desired Emby password
|
||||
|
||||
# ==============================================================================================
|
||||
# ── RSYNC ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Daily Sync Shares ━━━
|
||||
# Shares HOST1 pushes to all other nodes every night (1am via daily_sync_maintenance.sh).
|
||||
# Mesh model: every node pushes every media share — no ownership, no mirrors.
|
||||
# arr_sync ensures all arr libraries converge (union). rsync spreads files (additive, no --delete).
|
||||
# arr_cleanup removes true orphans based on local arr state.
|
||||
# Any node can download content to any share — it propagates to all nodes on the next cycle.
|
||||
# Nextcloud is intentionally one-directional (HOST1→HOST2 offsite backup — not arr-managed).
|
||||
# Uses DEFAULT_RSYNC_OPTS from master.conf — no profile needed.
|
||||
# For shares needing container stops or custom options — add a profile in master.conf.
|
||||
HOST1_DAILY_SYNC_SHARES=(
|
||||
/mnt/user/Books
|
||||
/mnt/user/Intros
|
||||
/mnt/user/Kids_Movies
|
||||
/mnt/user/Kids_Tv_Shows
|
||||
/mnt/user/Movies
|
||||
/mnt/user/Music
|
||||
/mnt/user/Music_Videos
|
||||
/mnt/user/Nextcloud
|
||||
/mnt/user/stand-up_comedy
|
||||
/mnt/user/Sports
|
||||
# /mnt/user/Tv_Shows
|
||||
/mnt/user/Anime_Shows-Old
|
||||
/mnt/user/Anime_Movies-Old
|
||||
/mnt/user/Anime_Movies
|
||||
/mnt/user/Anime_Shows
|
||||
)
|
||||
|
||||
# Personal encrypted shares — synced for offsite backup, independent of media shares.
|
||||
# ZFS encrypted at dataset level — remote receives encrypted blocks, cannot read content.
|
||||
# See README-Rsync_Setup.md for ZFS encryption setup before uncommenting.
|
||||
HOST1_PERSONAL_SHARES=(
|
||||
# /mnt/user/HOST1-Personal # uncomment after creating encrypted dataset
|
||||
)
|
||||
|
||||
# ━━━ Weekly Sync Shares ━━━
|
||||
# Appdata shares synced during the weekly maintenance window (Sunday 2:30am).
|
||||
# Containers stopped both sides before sync — full clean state guaranteed.
|
||||
# Profiles drive container stops, excludes, and options — configured in master.conf RSYNC section.
|
||||
# Order matters — Emby first (larger transfer), then Critical-Data (auth stack).
|
||||
HOST1_WEEKLY_SYNC_SHARES=(
|
||||
"/mnt/user/Media_Server/Emby" # emby profile — full clean mirror
|
||||
"/mnt/user/appdata-Fallback/Critical-Data" # critical-data profile — auth stack
|
||||
)
|
||||
|
||||
# ━━━ Intermediate Sync Shares ━━━
|
||||
# Shares synced every 4 hours by intermediate_sync_maintenance.sh.
|
||||
# Uses DEFAULT_RSYNC_OPTS (no --delete) — for sub-daily propagation of metadata or watch state.
|
||||
# Full media share sync stays in the daily window. Leave empty to skip mid-day rsync entirely.
|
||||
HOST1_INTERMEDIATE_SYNC_SHARES=(
|
||||
# Add shares here to enable mid-day rsync
|
||||
# Example: "/mnt/user/Emby_Metadata"
|
||||
)
|
||||
|
||||
# ━━━ Critical Sync Shares ━━━
|
||||
# Appdata shares synced every 30 minutes by critical_sync_maintenance.sh.
|
||||
# Format: "/path/to/share" or "/path/to/share|profile-name"
|
||||
# Order matters — Critical-Data first (auth stack), then Emby dirty sync.
|
||||
HOST1_CRITICAL_SYNC_SHARES=(
|
||||
"/mnt/user/appdata-Fallback/Critical-Data|critical-fallback" # auth dirty sync — stays running
|
||||
"/mnt/user/Media_Server/Emby|emby-fallback" # Emby dirty sync — stays running
|
||||
)
|
||||
|
||||
# ━━━ Backup Verify ━━━
|
||||
# Shares verified by backup_verify.sh — random file checksum comparison against remote.
|
||||
# Leave empty to use HOST1_DAILY_SYNC_SHARES automatically.
|
||||
# Sample size and minimum file size defined in master.conf.
|
||||
HOST1_BACKUP_VERIFY_SHARES=(
|
||||
# leave empty to use HOST1_DAILY_SYNC_SHARES automatically
|
||||
)
|
||||
|
||||
# ━━━ HOST1 Rsync Profile — host1-appdata ━━━
|
||||
# HOST1-specific appdata sync profile — extends the shared PROFILE_* arrays in master.conf.
|
||||
# Use for appdata unique to HOST1 (Organizrv2, VaultWarden, UptimeKuma etc.)
|
||||
# Shared appdata (auth stack, Emby) use dedicated profiles defined in master.conf.
|
||||
# Run manually: bash Rsync/rsync.sh /mnt/user/appdata-Fallback/HOST1-Appdata --profile=host1-appdata
|
||||
PROFILE_RSYNC_OPTS[host1-appdata]="-av --info=progress2 --bwlimit=${PROFILE_BW_LIMIT[host1-appdata]:-8000}"
|
||||
PROFILE_BW_LIMIT[host1-appdata]=8000
|
||||
PROFILE_RETRY_COUNT[host1-appdata]=3
|
||||
PROFILE_SLEEP[host1-appdata]=300
|
||||
PROFILE_CRITICAL_CONTAINER_NAMES[host1-appdata]="Organizrv2-Gmer4Lfe UptimeKuma-Gmer4Lfe VaultWarden-Gmer4Lfe"
|
||||
PROFILE_DELAYED_CONTAINERS[host1-appdata]=""
|
||||
PROFILE_CONTAINER_DELAY[host1-appdata]=5
|
||||
PROFILE_EXCLUDE_DIRS[host1-appdata]="logs *.tmp"
|
||||
|
||||
# ==============================================================================================
|
||||
# ── FALLBACK ──────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ DDNS ━━━
|
||||
# DDNS containers HOST1 manages — started/stopped by fallback.sh per DDNS absolute rules:
|
||||
# Internet loss → stop immediately
|
||||
# Failover → HOST2 starts HOST1's DDNS as Tier 1 (before any other containers)
|
||||
# Handback → stop HOST1's DDNS on HOST2 → rsync → start containers → start local DDNS last
|
||||
HOST1_DDNS_CONTAINERS=(
|
||||
"Gmer4Lfe.com"
|
||||
)
|
||||
|
||||
# ━━━ Internet Loss ━━━
|
||||
# Containers stopped immediately on HOST1 when internet connection is lost.
|
||||
# Prevents external-facing services from operating without connectivity.
|
||||
FALLBACK_HOST1_STOP_ON_NO_NET=(
|
||||
"Gmer4Lfe.com"
|
||||
)
|
||||
|
||||
# ━━━ Fallback Tiers — HOST1 Runs for HOST2 ━━━
|
||||
# Containers HOST1 starts when HOST2 goes down.
|
||||
# Tier 1 is always immediate — vital services cannot wait.
|
||||
# Higher tiers activate after HOST2_TIER*_DELAY minutes (set in host2.conf).
|
||||
FALLBACK_HOST1_COVERS_HOST2_TIER1=(
|
||||
"Gmer4Lfe.us"
|
||||
"VaultWarden-Jayred365"
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_COVERS_HOST2_TIER2=(
|
||||
# "container-placeholder"
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_COVERS_HOST2_TIER3=(
|
||||
# "container-placeholder"
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_COVERS_HOST2_TIER4=(
|
||||
# "container-placeholder"
|
||||
)
|
||||
|
||||
# ━━━ Tier Delays — HOST1's Containers on HOST2 ━━━
|
||||
# How long HOST1 must be down before each tier activates on HOST2 — in minutes.
|
||||
# Tier 1 is always immediate — no delay var needed.
|
||||
HOST1_TIER2_DELAY=240 # 4 hours — NextCloud, Immich
|
||||
HOST1_TIER3_DELAY=720 # 12 hours — secondary services
|
||||
HOST1_TIER4_DELAY=1440 # 24 hours — arrs + downloaders
|
||||
|
||||
# ━━━ Rsync Writeback — HOST1 Appdata Back on Handback ━━━
|
||||
# Syncs HOST1 appdata BACK to HOST1 when it comes back online after a failover.
|
||||
# Containers stopped before writeback — clean source, no competing writes.
|
||||
#
|
||||
# HOST1_TIER1_WRITEBACK_DELAY: short outages skip Tier 1 writeback — primary state
|
||||
# is more reliable than dirty sync data for brief outages.
|
||||
HOST1_TIER1_WRITEBACK_DELAY=60 # skip Emby writeback if outage under 1hr
|
||||
|
||||
# Tier 4 automatically syncs HOST1_DAILY_SYNC_SHARES — only list paths NOT in that array.
|
||||
FALLBACK_HOST1_WRITEBACK_TIER1=(
|
||||
"/mnt/user/Media_Server/Emby" # watch states built up during outage
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_WRITEBACK_TIER2=(
|
||||
"/mnt/user/appdata-Fallback/Important-Data" # NextCloud + Postgres — files added during outage
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_WRITEBACK_TIER3=(
|
||||
# "location-placeholder"
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_WRITEBACK_TIER4=(
|
||||
"/mnt/user/appdata-Fallback/Arrs_Stack" # arr databases — downloads queued during outage
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── DOCKER ────────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Docker Daily Restart ━━━
|
||||
# Containers restarted every day via DAILY_MAINTENANCE_SCRIPTS.
|
||||
# Dispatcharr degrades over time without restart — daily is intentional, not just housekeeping.
|
||||
# Order matters — auth stack first, then media services.
|
||||
HOST1_DAILY_RESTART_CONTAINERS=(
|
||||
"NginxProxyManager"
|
||||
"Lldap-Gmer4Lfe"
|
||||
"Authelia"
|
||||
"Authelia-Secondary"
|
||||
"Dispatcharr-Iptv-Users"
|
||||
"Dispatcharr" # Live TV scheduler — degrades without daily restart
|
||||
"Dispatcharr-Basic"
|
||||
"ErsatzTV-Emby"
|
||||
"slskd" # Soulseek connection drops after extended uptime; restart refreshes share index
|
||||
)
|
||||
|
||||
# ━━━ Docker Weekly Restart ━━━
|
||||
# Less critical services restarted weekly via WEEKLY_MAINTENANCE_SCRIPTS (Sunday 2:30am).
|
||||
# Containers already stopped for weekly sync — restart adds zero extra downtime.
|
||||
HOST1_WEEKLY_RESTART_CONTAINERS=(
|
||||
"NextCloud"
|
||||
"Organizrv2-Gmer4Lfe"
|
||||
"AdGuard-Home"
|
||||
"Immich-Gmer4Lfe"
|
||||
)
|
||||
|
||||
# ━━━ Docker Watchdog ━━━
|
||||
# Per-HOST1 container configuration for docker_watchdog.sh.
|
||||
# Shared thresholds and toggles live in master.conf.
|
||||
|
||||
# Memory hard limits in MB — immediate restart if exceeded.
|
||||
# Set at "container is clearly broken" not "container is busy".
|
||||
# 20GB=20480 18GB=18432 16GB=16384 12GB=12288 8GB=8192 4GB=4096 2GB=2048 1GB=1024
|
||||
declare -A HOST1_WATCHDOG_CONTAINERS=(
|
||||
["Emby"]=20480 # 20GB — large library + active transcodes
|
||||
["LidaTube"]=6144 # 6GB — memory leak over time
|
||||
["Tdarr"]=6144 # 6GB — encoding is memory intensive
|
||||
["Code-Server"]=1024 # 1GB — should never need more
|
||||
)
|
||||
|
||||
# HTTP health check URLs — checked every cycle, strike system before restart.
|
||||
# Only add containers with a meaningful web interface to check.
|
||||
declare -A HOST1_WATCHDOG_CONTAINER_URLS=(
|
||||
["Emby"]="http://localhost:8096"
|
||||
["NginxProxyManager"]="http://localhost:7818"
|
||||
["Authelia"]="http://localhost:9091/api/health"
|
||||
["Authelia-Secondary"]="http://localhost:9092/api/health"
|
||||
["Lldap-Gmer4Lfe"]="http://localhost:17170"
|
||||
)
|
||||
|
||||
# Required containers — must always be running on HOST1.
|
||||
# Strike system before restart — repeated failures go on skip list, auto-clears on recovery.
|
||||
# Listed in dependency order — dependencies before dependents.
|
||||
HOST1_WATCHDOG_REQUIRED_CONTAINERS=(
|
||||
"NginxProxyManager"
|
||||
"Lldap-Gmer4Lfe"
|
||||
"Mariadb-Authelia"
|
||||
"Mariadb-Authelia-Secondary"
|
||||
"Redis-Authelia"
|
||||
"Redis-Authelia-Secondary"
|
||||
"Authelia"
|
||||
"Authelia-Secondary"
|
||||
)
|
||||
|
||||
# Containers to skip in Tier 2 global scan — legitimately stopped or frequently restarting.
|
||||
# Watchdog leaves these alone entirely — no restart attempts, no crash loop tracking.
|
||||
HOST1_WATCHDOG_SCAN_IGNORE=(
|
||||
"DashGate"
|
||||
"PIA-WG-Config-Generator"
|
||||
"Aperture"
|
||||
"Aperture-Kids"
|
||||
"pgvector-18-Apeture-Kids"
|
||||
"Pgvector18-Aperture"
|
||||
"emby-test" # broken test container (exit 127 — bad image)
|
||||
)
|
||||
|
||||
# Dependency ordering — skip restarting a container if its dependency is also down.
|
||||
# Prevents watchdog from restarting Authelia before Mariadb is back up.
|
||||
# SPACE-SEPARATED STRINGS — converted to array at runtime.
|
||||
declare -A HOST1_WATCHDOG_DEPENDENCIES=(
|
||||
["Authelia"]="Mariadb-Authelia Redis-Authelia"
|
||||
["Authelia-Secondary"]="Mariadb-Authelia Redis-Authelia-Secondary"
|
||||
["NextCloud"]="Postgres-NextCloud"
|
||||
)
|
||||
|
||||
# Per-container appdata growth suppress ceilings in MB.
|
||||
# ONLY needed in specific cases — growth rate detection covers all containers automatically.
|
||||
# Use this when a container legitimately has large stable data and you want to guarantee
|
||||
# it never triggers a false-positive growth alert. Growth warnings are suppressed while the
|
||||
# container's dir stays below this ceiling; above it, warnings resume as normal.
|
||||
# 50GB=51200 25GB=25600 20GB=20480 15GB=15360 10GB=10240 5GB=5120
|
||||
declare -A HOST1_WATCHDOG_APPDATA_SIZES=(
|
||||
["Tdarr"]="25600" # 25GB — transcode cache grows legitimately during active jobs
|
||||
["7dtd"]="20480" # 20GB — game server world data, expected to be large
|
||||
)
|
||||
|
||||
# API-level health checks — checked every cycle alongside HTTP URL checks.
|
||||
# Format: ["ContainerName"]="url|expected_json_key|expected_value"
|
||||
# Empty = no API checks for this host.
|
||||
declare -A HOST1_WATCHDOG_CONTAINER_API_CHECKS=()
|
||||
|
||||
# ━━━ Network Watchdog ━━━
|
||||
# Host-specific connectivity config for Watchdogs/System/network_watchdog.sh.
|
||||
HOST1_NETWORK_WATCHDOG_DDNS_DOMAIN="gmer4lfe.com"
|
||||
HOST1_NETWORK_WATCHDOG_DDNS_CONTAINER="Gmer4Lfe.com"
|
||||
HOST1_NETWORK_WATCHDOG_NPM_URL="https://gmer4lfe.com"
|
||||
|
||||
# ━━━ Docker Network Connect ━━━
|
||||
# Containers connected to custom networks at array start by docker_network_connect.sh.
|
||||
# Networks created if they don't exist — idempotent, safe to re-run.
|
||||
HOST1_NETWORK_CONNECT_CONTAINERS=(
|
||||
"memcached"
|
||||
"Npm-CrowdSec"
|
||||
)
|
||||
|
||||
HOST1_NETWORK_CONNECT_NETWORKS=(
|
||||
"high-availability"
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── MEDIA ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Media Permissions ━━━
|
||||
# Shares that media_shares_permissions.sh applies PERMISSIONS_MODE and PERMISSIONS_OWNER to.
|
||||
# Runs first in DAILY_MAINTENANCE_SCRIPTS — arr cleanup depends on correct ownership.
|
||||
HOST1_MEDIA_PERMISSION_SHARES=(
|
||||
/mnt/user/Anime_Movies
|
||||
/mnt/user/Anime_Movies-Old
|
||||
/mnt/user/Anime_Shows
|
||||
/mnt/user/Anime_Shows-Old
|
||||
/mnt/user/appcache
|
||||
/mnt/user/Books
|
||||
/mnt/user/Downloads
|
||||
/mnt/user/Games
|
||||
/mnt/user/Intros
|
||||
/mnt/user/Kids_Movies
|
||||
/mnt/user/Kids_Tv_Shows
|
||||
/mnt/user/Movie_Recordings
|
||||
/mnt/user/Movies
|
||||
/mnt/user/Music
|
||||
/mnt/user/Music_Videos
|
||||
/mnt/user/Photo
|
||||
/mnt/user/Sports
|
||||
/mnt/user/stand-up_comedy
|
||||
/mnt/user/Tv_Recordings
|
||||
/mnt/user/Tv_Shows
|
||||
/mnt/user/YouTube
|
||||
)
|
||||
|
||||
# ━━━ Media Cleaner ━━━
|
||||
# Folder lists for media_cleaner.sh — two profiles: anime and media.
|
||||
# File patterns shared across all servers — defined in master.conf.
|
||||
# Called via DAILY_MAINTENANCE_SCRIPTS. Run manually: Media/media_cleaner.sh anime|media
|
||||
HOST1_ANIME_CLEAN_FOLDERS=(
|
||||
/mnt/user/Anime_Movies
|
||||
/mnt/user/Anime_Movies-Old
|
||||
/mnt/user/Anime_Shows
|
||||
/mnt/user/Anime_Shows-Old
|
||||
)
|
||||
|
||||
HOST1_MEDIA_CLEAN_FOLDERS=(
|
||||
/mnt/user/Kids_Movies
|
||||
/mnt/user/Kids_Tv_Shows
|
||||
/mnt/user/Movies
|
||||
/mnt/user/Music
|
||||
/mnt/user/Sports
|
||||
/mnt/user/stand-up_comedy
|
||||
/mnt/user/Tv_Shows
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── ARR STACK ─────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Used by arr cleanup scripts and arrs_failed_stalled_recovery.sh.
|
||||
# detect_hosts() selects HOST1 vars when running on HOST1.
|
||||
#
|
||||
# PATH MAPS — container path → host path translation.
|
||||
# Arr stores file paths using container-internal paths — scripts need host paths to scan.
|
||||
# Add one entry per root folder in arr Settings → Media Management → Root Folders.
|
||||
|
||||
# ━━━ Downloaders ━━━
|
||||
# Used by downloaders_reset.sh — runs every 30min via CRITICAL_MAINTENANCE_SCRIPTS.
|
||||
# Clears stuck states, purges old history, prepares each client for a clean cycle.
|
||||
|
||||
# slskd — clears stuck searches, dead transfers, purges expired failed imports.
|
||||
# SLSKD_FAILED_IMPORTS_DIR: where Soularr moves albums Lidarr rejected.
|
||||
HOST1_SLSKD_URL="http://localhost:8980"
|
||||
HOST1_SLSKD_API_KEY="4bF9kL2mNpQrT7vWxYz1A3dEgHjKoRsU"
|
||||
HOST1_SLSKD_FAILED_IMPORTS_DIR="/mnt/user/Temp_Storage/Slskd/completed/failed_imports"
|
||||
|
||||
# SABnzbd
|
||||
HOST1_SABNZBD_URL="http://localhost:8180"
|
||||
HOST1_SABNZBD_API_KEY="8bfefe41d83b4d50883e32859b55ca9a"
|
||||
|
||||
# qBittorrent — deleteFiles=false removes torrent from qBit but leaves files on disk.
|
||||
# Radarr/Sonarr manage actual files independently.
|
||||
HOST1_QBIT_URL="http://localhost:8080"
|
||||
HOST1_QBIT_USERNAME="root"
|
||||
HOST1_QBIT_PASSWORD="Stay0utD!ck"
|
||||
|
||||
# ━━━ Lidarr — HOST1 only ━━━
|
||||
# HOST2 does not run Lidarr — HOST1_LIDARR_RECOVERY flag handles the exit cleanly.
|
||||
HOST1_LIDARR_URL="http://localhost:8686"
|
||||
HOST1_LIDARR_API_KEY="b2977e71ef074bc0a0529d9fcce3b2dc"
|
||||
HOST1_LIDARR_MUSIC_ROOT="/mnt/user/Music-New"
|
||||
HOST1_FANART_API_KEY="Yd7147a43b692df0b364b94dc47efb81"
|
||||
HOST1_LASTFM_API_KEY="be6dc169c33ae263e690c30d18b7491d"
|
||||
|
||||
declare -A HOST1_LIDARR_PATH_MAP=(
|
||||
["/ext-music"]="/mnt/user/Music-New"
|
||||
)
|
||||
|
||||
# ━━━ Sonarr ━━━
|
||||
HOST1_SONARR_URL="http://localhost:8989"
|
||||
HOST1_SONARR_API_KEY="130decd3db5b4c25afad64864cd03f9f"
|
||||
HOST1_SONARR_TV_ROOT="/mnt/user/Tv_Shows"
|
||||
|
||||
# Note: stand-up_comedy in both Sonarr + Radarr — TV specials and movie specials, one folder
|
||||
declare -A HOST1_SONARR_PATH_MAP=(
|
||||
["/tv"]="/mnt/user/Tv_Shows"
|
||||
["/ext-standup-comedy"]="/mnt/user/stand-up_comedy/series"
|
||||
["/kids tv"]="/mnt/user/Kids_Tv_Shows"
|
||||
["/ext-anime-shows"]="/mnt/user/Anime_Shows-Old"
|
||||
)
|
||||
|
||||
# ━━━ Radarr ━━━
|
||||
HOST1_RADARR_URL="http://localhost:7878"
|
||||
HOST1_RADARR_API_KEY="d43a3ec6cf1549edb4af0cc63f98b2a9"
|
||||
HOST1_TMDB_API_KEY="3dac5e2e49b5540472d2eafec4f01260"
|
||||
HOST1_RADARR_MOVIES_ROOT="/mnt/user/Movies"
|
||||
|
||||
# Note: stand-up_comedy in both Radarr + Sonarr — movie specials and TV specials, one folder
|
||||
declare -A HOST1_RADARR_PATH_MAP=(
|
||||
["/movies"]="/mnt/user/Movies"
|
||||
["/kids movies"]="/mnt/user/Kids_Movies"
|
||||
["/ext-stand-up-comedy"]="/mnt/user/stand-up_comedy/specials"
|
||||
["/anime-movies"]="/mnt/user/Anime_Movies-Old"
|
||||
["/ext-anime-movies"]="/mnt/user/Anime_Movies-Old"
|
||||
)
|
||||
|
||||
# ━━━ Arr Recovery Toggles ━━━
|
||||
# false = skip that arr on this host — exits cleanly without error
|
||||
HOST1_SONARR_RECOVERY=true
|
||||
HOST1_RADARR_RECOVERY=true
|
||||
HOST1_LIDARR_RECOVERY=true # HOST1 only — exits cleanly on HOST2
|
||||
|
||||
# ==============================================================================================
|
||||
# ── TRANSCODES ────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# Ramdisk size ceiling — tmpfs only uses RAM actually needed, not the full size upfront.
|
||||
# Real-world: 9 streams peaked at ~5.5GB — 10G gives generous headroom on 128GB RAM.
|
||||
HOST1_RAMDISK_SIZE="10G"
|
||||
|
||||
# Usage thresholds — coupled to HOST1_RAMDISK_SIZE, adjust all three together if size changes.
|
||||
# Hysteresis gap (8.5 - 7 = 1.5GB) prevents flip-flop between ramdisk and SSD.
|
||||
HOST1_RAMDISK_WARN_GB=8.5 # flip to SSD when ramdisk usage reaches this
|
||||
HOST1_RAMDISK_LOW_GB=7 # flip back to ramdisk when usage drops to this
|
||||
|
||||
# SSD fallback path — where transcodes land when ramdisk exceeds HOST1_RAMDISK_WARN_GB.
|
||||
# Must be on cache pool — array disks too slow for active transcode writes.
|
||||
HOST1_TRANSCODE_SSD="/mnt/cache/Temp_Storage/Emby/Transcodes/"
|
||||
|
||||
# Media servers sharing the ramdisk transcode space on HOST1.
|
||||
# Format: "ContainerName|URL|APIKey|Type" — Type: emby | jellyfin | plex
|
||||
# Entries with placeholder API keys are skipped automatically.
|
||||
# ⚠️ Tdarr does NOT belong here — keep Tdarr on SSD, not ramdisk.
|
||||
HOST1_TRANSCODE_SERVERS=(
|
||||
"${HOST1_EMBY_CONTAINER}|${HOST1_EMBY_URL}|${HOST1_EMBY_API_KEY}|emby"
|
||||
"${HOST1_JELLYFIN_CONTAINER}|${HOST1_JELLYFIN_URL}|${HOST1_JELLYFIN_API_KEY}|jellyfin"
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── MONITORS ──────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Certificate Monitor ━━━
|
||||
# Domains checked via direct openssl connection — not relying on NPM's certificate state.
|
||||
# Checks the actual certificate served, not what NPM thinks it has.
|
||||
# Thresholds (CERT_WARN_DAYS, CERT_CRIT_DAYS) defined in master.conf.
|
||||
HOST1_CERT_MONITOR_DOMAINS=(
|
||||
"Gmer4Lfe.com"
|
||||
"Gmer4Lfe.us"
|
||||
)
|
||||
|
||||
# ━━━ SMART Health ━━━
|
||||
# Drives skipped in SMART attribute monitoring — hardware is server-specific.
|
||||
# Thresholds read from dynamix.cfg at runtime — fallbacks in master.conf.
|
||||
HOST1_SMART_IGNORE_DRIVES=(
|
||||
"sda" # boot USB — SMART not meaningful on flash drives
|
||||
)
|
||||
|
||||
# ━━━ ZFS Report ━━━
|
||||
# Pools excluded from the weekly ZFS health report — reduces noise from single-disk array pools.
|
||||
# These are individual array disks formatted as ZFS — converting to XFS over time via unBalance.
|
||||
# Pool health thresholds defined in master.conf.
|
||||
HOST1_ZFS_REPORT_IGNORE_POOLS=(
|
||||
"disk5"
|
||||
"disk6"
|
||||
"disk8"
|
||||
"disk9"
|
||||
"disk10"
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── RESOURCE MANAGER ──────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Containers to manage under pressure — see master.conf RW_CRITICAL_CONTAINERS for exclusions.
|
||||
|
||||
# docker pause at medium pressure (RAM < RW_RAM_MEDIUM_GB or load > medium threshold)
|
||||
# Suspended in-place — instant to pause/unpause, no state lost, no restart delay.
|
||||
HOST1_RW_PAUSE_CONTAINERS=(
|
||||
"Huntarr" # arr search automation — safe to suspend
|
||||
"Cleanuparr" # download cleanup — safe to suspend
|
||||
"Healarr" # arr health checks — safe to suspend
|
||||
"Soularr" # Slskd automation — background only
|
||||
"ChannelTube" # YouTube archiver — background only
|
||||
"Pinchflat" # YouTube archiver — background only
|
||||
)
|
||||
|
||||
# docker stop at hard pressure (RAM < RW_RAM_HARD_GB)
|
||||
# Full stop — these are optional/heavy services that free significant RAM when stopped.
|
||||
# resource_watchdog.sh restarts them when pressure fully clears (RAM >= RW_RAM_RECOVER_GB).
|
||||
HOST1_RW_STOP_CONTAINERS=(
|
||||
"LocalAI" # GPU/CPU heavy — largest RAM consumer when idle
|
||||
"7DaysToDie" # game server — optional
|
||||
"V-Rising" # game server — optional
|
||||
"Code-Server" # IDE — not needed during pressure events
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── SYSTEM WATCHDOG ───────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Per-host check toggles and NIC config for system_watchdog.sh.
|
||||
# Aliased by detect_hosts() — script uses unprefixed SYS_WATCHDOG_* names.
|
||||
# HOST1: TR1950X 128GB — full media server, active transcoding, ZFS cache pools.
|
||||
#
|
||||
# Three-tier response — all critical checks enabled by default on HOST1:
|
||||
# Tier 1 (bypass strikes, reboot now): docker daemon, rootfs full, kernel oops, FD, /boot
|
||||
# Tier 2 (bypass strikes with OOM): RAM critical + OOM kills in cycle
|
||||
# Tier 3 (standard strike system): everything else
|
||||
#
|
||||
# RAM tiers, OOM limits, and reboot loop settings in master.conf System Watchdog section.
|
||||
|
||||
# ━━━ Primary NIC ━━━
|
||||
# Network interface for NIC state check — verify with: ip link show | grep "^[0-9]"
|
||||
# Common values: eth0, bond0, br0, eno1
|
||||
HOST1_SYS_WATCHDOG_NIC="eth0"
|
||||
|
||||
# ━━━ Tier 1 — Critical Checks ━━━
|
||||
# These bypass the strike system — a single hit triggers immediate reboot.
|
||||
# Disabling any of these is not recommended — they protect against acute system failure.
|
||||
|
||||
# Docker daemon unresponsive → try restart, reboot if restart fails.
|
||||
# Without a working daemon docker_watchdog.sh is blind and containers cannot be managed.
|
||||
HOST1_SYS_WATCHDOG_CHECK_DOCKER_DAEMON=true
|
||||
|
||||
# rootfs at critical threshold (ROOTFS_CRITICAL_PCT=99) → reboot immediately.
|
||||
# At 99% rootfs writes fail silently — logs stop, Docker errors out, SSH may stop working.
|
||||
# Standard 95% threshold still uses strike system — only 99%+ is critical tier.
|
||||
HOST1_SYS_WATCHDOG_CHECK_ROOTFS=true
|
||||
|
||||
# Kernel BUG/Oops in dmesg delta since last cycle → reboot immediately.
|
||||
# A kernel oops means the kernel ran with a corrupted state — stability is not guaranteed.
|
||||
HOST1_SYS_WATCHDOG_CHECK_KERNEL_OOPS=true
|
||||
|
||||
# File descriptor exhaustion at FD_CRITICAL_PCT (95%) → reboot immediately.
|
||||
# At 95% FD: new connections fail, Docker can't spawn processes, SSH drops.
|
||||
HOST1_SYS_WATCHDOG_CHECK_FD=true
|
||||
|
||||
# /boot read-only detected → reboot immediately.
|
||||
# Unexpected read-only /boot means state files and config writes are silently failing.
|
||||
# Fallback state, watchdog reboot log, and lock files all go stale silently.
|
||||
HOST1_SYS_WATCHDOG_CHECK_BOOT=true
|
||||
|
||||
# ━━━ Tier 2 — Urgent OOM Check ━━━
|
||||
# Bypass strikes when RAM is critically low AND OOM kill rate confirms active crisis.
|
||||
# Both must be enabled for Tier 2 bypass to function — disable either to always use strikes.
|
||||
|
||||
# Track kernel OOM kills each cycle via /proc/vmstat oom_kill delta.
|
||||
# Also provides diagnostic context in reboot messages (which processes were killed).
|
||||
HOST1_SYS_WATCHDOG_CHECK_OOM=true
|
||||
|
||||
# Free RAM check — required for both Tier 2 bypass and RAM tier logic.
|
||||
# Tiers: MEM_WARN_GB(10) → notify | MEM_SHUTDOWN_GB(6) → stop containers | MEM_GB(4) → strikes
|
||||
HOST1_SYS_WATCHDOG_CHECK_RAM=true
|
||||
|
||||
# ━━━ Tier 3 — Standard Checks (strike system) ━━━
|
||||
# Each check must fail SYS_WATCHDOG_STRIKE_LIMIT consecutive cycles before action is taken.
|
||||
# Single spikes are ignored — sustained problems trigger reboot.
|
||||
|
||||
# /var/log filesystem usage above SYS_WATCHDOG_LOG_PCT.
|
||||
# Log spam (Docker log storms, syslog loops) fills rootfs — indicates something broken.
|
||||
HOST1_SYS_WATCHDOG_CHECK_LOG=true
|
||||
|
||||
# ZFS ARC memory pinned above SYS_WATCHDOG_ARC_PINNED_PCT after cache drop.
|
||||
# Enabled on HOST1 — ZFS cache pools actively used. Disable on hosts without ZFS.
|
||||
HOST1_SYS_WATCHDOG_CHECK_ARC=true
|
||||
|
||||
# CPU temperature above SYS_WATCHDOG_CPU_TEMP_MAX (95°C).
|
||||
# Sustained high temp causes kernel throttling or panic. Requires lm-sensors.
|
||||
HOST1_SYS_WATCHDOG_CHECK_CPU_TEMP=true
|
||||
|
||||
# Load average above SYS_WATCHDOG_LOAD_MULTIPLIER × core count.
|
||||
# DISABLED on HOST1 — Tdarr and Emby cause legitimate sustained load spikes during encoding.
|
||||
# Enable on idle servers or adjust SYS_WATCHDOG_LOAD_MULTIPLIER if load is always high.
|
||||
HOST1_SYS_WATCHDOG_CHECK_LOAD=false
|
||||
|
||||
# Zombie process count above SYS_WATCHDOG_ZOMBIE_LIMIT (50).
|
||||
# Large zombie counts indicate serious process management failure — something is stuck.
|
||||
HOST1_SYS_WATCHDOG_CHECK_ZOMBIES=true
|
||||
|
||||
# Check docker_watchdog.sh persistent skip list — required containers on skip list.
|
||||
# Cross-watchdog coordination: if docker_watchdog gave up, system_watchdog escalates.
|
||||
# ENABLED — HOST1 fully built and operational, skip list is meaningful.
|
||||
HOST1_SYS_WATCHDOG_CHECK_CONTAINERS=true
|
||||
|
||||
# /tmp filesystem usage above SYS_WATCHDOG_TMP_PCT with auto-clear attempt.
|
||||
# Script tries to clear aged /tmp files first — only strikes if clear fails.
|
||||
# Lock files, rsync temp files, and Docker ops use /tmp — 100% means lock failures.
|
||||
HOST1_SYS_WATCHDOG_CHECK_TMP=true
|
||||
|
||||
# Array disk error count delta in /proc/mdstat — accumulating errors = disk failing now.
|
||||
# Triggers on SYS_WATCHDOG_MDSTAT_ERROR_LIMIT new errors in one cycle.
|
||||
HOST1_SYS_WATCHDOG_CHECK_MDSTAT=true
|
||||
|
||||
# Primary NIC operstate — detects NIC going down (physical or driver failure).
|
||||
# Uses HOST1_SYS_WATCHDOG_NIC above. Strike system — brief flaps don't trigger reboot.
|
||||
HOST1_SYS_WATCHDOG_CHECK_NETWORK=true
|
||||
|
||||
# sshd running check — attempts restart before escalating.
|
||||
# sshd down = no remote access. Script tries rc.sshd start, notifies, strikes on failure.
|
||||
HOST1_SYS_WATCHDOG_CHECK_SSHD=true
|
||||
|
||||
# Runaway process detection — single process above SYS_WATCHDOG_RUNAWAY_CPU_PCT sustained.
|
||||
# DISABLED — Tdarr encoding and Emby transcoding legitimately peg CPU for extended periods.
|
||||
# Enable only if HOST1 has no CPU-intensive workloads.
|
||||
HOST1_SYS_WATCHDOG_CHECK_RUNAWAY=false
|
||||
@@ -0,0 +1,807 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ========================== HOST1 CONFIGURATION — unRAID-Gmer4Lfe ============================
|
||||
# ==============================================================================================
|
||||
# HOST1-specific variables — credentials, container names, share paths, failover lists.
|
||||
# Sourced after master.conf — values here extend shared profile arrays and add HOST1-specific
|
||||
# identity, credentials, and container configuration.
|
||||
#
|
||||
# Sparse checkout (git) ensures HOST2 never receives this file.
|
||||
# HOST2 never sees HOST1 credentials — clean separation at the file level.
|
||||
#
|
||||
# DO NOT put shared config here — thresholds, toggles, profiles belong in master.conf.
|
||||
# DO NOT put HOST2 variables here — they belong in host2.conf.
|
||||
#
|
||||
# ── INDEX ─────────────────────────────────────────────────────────────────────────────────────
|
||||
#
|
||||
# ── IDENTITY & CONNECTIVITY ────────────────────────────────────────────────────────────────
|
||||
# IDENTITY hostname, SSH key, Unraid API key
|
||||
# EMBY container name, URL, API key
|
||||
# JELLYFIN container name, URL, API key
|
||||
# GITEA API token for SSH key registration
|
||||
# NOTIFICATIONS Discord webhook
|
||||
#
|
||||
# ── PARTNERSHIP ────────────────────────────────────────────────────────────────────────────
|
||||
# PARTNERSHIP auth containers, backup paths, emby provisioning
|
||||
#
|
||||
# ── RSYNC ──────────────────────────────────────────────────────────────────────────────────
|
||||
# DAILY SYNC SHARES media shares HOST1 owns and pushes to HOST2
|
||||
# WEEKLY SYNC SHARES appdata shares synced weekly (Sunday 2:30am)
|
||||
# CRITICAL SYNC SHARES appdata shares synced every 30 minutes
|
||||
# BACKUP VERIFY shares for checksum verification against remote
|
||||
# HOST1 RSYNC PROFILE host1-appdata profile for HOST1-specific appdata syncs
|
||||
#
|
||||
# ── FALLBACK ───────────────────────────────────────────────────────────────────────────────
|
||||
# DDNS DDNS containers managed by HOST1
|
||||
# INTERNET LOSS containers stopped when internet is lost
|
||||
# FALLBACK TIERS what HOST1 runs for HOST2 per tier
|
||||
# TIER DELAYS how long HOST1 must be down before each tier activates on HOST2
|
||||
# RSYNC WRITEBACK HOST1 appdata synced back on handback
|
||||
#
|
||||
# ── DOCKER ─────────────────────────────────────────────────────────────────────────────────
|
||||
# DOCKER DAILY RESTART containers restarted daily
|
||||
# DOCKER WEEKLY RESTART containers restarted weekly
|
||||
# DOCKER WATCHDOG memory limits, health URLs, required containers, ignore list
|
||||
# NETWORK WATCHDOG DDNS domain, NPM URL for connectivity checks
|
||||
# DOCKER NETWORK CONNECT networks and containers for docker_network_connect.sh
|
||||
#
|
||||
# ── MEDIA ──────────────────────────────────────────────────────────────────────────────────
|
||||
# MEDIA PERMISSIONS share list for media_shares_permissions.sh
|
||||
# MEDIA CLEANER folder lists for media_cleaner.sh
|
||||
#
|
||||
# ── ARR STACK ──────────────────────────────────────────────────────────────────────────────
|
||||
# DOWNLOADERS slskd, SABnzbd, qBittorrent credentials and URLs
|
||||
# LIDARR URL, API key, path map
|
||||
# SONARR URL, API key, path map
|
||||
# RADARR URL, API key, path map
|
||||
# ARR RECOVERY per-arr recovery toggles
|
||||
#
|
||||
# ── TRANSCODES ─────────────────────────────────────────────────────────────────────────────
|
||||
# TRANSCODES ramdisk size, thresholds, SSD path, server array
|
||||
#
|
||||
# ── MONITORS ───────────────────────────────────────────────────────────────────────────────
|
||||
# CERTIFICATE MONITOR domains checked for SSL expiry
|
||||
# SMART HEALTH drives to skip in SMART monitoring
|
||||
# ZFS REPORT pools to exclude from ZFS health report
|
||||
#
|
||||
# ── RESOURCE MANAGER ───────────────────────────────────────────────────────────────────────
|
||||
# RESOURCE MANAGER containers paused/stopped under memory pressure
|
||||
#
|
||||
# ── SYSTEM WATCHDOG ────────────────────────────────────────────────────────────────────────
|
||||
# SYSTEM WATCHDOG per-host check toggles and NIC configuration
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
# ==============================================================================================
|
||||
# ── STORAGE MODE ──────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Storage mode ━━━
|
||||
# Controls where Varaverk stores scripts, conf, and state files.
|
||||
# true = internal NVMe/SSD — /boot/config/plugins/varaverk (write-safe, git-direct)
|
||||
# false = USB flash boot — /mnt/user/appdata/Varaverk (preserves flash lifetime)
|
||||
HOST1_STORAGE_MODE_INTERNAL=true
|
||||
|
||||
# ==============================================================================================
|
||||
# ── IDENTITY & CONNECTIVITY ───────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Identity ━━━
|
||||
# HOST1 hostname lives in master.conf (not a credential — safe for all servers).
|
||||
# SSH key used for all server-to-server operations — rsync, failover container commands.
|
||||
# Must be in /root/.ssh/ and authorised in HOST2's /root/.ssh/authorized_keys.
|
||||
HOST1_SSH_KEY="/root/.ssh/gmer4lfe_rsync_automation"
|
||||
HOST1_OWNER="gmer4lfe"
|
||||
HOST1_OWNER_EMAIL="gmer4lfe@gmail.com"
|
||||
|
||||
# ━━━ Unraid API ━━━
|
||||
# Used by the Varaverk plugin to query this server's Unraid GraphQL API.
|
||||
# Generate in Unraid: Settings → Management Access → API Keys → + New Key
|
||||
HOST1_UNRAID_API_KEY="1825c3a2e03ea5089974f4da2e171aa2d5907a1dea23cc479c33e492c8ff4dbb"
|
||||
|
||||
# ━━━ Emby ━━━
|
||||
# Referenced by transcode_manager.sh, emby_session_report.sh, emby_database_repair.sh,
|
||||
# weekly_sync_maintenance.sh, and HOST1_TRANSCODE_SERVERS below.
|
||||
# API key: Emby Dashboard → API Keys → + New Key
|
||||
HOST1_EMBY_CONTAINER="Emby"
|
||||
HOST1_EMBY_URL="http://localhost:8096"
|
||||
HOST1_EMBY_API_KEY="0c27448d93a7431f9ac63569f7655829"
|
||||
|
||||
# ━━━ Jellyfin ━━━
|
||||
# API key: Jellyfin Dashboard → Administration → API Keys → + New Key
|
||||
HOST1_JELLYFIN_CONTAINER="Jellyfin"
|
||||
HOST1_JELLYFIN_URL="http://localhost:8095"
|
||||
HOST1_JELLYFIN_API_KEY="4e820e7df74c4933acec212b1996314e"
|
||||
|
||||
# ━━━ Gitea ━━━
|
||||
# Personal access token for gitea_ssh_setup.sh — registers this server's SSH public key
|
||||
# with Gitea so git operations use key auth instead of passwords.
|
||||
# Create in Gitea: Settings → Applications → Generate Token → scope: write:user
|
||||
HOST1_GITEA_API_TOKEN=""
|
||||
|
||||
# ━━━ Notifications ━━━
|
||||
# Discord webhook — leave blank to disable.
|
||||
# Per-host so HOST1 and HOST2 can post to different channels or only one server notifies.
|
||||
HOST1_DISCORD_WEBHOOK=""
|
||||
|
||||
# ==============================================================================================
|
||||
# ── PARTNERSHIP ───────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# HOST1 is always the owner (source of truth) unless --transfer has been run.
|
||||
# See README-Partnership.md and master.conf PARTNERSHIP section for full lifecycle docs.
|
||||
|
||||
# Auth containers reconfigured on onboard/offboard.
|
||||
# Format: "ContainerName|WebUIPort"
|
||||
# On onboard → WebUI pointed at owner's Tailscale IP (mirror clicks NPM, gets owner's auth)
|
||||
# On offboard → WebUI pointed back at localhost
|
||||
HOST1_PARTNERSHIP_AUTH_WEBUIS=(
|
||||
"NginxProxyManager|81"
|
||||
"Lldap-Gmer4Lfe|17170"
|
||||
"Authelia|9091"
|
||||
"Authelia-Secondary|9092"
|
||||
)
|
||||
|
||||
# XML templates (from this server's templates-user/) pushed to mirror during onboard.
|
||||
# These become the mirror's active auth stack, backed by the rsync-synced appdata.
|
||||
# Update filename if Lldap is renamed to drop the host suffix.
|
||||
HOST1_PARTNERSHIP_AUTH_STACK=(
|
||||
# Dependencies first — Mariadb/Redis must be healthy before Authelia starts
|
||||
"my-Mariadb-Authelia.xml"
|
||||
"my-Mariadb-Authelia-Secondary.xml"
|
||||
"my-Redis-Authelia.xml"
|
||||
"my-Redis-Authelia-Secondary.xml"
|
||||
# Auth apps — deployed after their deps are confirmed healthy
|
||||
"my-Authelia.xml"
|
||||
"my-Authelia-Secondary.xml"
|
||||
"my-NginxProxyManager.xml"
|
||||
"my-Lldap-Gmer4Lfe.xml"
|
||||
# Source of truth — must be available on HOST2 independently of the auth stack
|
||||
"my-Gitea.xml"
|
||||
)
|
||||
|
||||
# XML templates pushed to mirror for the arr stack during onboard.
|
||||
# Deps (e.g. databases) first if any — same ordering rule as auth stack.
|
||||
HOST1_PARTNERSHIP_ARR_STACK=(
|
||||
# "my-Sonarr.xml"
|
||||
# "my-Radarr.xml"
|
||||
# "my-Lidarr.xml"
|
||||
# "my-Prowlarr.xml"
|
||||
# "my-Bazarr.xml"
|
||||
)
|
||||
|
||||
# Containers stopped on THIS server before deploying the mirror's stack on onboard.
|
||||
# Only needed when this server parks its own stack to make room for the mirror's.
|
||||
HOST1_PARTNERSHIP_REPLACE_CONTAINERS=(
|
||||
)
|
||||
|
||||
# Arr containers stopped on this server when mirror's arr stack is deployed.
|
||||
HOST1_PARTNERSHIP_ARR_REPLACE_CONTAINERS=(
|
||||
)
|
||||
|
||||
# Paths HOST2 should collect during the grace window after offboard.
|
||||
# Notified on offboard — no auto-deletion, HOST2 must collect manually within PARTNERSHIP_GRACE_HOURS.
|
||||
HOST1_PARTNERSHIP_MIRROR_BACKUPS=(
|
||||
# "/mnt/user/appdata-Fallback/Jayred365-Emby"
|
||||
)
|
||||
|
||||
# Containers parked on this server when partnership is active.
|
||||
# Stopped on onboard (owner deploys its stack instead), restarted on offboard.
|
||||
HOST1_PARTNERSHIP_OWN_CONTAINERS=(
|
||||
# "Emby"
|
||||
# "NginxProxyManager"
|
||||
)
|
||||
|
||||
# Emby admin provisioning — toggle is owner-only, credentials are per-host.
|
||||
# Owner enables/disables the feature. Each host sets the account they want on the shared Emby.
|
||||
# On onboard: owner reads mirror's HOST*_PARTNERSHIP_EMBY_ADMIN_* and creates that account.
|
||||
# On offboard: account is deleted. Username collision → onboard exits with error.
|
||||
HOST1_PARTNERSHIP_PROVISION_EMBY_ADMIN=false # owner controls whether Emby is shared
|
||||
HOST1_PARTNERSHIP_EMBY_PORT=8096
|
||||
HOST1_PARTNERSHIP_EMBY_ADMIN_USER="" # this server's desired Emby username
|
||||
HOST1_PARTNERSHIP_EMBY_ADMIN_PASS="" # this server's desired Emby password
|
||||
|
||||
# ==============================================================================================
|
||||
# ── RSYNC ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Daily Sync Shares ━━━
|
||||
# Shares HOST1 pushes to all other nodes every night (1am via daily_sync_maintenance.sh).
|
||||
# Mesh model: every node pushes every media share — no ownership, no mirrors.
|
||||
# arr_sync ensures all arr libraries converge (union). rsync spreads files (additive, no --delete).
|
||||
# arr_cleanup removes true orphans based on local arr state.
|
||||
# Any node can download content to any share — it propagates to all nodes on the next cycle.
|
||||
# Nextcloud is intentionally one-directional (HOST1→HOST2 offsite backup — not arr-managed).
|
||||
# Uses DEFAULT_RSYNC_OPTS from master.conf — no profile needed.
|
||||
# For shares needing container stops or custom options — add a profile in master.conf.
|
||||
HOST1_DAILY_SYNC_SHARES=(
|
||||
/mnt/user/Books
|
||||
/mnt/user/Intros
|
||||
/mnt/user/Kids_Movies
|
||||
/mnt/user/Kids_Tv_Shows
|
||||
/mnt/user/Movies
|
||||
/mnt/user/Music
|
||||
/mnt/user/Music_Videos
|
||||
/mnt/user/Nextcloud
|
||||
/mnt/user/stand-up_comedy
|
||||
/mnt/user/Sports
|
||||
# /mnt/user/Tv_Shows
|
||||
/mnt/user/Anime_Shows-Old
|
||||
/mnt/user/Anime_Movies-Old
|
||||
/mnt/user/Anime_Movies
|
||||
/mnt/user/Anime_Shows
|
||||
)
|
||||
|
||||
# Personal encrypted shares — synced for offsite backup, independent of media shares.
|
||||
# ZFS encrypted at dataset level — remote receives encrypted blocks, cannot read content.
|
||||
# See README-Rsync_Setup.md for ZFS encryption setup before uncommenting.
|
||||
HOST1_PERSONAL_SHARES=(
|
||||
# /mnt/user/HOST1-Personal # uncomment after creating encrypted dataset
|
||||
)
|
||||
|
||||
# ━━━ Weekly Sync Shares ━━━
|
||||
# Appdata shares synced during the weekly maintenance window (Sunday 2:30am).
|
||||
# Containers stopped both sides before sync — full clean state guaranteed.
|
||||
# Profiles drive container stops, excludes, and options — configured in master.conf RSYNC section.
|
||||
# Order matters — Emby first (larger transfer), then Critical-Data (auth stack).
|
||||
HOST1_WEEKLY_SYNC_SHARES=(
|
||||
"/mnt/user/Media_Server/Emby" # emby profile — full clean mirror
|
||||
"/mnt/user/appdata-Fallback/Critical-Data" # critical-data profile — auth stack
|
||||
)
|
||||
|
||||
# ━━━ Intermediate Sync Shares ━━━
|
||||
# Shares synced every 4 hours by intermediate_sync_maintenance.sh.
|
||||
# Uses DEFAULT_RSYNC_OPTS (no --delete) — for sub-daily propagation of metadata or watch state.
|
||||
# Full media share sync stays in the daily window. Leave empty to skip mid-day rsync entirely.
|
||||
HOST1_INTERMEDIATE_SYNC_SHARES=(
|
||||
# Add shares here to enable mid-day rsync
|
||||
# Example: "/mnt/user/Emby_Metadata"
|
||||
)
|
||||
|
||||
# ━━━ Critical Sync Shares ━━━
|
||||
# Appdata shares synced every 30 minutes by critical_sync_maintenance.sh.
|
||||
# Format: "/path/to/share" or "/path/to/share|profile-name"
|
||||
# Order matters — Critical-Data first (auth stack), then Emby dirty sync.
|
||||
HOST1_CRITICAL_SYNC_SHARES=(
|
||||
"/mnt/user/appdata-Fallback/Critical-Data|critical-fallback" # auth dirty sync — stays running
|
||||
"/mnt/user/Media_Server/Emby|emby-fallback" # Emby dirty sync — stays running
|
||||
)
|
||||
|
||||
# ━━━ Backup Verify ━━━
|
||||
# Shares verified by backup_verify.sh — random file checksum comparison against remote.
|
||||
# Leave empty to use HOST1_DAILY_SYNC_SHARES automatically.
|
||||
# Sample size and minimum file size defined in master.conf.
|
||||
HOST1_BACKUP_VERIFY_SHARES=(
|
||||
# leave empty to use HOST1_DAILY_SYNC_SHARES automatically
|
||||
)
|
||||
|
||||
# ━━━ HOST1 Rsync Profile — host1-appdata ━━━
|
||||
# HOST1-specific appdata sync profile — extends the shared PROFILE_* arrays in master.conf.
|
||||
# Use for appdata unique to HOST1 (Organizrv2, VaultWarden, UptimeKuma etc.)
|
||||
# Shared appdata (auth stack, Emby) use dedicated profiles defined in master.conf.
|
||||
# Run manually: bash Rsync/rsync.sh /mnt/user/appdata-Fallback/HOST1-Appdata --profile=host1-appdata
|
||||
PROFILE_RSYNC_OPTS[host1-appdata]="-av --info=progress2 --bwlimit=${PROFILE_BW_LIMIT[host1-appdata]:-8000}"
|
||||
PROFILE_BW_LIMIT[host1-appdata]=8000
|
||||
PROFILE_RETRY_COUNT[host1-appdata]=3
|
||||
PROFILE_SLEEP[host1-appdata]=300
|
||||
PROFILE_CRITICAL_CONTAINER_NAMES[host1-appdata]="Organizrv2-Gmer4Lfe UptimeKuma-Gmer4Lfe VaultWarden-Gmer4Lfe"
|
||||
PROFILE_DELAYED_CONTAINERS[host1-appdata]=""
|
||||
PROFILE_CONTAINER_DELAY[host1-appdata]=5
|
||||
PROFILE_EXCLUDE_DIRS[host1-appdata]="logs *.tmp"
|
||||
|
||||
# ==============================================================================================
|
||||
# ── FALLBACK ──────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ DDNS ━━━
|
||||
# DDNS containers HOST1 manages — started/stopped by fallback.sh per DDNS absolute rules:
|
||||
# Internet loss → stop immediately
|
||||
# Failover → HOST2 starts HOST1's DDNS as Tier 1 (before any other containers)
|
||||
# Handback → stop HOST1's DDNS on HOST2 → rsync → start containers → start local DDNS last
|
||||
HOST1_DDNS_CONTAINERS=(
|
||||
"Gmer4Lfe.com"
|
||||
)
|
||||
|
||||
# ━━━ Internet Loss ━━━
|
||||
# Containers stopped immediately on HOST1 when internet connection is lost.
|
||||
# Prevents external-facing services from operating without connectivity.
|
||||
FALLBACK_HOST1_STOP_ON_NO_NET=(
|
||||
"Gmer4Lfe.com"
|
||||
)
|
||||
|
||||
# ━━━ Fallback Tiers — HOST1 Runs for HOST2 ━━━
|
||||
# Containers HOST1 starts when HOST2 goes down.
|
||||
# Tier 1 is always immediate — vital services cannot wait.
|
||||
# Higher tiers activate after HOST2_TIER*_DELAY minutes (set in host2.conf).
|
||||
FALLBACK_HOST1_COVERS_HOST2_TIER1=(
|
||||
"Gmer4Lfe.us"
|
||||
"VaultWarden-Jayred365"
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_COVERS_HOST2_TIER2=(
|
||||
# "container-placeholder"
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_COVERS_HOST2_TIER3=(
|
||||
# "container-placeholder"
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_COVERS_HOST2_TIER4=(
|
||||
# "container-placeholder"
|
||||
)
|
||||
|
||||
# ━━━ Tier Delays — HOST1's Containers on HOST2 ━━━
|
||||
# How long HOST1 must be down before each tier activates on HOST2 — in minutes.
|
||||
# Tier 1 is always immediate — no delay var needed.
|
||||
HOST1_TIER2_DELAY=240 # 4 hours — NextCloud, Immich
|
||||
HOST1_TIER3_DELAY=720 # 12 hours — secondary services
|
||||
HOST1_TIER4_DELAY=1440 # 24 hours — arrs + downloaders
|
||||
|
||||
# ━━━ Rsync Writeback — HOST1 Appdata Back on Handback ━━━
|
||||
# Syncs HOST1 appdata BACK to HOST1 when it comes back online after a failover.
|
||||
# Containers stopped before writeback — clean source, no competing writes.
|
||||
#
|
||||
# HOST1_TIER1_WRITEBACK_DELAY: short outages skip Tier 1 writeback — primary state
|
||||
# is more reliable than dirty sync data for brief outages.
|
||||
HOST1_TIER1_WRITEBACK_DELAY=60 # skip Emby writeback if outage under 1hr
|
||||
|
||||
# Tier 4 automatically syncs HOST1_DAILY_SYNC_SHARES — only list paths NOT in that array.
|
||||
FALLBACK_HOST1_WRITEBACK_TIER1=(
|
||||
"/mnt/user/Media_Server/Emby" # watch states built up during outage
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_WRITEBACK_TIER2=(
|
||||
"/mnt/user/appdata-Fallback/Important-Data" # NextCloud + Postgres — files added during outage
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_WRITEBACK_TIER3=(
|
||||
# "location-placeholder"
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_WRITEBACK_TIER4=(
|
||||
"/mnt/user/appdata-Fallback/Arrs_Stack" # arr databases — downloads queued during outage
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── DOCKER ────────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Docker Daily Restart ━━━
|
||||
# Containers restarted every day via DAILY_MAINTENANCE_SCRIPTS.
|
||||
# Dispatcharr degrades over time without restart — daily is intentional, not just housekeeping.
|
||||
# Order matters — auth stack first, then media services.
|
||||
HOST1_DAILY_RESTART_CONTAINERS=(
|
||||
"NginxProxyManager"
|
||||
"Lldap-Gmer4Lfe"
|
||||
"Authelia"
|
||||
"Authelia-Secondary"
|
||||
"Dispatcharr-Iptv-Users"
|
||||
"Dispatcharr" # Live TV scheduler — degrades without daily restart
|
||||
"Dispatcharr-Basic"
|
||||
"ErsatzTV-Emby"
|
||||
"slskd" # Soulseek connection drops after extended uptime; restart refreshes share index
|
||||
)
|
||||
|
||||
# ━━━ Docker Weekly Restart ━━━
|
||||
# Less critical services restarted weekly via WEEKLY_MAINTENANCE_SCRIPTS (Sunday 2:30am).
|
||||
# Containers already stopped for weekly sync — restart adds zero extra downtime.
|
||||
HOST1_WEEKLY_RESTART_CONTAINERS=(
|
||||
"NextCloud"
|
||||
"Organizrv2-Gmer4Lfe"
|
||||
"AdGuard-Home"
|
||||
"Immich-Gmer4Lfe"
|
||||
)
|
||||
|
||||
# ━━━ Docker Watchdog ━━━
|
||||
# Per-HOST1 container configuration for docker_watchdog.sh.
|
||||
# Shared thresholds and toggles live in master.conf.
|
||||
|
||||
# Memory hard limits in MB — immediate restart if exceeded.
|
||||
# Set at "container is clearly broken" not "container is busy".
|
||||
# 20GB=20480 18GB=18432 16GB=16384 12GB=12288 8GB=8192 4GB=4096 2GB=2048 1GB=1024
|
||||
declare -A HOST1_WATCHDOG_CONTAINERS=(
|
||||
["Emby"]=20480 # 20GB — large library + active transcodes
|
||||
["LidaTube"]=6144 # 6GB — memory leak over time
|
||||
["Tdarr"]=6144 # 6GB — encoding is memory intensive
|
||||
["Code-Server"]=1024 # 1GB — should never need more
|
||||
)
|
||||
|
||||
# HTTP health check URLs — checked every cycle, strike system before restart.
|
||||
# Only add containers with a meaningful web interface to check.
|
||||
declare -A HOST1_WATCHDOG_CONTAINER_URLS=(
|
||||
["Emby"]="http://localhost:8096"
|
||||
["NginxProxyManager"]="http://localhost:7818"
|
||||
["Authelia"]="http://localhost:9091/api/health"
|
||||
["Authelia-Secondary"]="http://localhost:9092/api/health"
|
||||
["Lldap-Gmer4Lfe"]="http://localhost:17170"
|
||||
)
|
||||
|
||||
# Required containers — must always be running on HOST1.
|
||||
# Strike system before restart — repeated failures go on skip list, auto-clears on recovery.
|
||||
# Listed in dependency order — dependencies before dependents.
|
||||
HOST1_WATCHDOG_REQUIRED_CONTAINERS=(
|
||||
"NginxProxyManager"
|
||||
"Lldap-Gmer4Lfe"
|
||||
"Mariadb-Authelia"
|
||||
"Mariadb-Authelia-Secondary"
|
||||
"Redis-Authelia"
|
||||
"Redis-Authelia-Secondary"
|
||||
"Authelia"
|
||||
"Authelia-Secondary"
|
||||
)
|
||||
|
||||
# Containers to skip in Tier 2 global scan — legitimately stopped or frequently restarting.
|
||||
# Watchdog leaves these alone entirely — no restart attempts, no crash loop tracking.
|
||||
HOST1_WATCHDOG_SCAN_IGNORE=(
|
||||
"DashGate"
|
||||
"PIA-WG-Config-Generator"
|
||||
"Aperture"
|
||||
"Aperture-Kids"
|
||||
"pgvector-18-Apeture-Kids"
|
||||
"Pgvector18-Aperture"
|
||||
"emby-test" # broken test container (exit 127 — bad image)
|
||||
)
|
||||
|
||||
# Dependency ordering — skip restarting a container if its dependency is also down.
|
||||
# Prevents watchdog from restarting Authelia before Mariadb is back up.
|
||||
# SPACE-SEPARATED STRINGS — converted to array at runtime.
|
||||
declare -A HOST1_WATCHDOG_DEPENDENCIES=(
|
||||
["Authelia"]="Mariadb-Authelia Redis-Authelia"
|
||||
["Authelia-Secondary"]="Mariadb-Authelia Redis-Authelia-Secondary"
|
||||
["NextCloud"]="Postgres-NextCloud"
|
||||
)
|
||||
|
||||
# Per-container appdata growth suppress ceilings in MB.
|
||||
# ONLY needed in specific cases — growth rate detection covers all containers automatically.
|
||||
# Use this when a container legitimately has large stable data and you want to guarantee
|
||||
# it never triggers a false-positive growth alert. Growth warnings are suppressed while the
|
||||
# container's dir stays below this ceiling; above it, warnings resume as normal.
|
||||
# 50GB=51200 25GB=25600 20GB=20480 15GB=15360 10GB=10240 5GB=5120
|
||||
declare -A HOST1_WATCHDOG_APPDATA_SIZES=(
|
||||
["Tdarr"]="25600" # 25GB — transcode cache grows legitimately during active jobs
|
||||
["7dtd"]="20480" # 20GB — game server world data, expected to be large
|
||||
)
|
||||
|
||||
# API-level health checks — checked every cycle alongside HTTP URL checks.
|
||||
# Format: ["ContainerName"]="url|expected_json_key|expected_value"
|
||||
# Empty = no API checks for this host.
|
||||
declare -A HOST1_WATCHDOG_CONTAINER_API_CHECKS=(
|
||||
)
|
||||
|
||||
# ━━━ Network Watchdog ━━━
|
||||
# Host-specific connectivity config for Watchdogs/System/network_watchdog.sh.
|
||||
HOST1_NETWORK_WATCHDOG_DDNS_DOMAIN="gmer4lfe.com"
|
||||
HOST1_NETWORK_WATCHDOG_DDNS_CONTAINER="Gmer4Lfe.com"
|
||||
HOST1_NETWORK_WATCHDOG_NPM_URL="https://gmer4lfe.com"
|
||||
|
||||
# ━━━ Docker Network Connect ━━━
|
||||
# Containers connected to custom networks at array start by docker_network_connect.sh.
|
||||
# Networks created if they don't exist — idempotent, safe to re-run.
|
||||
HOST1_NETWORK_CONNECT_CONTAINERS=(
|
||||
"memcached"
|
||||
"Npm-CrowdSec"
|
||||
)
|
||||
|
||||
HOST1_NETWORK_CONNECT_NETWORKS=(
|
||||
"high-availability"
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── MEDIA ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Media Permissions ━━━
|
||||
# Shares that media_shares_permissions.sh applies PERMISSIONS_MODE and PERMISSIONS_OWNER to.
|
||||
# Runs first in DAILY_MAINTENANCE_SCRIPTS — arr cleanup depends on correct ownership.
|
||||
HOST1_MEDIA_PERMISSION_SHARES=(
|
||||
/mnt/user/Anime_Movies
|
||||
/mnt/user/Anime_Movies-Old
|
||||
/mnt/user/Anime_Shows
|
||||
/mnt/user/Anime_Shows-Old
|
||||
/mnt/user/appcache
|
||||
/mnt/user/Books
|
||||
/mnt/user/Downloads
|
||||
/mnt/user/Games
|
||||
/mnt/user/Intros
|
||||
/mnt/user/Kids_Movies
|
||||
/mnt/user/Kids_Tv_Shows
|
||||
/mnt/user/Movie_Recordings
|
||||
/mnt/user/Movies
|
||||
/mnt/user/Music
|
||||
/mnt/user/Music_Videos
|
||||
/mnt/user/Photo
|
||||
/mnt/user/Sports
|
||||
/mnt/user/stand-up_comedy
|
||||
/mnt/user/Tv_Recordings
|
||||
/mnt/user/Tv_Shows
|
||||
/mnt/user/YouTube
|
||||
)
|
||||
|
||||
# ━━━ Media Cleaner ━━━
|
||||
# Folder lists for media_cleaner.sh — two profiles: anime and media.
|
||||
# File patterns shared across all servers — defined in master.conf.
|
||||
# Called via DAILY_MAINTENANCE_SCRIPTS. Run manually: Media/media_cleaner.sh anime|media
|
||||
HOST1_ANIME_CLEAN_FOLDERS=(
|
||||
/mnt/user/Anime_Movies
|
||||
/mnt/user/Anime_Movies-Old
|
||||
/mnt/user/Anime_Shows
|
||||
/mnt/user/Anime_Shows-Old
|
||||
)
|
||||
|
||||
HOST1_MEDIA_CLEAN_FOLDERS=(
|
||||
/mnt/user/Kids_Movies
|
||||
/mnt/user/Kids_Tv_Shows
|
||||
/mnt/user/Movies
|
||||
/mnt/user/Music
|
||||
/mnt/user/Sports
|
||||
/mnt/user/stand-up_comedy
|
||||
/mnt/user/Tv_Shows
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── ARR STACK ─────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Used by arr cleanup scripts and arrs_failed_stalled_recovery.sh.
|
||||
# detect_hosts() selects HOST1 vars when running on HOST1.
|
||||
#
|
||||
# PATH MAPS — container path → host path translation.
|
||||
# Arr stores file paths using container-internal paths — scripts need host paths to scan.
|
||||
# Add one entry per root folder in arr Settings → Media Management → Root Folders.
|
||||
|
||||
# ━━━ Downloaders ━━━
|
||||
# Used by downloaders_reset.sh — runs every 30min via CRITICAL_MAINTENANCE_SCRIPTS.
|
||||
# Clears stuck states, purges old history, prepares each client for a clean cycle.
|
||||
|
||||
# slskd — clears stuck searches, dead transfers, purges expired failed imports.
|
||||
# SLSKD_FAILED_IMPORTS_DIR: where Soularr moves albums Lidarr rejected.
|
||||
HOST1_SLSKD_URL="http://localhost:8980"
|
||||
HOST1_SLSKD_API_KEY="4bF9kL2mNpQrT7vWxYz1A3dEgHjKoRsU"
|
||||
HOST1_SLSKD_FAILED_IMPORTS_DIR="/mnt/user/Temp_Storage/Slskd/completed/failed_imports"
|
||||
|
||||
# SABnzbd
|
||||
HOST1_SABNZBD_URL="http://localhost:8180"
|
||||
HOST1_SABNZBD_API_KEY="8bfefe41d83b4d50883e32859b55ca9a"
|
||||
|
||||
# qBittorrent — deleteFiles=false removes torrent from qBit but leaves files on disk.
|
||||
# Radarr/Sonarr manage actual files independently.
|
||||
HOST1_QBIT_URL="http://localhost:8080"
|
||||
HOST1_QBIT_USERNAME="root"
|
||||
HOST1_QBIT_PASSWORD="Stay0utD!ck"
|
||||
|
||||
# ━━━ Lidarr — HOST1 only ━━━
|
||||
# HOST2 does not run Lidarr — HOST1_LIDARR_RECOVERY flag handles the exit cleanly.
|
||||
HOST1_LIDARR_URL="http://localhost:8686"
|
||||
HOST1_LIDARR_API_KEY="b2977e71ef074bc0a0529d9fcce3b2dc"
|
||||
HOST1_LIDARR_MUSIC_ROOT="/mnt/user/Music-New"
|
||||
HOST1_FANART_API_KEY="Yd7147a43b692df0b364b94dc47efb81"
|
||||
HOST1_LASTFM_API_KEY="be6dc169c33ae263e690c30d18b7491d"
|
||||
|
||||
declare -A HOST1_LIDARR_PATH_MAP=(
|
||||
["/ext-music"]="/mnt/user/Music-New"
|
||||
)
|
||||
|
||||
# ━━━ Sonarr ━━━
|
||||
HOST1_SONARR_URL="http://localhost:8989"
|
||||
HOST1_SONARR_API_KEY="130decd3db5b4c25afad64864cd03f9f"
|
||||
HOST1_SONARR_TV_ROOT="/mnt/user/Tv_Shows"
|
||||
|
||||
# Note: stand-up_comedy in both Sonarr + Radarr — TV specials and movie specials, one folder
|
||||
declare -A HOST1_SONARR_PATH_MAP=(
|
||||
["/tv"]="/mnt/user/Tv_Shows"
|
||||
["/ext-standup-comedy"]="/mnt/user/stand-up_comedy/series"
|
||||
["/kids tv"]="/mnt/user/Kids_Tv_Shows"
|
||||
["/ext-anime-shows"]="/mnt/user/Anime_Shows-Old"
|
||||
)
|
||||
|
||||
# ━━━ Radarr ━━━
|
||||
HOST1_RADARR_URL="http://localhost:7878"
|
||||
HOST1_RADARR_API_KEY="d43a3ec6cf1549edb4af0cc63f98b2a9"
|
||||
HOST1_TMDB_API_KEY="3dac5e2e49b5540472d2eafec4f01260"
|
||||
HOST1_RADARR_MOVIES_ROOT="/mnt/user/Movies"
|
||||
|
||||
# Note: stand-up_comedy in both Radarr + Sonarr — movie specials and TV specials, one folder
|
||||
declare -A HOST1_RADARR_PATH_MAP=(
|
||||
["/movies"]="/mnt/user/Movies"
|
||||
["/kids movies"]="/mnt/user/Kids_Movies"
|
||||
["/ext-stand-up-comedy"]="/mnt/user/stand-up_comedy/specials"
|
||||
["/anime-movies"]="/mnt/user/Anime_Movies-Old"
|
||||
["/ext-anime-movies"]="/mnt/user/Anime_Movies-Old"
|
||||
)
|
||||
|
||||
# ━━━ Arr Recovery Toggles ━━━
|
||||
# false = skip that arr on this host — exits cleanly without error
|
||||
HOST1_SONARR_RECOVERY=true
|
||||
HOST1_RADARR_RECOVERY=true
|
||||
HOST1_LIDARR_RECOVERY=true # HOST1 only — exits cleanly on HOST2
|
||||
|
||||
# ==============================================================================================
|
||||
# ── TRANSCODES ────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# Ramdisk size ceiling — tmpfs only uses RAM actually needed, not the full size upfront.
|
||||
# Real-world: 9 streams peaked at ~5.5GB — 10G gives generous headroom on 128GB RAM.
|
||||
HOST1_RAMDISK_SIZE="10G"
|
||||
|
||||
# Usage thresholds — coupled to HOST1_RAMDISK_SIZE, adjust all three together if size changes.
|
||||
# Hysteresis gap (8.5 - 7 = 1.5GB) prevents flip-flop between ramdisk and SSD.
|
||||
HOST1_RAMDISK_WARN_GB=8.5 # flip to SSD when ramdisk usage reaches this
|
||||
HOST1_RAMDISK_LOW_GB=7 # flip back to ramdisk when usage drops to this
|
||||
|
||||
# SSD fallback path — where transcodes land when ramdisk exceeds HOST1_RAMDISK_WARN_GB.
|
||||
# Must be on cache pool — array disks too slow for active transcode writes.
|
||||
HOST1_TRANSCODE_SSD="/mnt/cache/Temp_Storage/Emby/Transcodes/"
|
||||
|
||||
# Media servers sharing the ramdisk transcode space on HOST1.
|
||||
# Format: "ContainerName|URL|APIKey|Type" — Type: emby | jellyfin | plex
|
||||
# Entries with placeholder API keys are skipped automatically.
|
||||
# ⚠️ Tdarr does NOT belong here — keep Tdarr on SSD, not ramdisk.
|
||||
HOST1_TRANSCODE_SERVERS=(
|
||||
"${HOST1_EMBY_CONTAINER}|${HOST1_EMBY_URL}|${HOST1_EMBY_API_KEY}|emby"
|
||||
"${HOST1_JELLYFIN_CONTAINER}|${HOST1_JELLYFIN_URL}|${HOST1_JELLYFIN_API_KEY}|jellyfin"
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── MONITORS ──────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Certificate Monitor ━━━
|
||||
# Domains checked via direct openssl connection — not relying on NPM's certificate state.
|
||||
# Checks the actual certificate served, not what NPM thinks it has.
|
||||
# Thresholds (CERT_WARN_DAYS, CERT_CRIT_DAYS) defined in master.conf.
|
||||
HOST1_CERT_MONITOR_DOMAINS=(
|
||||
"Gmer4Lfe.com"
|
||||
"Gmer4Lfe.us"
|
||||
)
|
||||
|
||||
# ━━━ SMART Health ━━━
|
||||
# Drives skipped in SMART attribute monitoring — hardware is server-specific.
|
||||
# Thresholds read from dynamix.cfg at runtime — fallbacks in master.conf.
|
||||
HOST1_SMART_IGNORE_DRIVES=(
|
||||
"sda" # boot USB — SMART not meaningful on flash drives
|
||||
)
|
||||
|
||||
# ━━━ ZFS Report ━━━
|
||||
# Pools excluded from the weekly ZFS health report — reduces noise from single-disk array pools.
|
||||
# These are individual array disks formatted as ZFS — converting to XFS over time via unBalance.
|
||||
# Pool health thresholds defined in master.conf.
|
||||
HOST1_ZFS_REPORT_IGNORE_POOLS=(
|
||||
"disk5"
|
||||
"disk6"
|
||||
"disk8"
|
||||
"disk9"
|
||||
"disk10"
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── RESOURCE MANAGER ──────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Containers to manage under pressure — see master.conf RW_CRITICAL_CONTAINERS for exclusions.
|
||||
|
||||
# docker pause at medium pressure (RAM < RW_RAM_MEDIUM_GB or load > medium threshold)
|
||||
# Suspended in-place — instant to pause/unpause, no state lost, no restart delay.
|
||||
HOST1_RW_PAUSE_CONTAINERS=(
|
||||
"Huntarr" # arr search automation — safe to suspend
|
||||
"Cleanuparr" # download cleanup — safe to suspend
|
||||
"Healarr" # arr health checks — safe to suspend
|
||||
"Soularr" # Slskd automation — background only
|
||||
"ChannelTube" # YouTube archiver — background only
|
||||
"Pinchflat" # YouTube archiver — background only
|
||||
)
|
||||
|
||||
# docker stop at hard pressure (RAM < RW_RAM_HARD_GB)
|
||||
# Full stop — these are optional/heavy services that free significant RAM when stopped.
|
||||
# resource_watchdog.sh restarts them when pressure fully clears (RAM >= RW_RAM_RECOVER_GB).
|
||||
HOST1_RW_STOP_CONTAINERS=(
|
||||
"LocalAI" # GPU/CPU heavy — largest RAM consumer when idle
|
||||
"7DaysToDie" # game server — optional
|
||||
"V-Rising" # game server — optional
|
||||
"Code-Server" # IDE — not needed during pressure events
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── SYSTEM WATCHDOG ───────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Per-host check toggles and NIC config for system_watchdog.sh.
|
||||
# Aliased by detect_hosts() — script uses unprefixed SYS_WATCHDOG_* names.
|
||||
# HOST1: TR1950X 128GB — full media server, active transcoding, ZFS cache pools.
|
||||
#
|
||||
# Three-tier response — all critical checks enabled by default on HOST1:
|
||||
# Tier 1 (bypass strikes, reboot now): docker daemon, rootfs full, kernel oops, FD, /boot
|
||||
# Tier 2 (bypass strikes with OOM): RAM critical + OOM kills in cycle
|
||||
# Tier 3 (standard strike system): everything else
|
||||
#
|
||||
# RAM tiers, OOM limits, and reboot loop settings in master.conf System Watchdog section.
|
||||
|
||||
# ━━━ Primary NIC ━━━
|
||||
# Network interface for NIC state check — verify with: ip link show | grep "^[0-9]"
|
||||
# Common values: eth0, bond0, br0, eno1
|
||||
HOST1_SYS_WATCHDOG_NIC="eth0"
|
||||
|
||||
# ━━━ Tier 1 — Critical Checks ━━━
|
||||
# These bypass the strike system — a single hit triggers immediate reboot.
|
||||
# Disabling any of these is not recommended — they protect against acute system failure.
|
||||
|
||||
# Docker daemon unresponsive → try restart, reboot if restart fails.
|
||||
# Without a working daemon docker_watchdog.sh is blind and containers cannot be managed.
|
||||
HOST1_SYS_WATCHDOG_CHECK_DOCKER_DAEMON=true
|
||||
|
||||
# rootfs at critical threshold (ROOTFS_CRITICAL_PCT=99) → reboot immediately.
|
||||
# At 99% rootfs writes fail silently — logs stop, Docker errors out, SSH may stop working.
|
||||
# Standard 95% threshold still uses strike system — only 99%+ is critical tier.
|
||||
HOST1_SYS_WATCHDOG_CHECK_ROOTFS=true
|
||||
|
||||
# Kernel BUG/Oops in dmesg delta since last cycle → reboot immediately.
|
||||
# A kernel oops means the kernel ran with a corrupted state — stability is not guaranteed.
|
||||
HOST1_SYS_WATCHDOG_CHECK_KERNEL_OOPS=true
|
||||
|
||||
# File descriptor exhaustion at FD_CRITICAL_PCT (95%) → reboot immediately.
|
||||
# At 95% FD: new connections fail, Docker can't spawn processes, SSH drops.
|
||||
HOST1_SYS_WATCHDOG_CHECK_FD=true
|
||||
|
||||
# /boot read-only detected → reboot immediately.
|
||||
# Unexpected read-only /boot means state files and config writes are silently failing.
|
||||
# Fallback state, watchdog reboot log, and lock files all go stale silently.
|
||||
HOST1_SYS_WATCHDOG_CHECK_BOOT=true
|
||||
|
||||
# ━━━ Tier 2 — Urgent OOM Check ━━━
|
||||
# Bypass strikes when RAM is critically low AND OOM kill rate confirms active crisis.
|
||||
# Both must be enabled for Tier 2 bypass to function — disable either to always use strikes.
|
||||
|
||||
# Track kernel OOM kills each cycle via /proc/vmstat oom_kill delta.
|
||||
# Also provides diagnostic context in reboot messages (which processes were killed).
|
||||
HOST1_SYS_WATCHDOG_CHECK_OOM=true
|
||||
|
||||
# Free RAM check — required for both Tier 2 bypass and RAM tier logic.
|
||||
# Tiers: MEM_WARN_GB(10) → notify | MEM_SHUTDOWN_GB(6) → stop containers | MEM_GB(4) → strikes
|
||||
HOST1_SYS_WATCHDOG_CHECK_RAM=true
|
||||
|
||||
# ━━━ Tier 3 — Standard Checks (strike system) ━━━
|
||||
# Each check must fail SYS_WATCHDOG_STRIKE_LIMIT consecutive cycles before action is taken.
|
||||
# Single spikes are ignored — sustained problems trigger reboot.
|
||||
|
||||
# /var/log filesystem usage above SYS_WATCHDOG_LOG_PCT.
|
||||
# Log spam (Docker log storms, syslog loops) fills rootfs — indicates something broken.
|
||||
HOST1_SYS_WATCHDOG_CHECK_LOG=true
|
||||
|
||||
# ZFS ARC memory pinned above SYS_WATCHDOG_ARC_PINNED_PCT after cache drop.
|
||||
# Enabled on HOST1 — ZFS cache pools actively used. Disable on hosts without ZFS.
|
||||
HOST1_SYS_WATCHDOG_CHECK_ARC=true
|
||||
|
||||
# CPU temperature above SYS_WATCHDOG_CPU_TEMP_MAX (95°C).
|
||||
# Sustained high temp causes kernel throttling or panic. Requires lm-sensors.
|
||||
HOST1_SYS_WATCHDOG_CHECK_CPU_TEMP=true
|
||||
|
||||
# Load average above SYS_WATCHDOG_LOAD_MULTIPLIER × core count.
|
||||
# DISABLED on HOST1 — Tdarr and Emby cause legitimate sustained load spikes during encoding.
|
||||
# Enable on idle servers or adjust SYS_WATCHDOG_LOAD_MULTIPLIER if load is always high.
|
||||
HOST1_SYS_WATCHDOG_CHECK_LOAD=false
|
||||
|
||||
# Zombie process count above SYS_WATCHDOG_ZOMBIE_LIMIT (50).
|
||||
# Large zombie counts indicate serious process management failure — something is stuck.
|
||||
HOST1_SYS_WATCHDOG_CHECK_ZOMBIES=true
|
||||
|
||||
# Check docker_watchdog.sh persistent skip list — required containers on skip list.
|
||||
# Cross-watchdog coordination: if docker_watchdog gave up, system_watchdog escalates.
|
||||
# ENABLED — HOST1 fully built and operational, skip list is meaningful.
|
||||
HOST1_SYS_WATCHDOG_CHECK_CONTAINERS=true
|
||||
|
||||
# /tmp filesystem usage above SYS_WATCHDOG_TMP_PCT with auto-clear attempt.
|
||||
# Script tries to clear aged /tmp files first — only strikes if clear fails.
|
||||
# Lock files, rsync temp files, and Docker ops use /tmp — 100% means lock failures.
|
||||
HOST1_SYS_WATCHDOG_CHECK_TMP=true
|
||||
|
||||
# Array disk error count delta in /proc/mdstat — accumulating errors = disk failing now.
|
||||
# Triggers on SYS_WATCHDOG_MDSTAT_ERROR_LIMIT new errors in one cycle.
|
||||
HOST1_SYS_WATCHDOG_CHECK_MDSTAT=true
|
||||
|
||||
# Primary NIC operstate — detects NIC going down (physical or driver failure).
|
||||
# Uses HOST1_SYS_WATCHDOG_NIC above. Strike system — brief flaps don't trigger reboot.
|
||||
HOST1_SYS_WATCHDOG_CHECK_NETWORK=true
|
||||
|
||||
# sshd running check — attempts restart before escalating.
|
||||
# sshd down = no remote access. Script tries rc.sshd start, notifies, strikes on failure.
|
||||
HOST1_SYS_WATCHDOG_CHECK_SSHD=true
|
||||
|
||||
# Runaway process detection — single process above SYS_WATCHDOG_RUNAWAY_CPU_PCT sustained.
|
||||
# DISABLED — Tdarr encoding and Emby transcoding legitimately peg CPU for extended periods.
|
||||
# Enable only if HOST1 has no CPU-intensive workloads.
|
||||
HOST1_SYS_WATCHDOG_CHECK_RUNAWAY=false
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
+333
@@ -0,0 +1,333 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ================================= Git Pull & Execute =========================================
|
||||
# ==============================================================================================
|
||||
# Pulls the latest scripts from the Gitea repository via SSH.
|
||||
# Lives at the repo root — sources load_config.sh from the same directory.
|
||||
#
|
||||
# ── WHAT THIS SCRIPT DOES ─────────────────────────────────────────────────────────────────────
|
||||
# 1. Detects which server it's running on via detect_hosts() (MY_ID)
|
||||
# 2. Configures sparse checkout to exclude other servers' credential files
|
||||
# Each server only pulls its own host*.conf — never sees peer credentials
|
||||
# 3. Pulls or clones latest scripts from Gitea
|
||||
# 4. Sets executable permissions on all .sh files
|
||||
#
|
||||
# ── SPARSE CHECKOUT ───────────────────────────────────────────────────────────────────────────
|
||||
# Sparse checkout ensures each server only receives its own host conf:
|
||||
# HOST1 pulls: master.conf + host1.conf + all scripts
|
||||
# HOST1 skips: host2.conf, host3.conf etc.
|
||||
# HOST2 pulls: master.conf + host2.conf + all scripts
|
||||
# HOST2 skips: host1.conf, host3.conf etc.
|
||||
#
|
||||
# Adding a new server:
|
||||
# Create host3.conf in the repo
|
||||
# All existing servers automatically exclude it on next pull
|
||||
# New server gets only its own conf ✅
|
||||
#
|
||||
# ── GITEA LOCATION DETECTION ──────────────────────────────────────────────────────────────────
|
||||
# Detects where Gitea is running at runtime — works through fallback:
|
||||
# Gitea local → connects via local IP
|
||||
# Gitea remote → connects via Tailscale IP
|
||||
# Both fail → falls back to GITEA_DOMAIN if configured
|
||||
#
|
||||
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||||
# GITEA_CONTAINER — Docker container name for Gitea
|
||||
# GITEA_REPO_PATH — repo path on Gitea (e.g. Varaverk/varaverk.git)
|
||||
# GITEA_DOMAIN — public domain fallback (optional)
|
||||
# TARGET_DIR — local path to clone/pull into
|
||||
# GITEA_SSH_KEY — SSH key for Gitea authentication
|
||||
# SSH_PORT — Gitea SSH port (often 221 or 222)
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# git_pull_execute.sh — normal pull
|
||||
# git_pull_execute.sh --dry-run — preview without making changes
|
||||
# git_pull_execute.sh --log — verbose output
|
||||
# git_pull_execute.sh --status — show config and exit
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
# Root level script — load_config.sh is in the same directory
|
||||
source "$SCRIPT_DIR/load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
acquire_lock
|
||||
|
||||
if ! command -v docker &>/dev/null; then
|
||||
error "Docker command not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# detect_hosts() sets MY_ID — needed for sparse checkout configuration
|
||||
detect_hosts
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Locate Gitea ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_CONTAINERS Locate Gitea ━━━"
|
||||
|
||||
if docker ps --format "{{.Names}}" 2>/dev/null | grep -q "^${GITEA_CONTAINER}$"; then
|
||||
# Gitea is running on this server — use local IP
|
||||
GITEA_IP=$(hostname -I | awk '{print $1}')
|
||||
log "Gitea running locally — connecting via $GITEA_IP"
|
||||
else
|
||||
# Gitea not running locally — find it on the remote server via Tailscale
|
||||
log "Gitea not running locally — checking remote server"
|
||||
GITEA_IP=$(resolve_tailscale_ip "${REMOTE_SERVER_NAME}")
|
||||
if [[ -n "$GITEA_IP" ]]; then
|
||||
echo " Gitea on $REMOTE_SERVER_NAME — connecting via Tailscale $GITEA_IP"
|
||||
elif [[ -n "${GITEA_DOMAIN:-}" ]]; then
|
||||
warn "Tailscale resolution failed — falling back to $GITEA_DOMAIN"
|
||||
GITEA_IP="$GITEA_DOMAIN"
|
||||
else
|
||||
error "Cannot find Gitea — local: not running, Tailscale: failed, domain: not configured"
|
||||
notify "Git pull failed on $(hostname) — cannot locate Gitea container" "Git Sync" "alert"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
REPO_SSH="git@${GITEA_IP}:${GITEA_REPO_PATH}"
|
||||
|
||||
require_var REPO_SSH
|
||||
require_var TARGET_DIR
|
||||
require_var GITEA_SSH_KEY
|
||||
require_var SSH_PORT
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_NET Repo: $REPO_SSH"
|
||||
echo "$ICON_GEAR Target: $TARGET_DIR"
|
||||
echo "$ICON_GEAR SSH Key: $GITEA_SSH_KEY"
|
||||
echo "$ICON_GEAR SSH Port: $SSH_PORT"
|
||||
echo "$ICON_HOST My ID: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_HOST Remote ID: $REMOTE_ID ($REMOTE_SERVER_NAME)"
|
||||
echo "$ICON_NOTIFY Notify: unRAID=${NOTIFY_UNRAID:-false} Discord=$([[ -n "${MY_DISCORD_WEBHOOK:-}" ]] && echo enabled || echo disabled)"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Sparse Checkout Configuration ━━━
|
||||
# ==============================================================================================
|
||||
# Build the list of host*.conf files that belong to OTHER servers.
|
||||
# This server pulls everything EXCEPT those files.
|
||||
# MY_ID is set by detect_hosts() — e.g. "HOST1"
|
||||
|
||||
configure_sparse_checkout() {
|
||||
local repo_dir="$1"
|
||||
|
||||
log "Configuring sparse checkout for $MY_ID..."
|
||||
|
||||
# Enable sparse checkout
|
||||
git -C "$repo_dir" config core.sparseCheckout true 2>/dev/null
|
||||
|
||||
# Build exclusion list — all host*.conf files except MY_ID's
|
||||
local sparse_file="$repo_dir/.git/info/sparse-checkout"
|
||||
mkdir -p "$(dirname "$sparse_file")"
|
||||
|
||||
# Start with: pull everything
|
||||
echo "/*" > "$sparse_file"
|
||||
|
||||
# Exclude each other server's conf file
|
||||
# Find all host*.conf files present in the repo
|
||||
local excluded=0
|
||||
for conf_file in "$repo_dir"/host*.conf; do
|
||||
[[ -f "$conf_file" ]] || continue
|
||||
local conf_name
|
||||
conf_name=$(basename "$conf_file")
|
||||
|
||||
# Determine which HOST ID owns this conf by grepping its hostname var
|
||||
# Pattern: HOST1="unRAID-..." or HOST2="unRAID-..."
|
||||
local conf_host_id
|
||||
conf_host_id=$(grep -m1 -oP '^\s+HOST[0-9]+(?==)' "$conf_file" 2>/dev/null | tr -d ' ')
|
||||
|
||||
if [[ -z "$conf_host_id" ]]; then
|
||||
log "Cannot determine HOST ID for $conf_name — including in pull (safe default)"
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ "$conf_host_id" != "$MY_ID" ]]; then
|
||||
echo "!$conf_name" >> "$sparse_file"
|
||||
log "Sparse checkout: excluding $conf_name (belongs to $conf_host_id)"
|
||||
((excluded++))
|
||||
else
|
||||
log "Sparse checkout: including $conf_name (belongs to $MY_ID — this server)"
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ "$excluded" -gt 0 ]]; then
|
||||
echo " Sparse checkout: excluding $excluded peer conf file(s) — credentials protected"
|
||||
else
|
||||
log "Sparse checkout: no peer conf files to exclude (single server or first run)"
|
||||
fi
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Git Sync ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Git Sync ━━━"
|
||||
echo "$ICON_NET Repo: $REPO_SSH"
|
||||
echo "$ICON_GEAR Target: $TARGET_DIR"
|
||||
echo ""
|
||||
|
||||
START=$(date +%s)
|
||||
SYNC_SUCCESS=false
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would sync $REPO_SSH → $TARGET_DIR"
|
||||
warn "DRY RUN — would configure sparse checkout for $MY_ID"
|
||||
warn "DRY RUN — would exclude peer host*.conf files"
|
||||
SYNC_SUCCESS=true
|
||||
else
|
||||
mkdir -p "$TARGET_DIR"
|
||||
git config --global --add safe.directory "$TARGET_DIR"
|
||||
cd "$TARGET_DIR" || { error "Cannot cd into $TARGET_DIR"; exit 1; }
|
||||
|
||||
if [[ -d ".git" ]]; then
|
||||
# ── Existing repository ──────────────────────────────────────────────
|
||||
echo " Existing repository — updating"
|
||||
|
||||
# Configure sparse checkout BEFORE pull
|
||||
# Uses conf files already present from last pull to determine exclusions
|
||||
configure_sparse_checkout "$TARGET_DIR"
|
||||
|
||||
echo " Pulling latest changes..."
|
||||
if GIT_SSH_COMMAND="ssh -i $GITEA_SSH_KEY -p $SSH_PORT" git pull --ff-only; then
|
||||
echo " Git pull successful"
|
||||
SYNC_SUCCESS=true
|
||||
else
|
||||
# ff-only fails when local commits or tracked changes exist that can't
|
||||
# fast-forward. Fail loudly — never silently destroy local work.
|
||||
error "Git pull failed — local changes conflict with remote (will not force-reset)"
|
||||
notify "Git pull failed on $(hostname) — local changes conflict, manual resolve needed" "Git Sync" "alert"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
else
|
||||
# ── Fresh clone ──────────────────────────────────────────────────────
|
||||
echo " No repository found — cloning"
|
||||
|
||||
# Clone first — need the repo to exist before configuring sparse checkout
|
||||
if GIT_SSH_COMMAND="ssh -i $GITEA_SSH_KEY -p $SSH_PORT" git clone "$REPO_SSH" .; then
|
||||
echo " Clone successful"
|
||||
|
||||
# Configure sparse checkout after clone
|
||||
# Now all host*.conf files are present — can detect exclusions
|
||||
configure_sparse_checkout "$TARGET_DIR"
|
||||
|
||||
# Apply sparse checkout — removes excluded files from working tree
|
||||
echo " Applying sparse checkout..."
|
||||
git read-tree -mu HEAD
|
||||
echo " Sparse checkout applied — peer credentials removed from working tree"
|
||||
|
||||
SYNC_SUCCESS=true
|
||||
else
|
||||
error "Clone failed"
|
||||
notify "Git clone failed on $(hostname) — check Gitea connectivity" "Git Sync" "alert"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Permissions ──────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Permissions ━━━"
|
||||
log "Setting executable permissions on all .sh files..."
|
||||
find "$TARGET_DIR" -type f -name "*.sh" -exec chmod +x {} \;
|
||||
echo " Permissions set on .sh files"
|
||||
|
||||
# ── Flash mode: sync Plugin/ to /boot/ so the webUI picks up updates ─────
|
||||
# In flash mode SCRIPTS_DIR is in appdata — Plugin/ lives in the repo there
|
||||
# but Unraid serves PHP from /boot/. Sync after every pull to keep them in step.
|
||||
_BOOT_DIR="/boot/config/plugins/varaverk"
|
||||
if [[ "$TARGET_DIR" != "$_BOOT_DIR" ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Flash mode: sync Plugin/ → /boot/ ━━━"
|
||||
if rsync -a --delete "$TARGET_DIR/Plugin/" "$_BOOT_DIR/Plugin/" 2>/dev/null; then
|
||||
echo " Plugin/ synced to /boot/ ✅"
|
||||
else
|
||||
warn "Plugin/ sync to /boot/ failed — webUI may be stale until next pull"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Conf Upgrade ━━━
|
||||
# ==============================================================================================
|
||||
# Merges new conf structure into the live conf files after every pull.
|
||||
# New keys → added with template defaults (user fills in once).
|
||||
# Removed keys → dropped. Existing values → always preserved.
|
||||
# Silent when already up to date — no overhead on unchanged pulls.
|
||||
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Conf Upgrade ━━━"
|
||||
|
||||
UPGRADE_SCRIPT="$TARGET_DIR/Deployment/conf_upgrade.sh"
|
||||
CONF_DIR="$TARGET_DIR/Configurations"
|
||||
|
||||
if [[ ! -f "$UPGRADE_SCRIPT" ]]; then
|
||||
log "conf_upgrade.sh not found — skipping (pre-deployment-folder repo)"
|
||||
elif [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would upgrade master.conf and ${MY_ID,,}.conf"
|
||||
elif [[ "$SYNC_SUCCESS" == true ]]; then
|
||||
_DRY=""
|
||||
|
||||
# master.conf
|
||||
if [[ -f "$CONF_DIR/master.conf.template" && -f "$CONF_DIR/master.conf" ]]; then
|
||||
bash "$UPGRADE_SCRIPT" \
|
||||
--template "$CONF_DIR/master.conf.template" \
|
||||
--target "$CONF_DIR/master.conf" \
|
||||
--backup $_DRY
|
||||
else
|
||||
warn "master.conf.template or master.conf not found — skipping"
|
||||
fi
|
||||
|
||||
# This server's host conf only — sparse checkout ensures we have it
|
||||
HOST_CONF="$CONF_DIR/${MY_ID,,}.conf"
|
||||
if [[ -f "$CONF_DIR/host.conf.template" && -f "$HOST_CONF" ]]; then
|
||||
bash "$UPGRADE_SCRIPT" \
|
||||
--template "$CONF_DIR/host.conf.template" \
|
||||
--target "$HOST_CONF" \
|
||||
--backup $_DRY
|
||||
else
|
||||
warn "${MY_ID,,}.conf or host.conf.template not found — skipping"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY GIT SYNC SUMMARY ━━━━━"
|
||||
echo "$ICON_NET Repo: $REPO_SSH"
|
||||
echo "$ICON_GEAR Target: $TARGET_DIR"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_LOCK Excluded: peer host*.conf files"
|
||||
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "$ICON_WARN Status: DRY RUN — no changes made"
|
||||
elif [[ "$SYNC_SUCCESS" == true ]]; then
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS DONE"
|
||||
notify "Repository synced successfully on $(hostname)" "Git Sync" "normal"
|
||||
else
|
||||
echo "$ICON_ERROR Status: $ICON_ERROR FAILED"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
+338
@@ -0,0 +1,338 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ================================= Git Pull & Execute =========================================
|
||||
# ==============================================================================================
|
||||
# Pulls the latest scripts from the Gitea repository via SSH.
|
||||
# Lives at the repo root — sources load_config.sh from the same directory.
|
||||
#
|
||||
# ── WHAT THIS SCRIPT DOES ─────────────────────────────────────────────────────────────────────
|
||||
# 1. Detects which server it's running on via detect_hosts() (MY_ID)
|
||||
# 2. Configures sparse checkout to exclude other servers' credential files
|
||||
# Each server only pulls its own host*.conf — never sees peer credentials
|
||||
# 3. Pulls or clones latest scripts from Gitea
|
||||
# 4. Sets executable permissions on all .sh files
|
||||
#
|
||||
# ── SPARSE CHECKOUT ───────────────────────────────────────────────────────────────────────────
|
||||
# Sparse checkout ensures each server only receives its own host conf:
|
||||
# HOST1 pulls: master.conf + host1.conf + all scripts
|
||||
# HOST1 skips: host2.conf, host3.conf etc.
|
||||
# HOST2 pulls: master.conf + host2.conf + all scripts
|
||||
# HOST2 skips: host1.conf, host3.conf etc.
|
||||
#
|
||||
# Adding a new server:
|
||||
# Create host3.conf in the repo
|
||||
# All existing servers automatically exclude it on next pull
|
||||
# New server gets only its own conf ✅
|
||||
#
|
||||
# ── GITEA LOCATION DETECTION ──────────────────────────────────────────────────────────────────
|
||||
# Detects where Gitea is running at runtime — works through fallback:
|
||||
# Gitea local → connects via local IP
|
||||
# Gitea remote → connects via Tailscale IP
|
||||
# Both fail → falls back to GITEA_DOMAIN if configured
|
||||
#
|
||||
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||||
# GITEA_CONTAINER — Docker container name for Gitea
|
||||
# GITEA_REPO_PATH — repo path on Gitea (e.g. Varaverk/varaverk.git)
|
||||
# GITEA_DOMAIN — public domain fallback (optional)
|
||||
# TARGET_DIR — local path to clone/pull into
|
||||
# GITEA_SSH_KEY — SSH key for Gitea authentication
|
||||
# SSH_PORT — Gitea SSH port (often 221 or 222)
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# git_pull_execute.sh — normal pull
|
||||
# git_pull_execute.sh --dry-run — preview without making changes
|
||||
# git_pull_execute.sh --log — verbose output
|
||||
# git_pull_execute.sh --status — show config and exit
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
# Root level script — load_config.sh is in the same directory
|
||||
source "$SCRIPT_DIR/load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
acquire_lock
|
||||
|
||||
if ! command -v docker &>/dev/null; then
|
||||
error "Docker command not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# detect_hosts() sets MY_ID — needed for sparse checkout configuration
|
||||
detect_hosts
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Locate Gitea ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_CONTAINERS Locate Gitea ━━━"
|
||||
|
||||
if docker ps --format "{{.Names}}" 2>/dev/null | grep -q "^${GITEA_CONTAINER}$"; then
|
||||
# Gitea is running on this server — use local IP
|
||||
GITEA_IP=$(hostname -I | awk '{print $1}')
|
||||
log "Gitea running locally — connecting via $GITEA_IP"
|
||||
else
|
||||
# Gitea not running locally — find it on the remote server via Tailscale
|
||||
log "Gitea not running locally — checking remote server"
|
||||
GITEA_IP=$(resolve_tailscale_ip "${REMOTE_SERVER_NAME}")
|
||||
if [[ -n "$GITEA_IP" ]]; then
|
||||
echo " Gitea on $REMOTE_SERVER_NAME — connecting via Tailscale $GITEA_IP"
|
||||
elif [[ -n "${GITEA_DOMAIN:-}" ]]; then
|
||||
warn "Tailscale resolution failed — falling back to $GITEA_DOMAIN"
|
||||
GITEA_IP="$GITEA_DOMAIN"
|
||||
else
|
||||
error "Cannot find Gitea — local: not running, Tailscale: failed, domain: not configured"
|
||||
notify "Git pull failed on $(hostname) — cannot locate Gitea container" "Git Sync" "alert"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
REPO_SSH="git@${GITEA_IP}:${GITEA_REPO_PATH}"
|
||||
|
||||
require_var REPO_SSH
|
||||
require_var TARGET_DIR
|
||||
require_var GITEA_SSH_KEY
|
||||
require_var SSH_PORT
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_NET Repo: $REPO_SSH"
|
||||
echo "$ICON_GEAR Target: $TARGET_DIR"
|
||||
echo "$ICON_GEAR SSH Key: $GITEA_SSH_KEY"
|
||||
echo "$ICON_GEAR SSH Port: $SSH_PORT"
|
||||
echo "$ICON_HOST My ID: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_HOST Remote ID: $REMOTE_ID ($REMOTE_SERVER_NAME)"
|
||||
echo "$ICON_NOTIFY Notify: unRAID=${NOTIFY_UNRAID:-false} Discord=$([[ -n "${MY_DISCORD_WEBHOOK:-}" ]] && echo enabled || echo disabled)"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Sparse Checkout Configuration ━━━
|
||||
# ==============================================================================================
|
||||
# Build the list of host*.conf files that belong to OTHER servers.
|
||||
# This server pulls everything EXCEPT those files.
|
||||
# MY_ID is set by detect_hosts() — e.g. "HOST1"
|
||||
|
||||
configure_sparse_checkout() {
|
||||
local repo_dir="$1"
|
||||
|
||||
log "Configuring sparse checkout for $MY_ID..."
|
||||
|
||||
# Enable sparse checkout
|
||||
git -C "$repo_dir" config core.sparseCheckout true 2>/dev/null
|
||||
|
||||
# Build exclusion list — all host*.conf files except MY_ID's
|
||||
local sparse_file="$repo_dir/.git/info/sparse-checkout"
|
||||
mkdir -p "$(dirname "$sparse_file")"
|
||||
|
||||
# Start with: pull everything
|
||||
echo "/*" > "$sparse_file"
|
||||
|
||||
# Exclude each other server's conf file
|
||||
# Find all host*.conf files present in the repo
|
||||
local excluded=0
|
||||
for conf_file in "$repo_dir"/host*.conf; do
|
||||
[[ -f "$conf_file" ]] || continue
|
||||
local conf_name
|
||||
conf_name=$(basename "$conf_file")
|
||||
|
||||
# Determine which HOST ID owns this conf by grepping its hostname var
|
||||
# Pattern: HOST1="unRAID-..." or HOST2="unRAID-..."
|
||||
local conf_host_id
|
||||
conf_host_id=$(grep -m1 -oP '^\s+HOST[0-9]+(?==)' "$conf_file" 2>/dev/null | tr -d ' ')
|
||||
|
||||
if [[ -z "$conf_host_id" ]]; then
|
||||
log "Cannot determine HOST ID for $conf_name — including in pull (safe default)"
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ "$conf_host_id" != "$MY_ID" ]]; then
|
||||
echo "!$conf_name" >> "$sparse_file"
|
||||
log "Sparse checkout: excluding $conf_name (belongs to $conf_host_id)"
|
||||
((excluded++))
|
||||
else
|
||||
log "Sparse checkout: including $conf_name (belongs to $MY_ID — this server)"
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ "$excluded" -gt 0 ]]; then
|
||||
echo " Sparse checkout: excluding $excluded peer conf file(s) — credentials protected"
|
||||
else
|
||||
log "Sparse checkout: no peer conf files to exclude (single server or first run)"
|
||||
fi
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Git Sync ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Git Sync ━━━"
|
||||
echo "$ICON_NET Repo: $REPO_SSH"
|
||||
echo "$ICON_GEAR Target: $TARGET_DIR"
|
||||
echo ""
|
||||
|
||||
START=$(date +%s)
|
||||
SYNC_SUCCESS=false
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would sync $REPO_SSH → $TARGET_DIR"
|
||||
warn "DRY RUN — would configure sparse checkout for $MY_ID"
|
||||
warn "DRY RUN — would exclude peer host*.conf files"
|
||||
SYNC_SUCCESS=true
|
||||
else
|
||||
mkdir -p "$TARGET_DIR"
|
||||
git config --global --add safe.directory "$TARGET_DIR"
|
||||
cd "$TARGET_DIR" || { error "Cannot cd into $TARGET_DIR"; exit 1; }
|
||||
|
||||
if [[ -d ".git" ]]; then
|
||||
# ── Existing repository ──────────────────────────────────────────────
|
||||
echo " Existing repository — updating"
|
||||
|
||||
# Configure sparse checkout BEFORE pull
|
||||
# Uses conf files already present from last pull to determine exclusions
|
||||
configure_sparse_checkout "$TARGET_DIR"
|
||||
|
||||
echo " Pulling latest changes..."
|
||||
if GIT_SSH_COMMAND="ssh -i $GITEA_SSH_KEY -p $SSH_PORT" git pull --ff-only; then
|
||||
echo " Git pull successful"
|
||||
SYNC_SUCCESS=true
|
||||
else
|
||||
# ff-only fails when local commits or tracked changes exist that can't
|
||||
# fast-forward. Fail loudly — never silently destroy local work.
|
||||
error "Git pull failed — local changes conflict with remote (will not force-reset)"
|
||||
notify "Git pull failed on $(hostname) — local changes conflict, manual resolve needed" "Git Sync" "alert"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
else
|
||||
# ── Fresh clone ──────────────────────────────────────────────────────
|
||||
echo " No repository found — cloning"
|
||||
|
||||
# Clone first — need the repo to exist before configuring sparse checkout
|
||||
if GIT_SSH_COMMAND="ssh -i $GITEA_SSH_KEY -p $SSH_PORT" git clone "$REPO_SSH" .; then
|
||||
echo " Clone successful"
|
||||
|
||||
# Configure sparse checkout after clone
|
||||
# Now all host*.conf files are present — can detect exclusions
|
||||
configure_sparse_checkout "$TARGET_DIR"
|
||||
|
||||
# Apply sparse checkout — removes excluded files from working tree
|
||||
echo " Applying sparse checkout..."
|
||||
git read-tree -mu HEAD
|
||||
echo " Sparse checkout applied — peer credentials removed from working tree"
|
||||
|
||||
SYNC_SUCCESS=true
|
||||
else
|
||||
error "Clone failed"
|
||||
notify "Git clone failed on $(hostname) — check Gitea connectivity" "Git Sync" "alert"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Permissions ──────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Permissions ━━━"
|
||||
log "Setting executable permissions on all .sh files..."
|
||||
find "$TARGET_DIR" -type f -name "*.sh" -exec chmod +x {} \;
|
||||
echo " Permissions set on .sh files"
|
||||
|
||||
# ── Flash mode: sync Plugin/ to /boot/ so the webUI picks up updates ─────
|
||||
# In flash mode SCRIPTS_DIR is in appdata — Plugin/ lives in the repo there
|
||||
# but Unraid serves PHP from /boot/. Sync after every pull to keep them in step.
|
||||
_BOOT_DIR="/boot/config/plugins/varaverk"
|
||||
if [[ "$TARGET_DIR" != "$_BOOT_DIR" ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Flash mode: sync Plugin/ → /boot/ ━━━"
|
||||
if rsync -a --delete "$TARGET_DIR/Plugin/" "$_BOOT_DIR/Plugin/" 2>/dev/null; then
|
||||
echo " Plugin/ synced to /boot/ ✅"
|
||||
else
|
||||
warn "Plugin/ sync to /boot/ failed — webUI may be stale until next pull"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Conf Upgrade ━━━
|
||||
# ==============================================================================================
|
||||
# Merges new conf structure into the live conf files after every pull.
|
||||
# New keys → added with template defaults (user fills in once).
|
||||
# Removed keys → dropped. Existing values → always preserved.
|
||||
# Silent when already up to date — no overhead on unchanged pulls.
|
||||
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Conf Upgrade ━━━"
|
||||
|
||||
UPGRADE_SCRIPT="$TARGET_DIR/Deployment/conf_upgrade.sh"
|
||||
CONF_DIR="$TARGET_DIR/Configurations"
|
||||
|
||||
if [[ ! -f "$UPGRADE_SCRIPT" ]]; then
|
||||
log "conf_upgrade.sh not found — skipping (pre-deployment-folder repo)"
|
||||
elif [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would upgrade master.conf and ${MY_ID,,}.conf"
|
||||
elif [[ "$SYNC_SUCCESS" == true ]]; then
|
||||
_DRY=""
|
||||
|
||||
# master.conf
|
||||
if [[ -f "$CONF_DIR/master.conf.template" && -f "$CONF_DIR/master.conf" ]]; then
|
||||
bash "$UPGRADE_SCRIPT" \
|
||||
--template "$CONF_DIR/master.conf.template" \
|
||||
--target "$CONF_DIR/master.conf" \
|
||||
--backup $_DRY
|
||||
else
|
||||
warn "master.conf.template or master.conf not found — skipping"
|
||||
fi
|
||||
|
||||
# This server's host conf only — sparse checkout ensures we have it
|
||||
HOST_CONF="$CONF_DIR/${MY_ID,,}.conf"
|
||||
if [[ -f "$CONF_DIR/host.conf.template" && -f "$HOST_CONF" ]]; then
|
||||
# Template uses HOSTN_ as generic prefix. Substitute MY_ID before merging
|
||||
# so keys match the target (HOST1_* or HOST2_*) and real values are preserved.
|
||||
TMPL_RESOLVED=$(mktemp)
|
||||
sed "s/HOSTN_/${MY_ID}_/g; s/REMOTE_ID/${REMOTE_ID}/g" "$CONF_DIR/host.conf.template" > "$TMPL_RESOLVED"
|
||||
bash "$UPGRADE_SCRIPT" \
|
||||
--template "$TMPL_RESOLVED" \
|
||||
--target "$HOST_CONF" \
|
||||
--backup $_DRY
|
||||
rm -f "$TMPL_RESOLVED"
|
||||
else
|
||||
warn "${MY_ID,,}.conf or host.conf.template not found — skipping"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY GIT SYNC SUMMARY ━━━━━"
|
||||
echo "$ICON_NET Repo: $REPO_SSH"
|
||||
echo "$ICON_GEAR Target: $TARGET_DIR"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_LOCK Excluded: peer host*.conf files"
|
||||
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "$ICON_WARN Status: DRY RUN — no changes made"
|
||||
elif [[ "$SYNC_SUCCESS" == true ]]; then
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS DONE"
|
||||
notify "Repository synced successfully on $(hostname)" "Git Sync" "normal"
|
||||
else
|
||||
echo "$ICON_ERROR Status: $ICON_ERROR FAILED"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
+123
@@ -0,0 +1,123 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ================================= Claude Code Startup ========================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Restores Claude Code's persistent data after an Unraid reboot and launches Claude.
|
||||
# Unraid's root filesystem lives in RAM — /root/.claude and /root/.local are wiped on
|
||||
# every reboot. This script symlinks both directories back to persistent appdata storage
|
||||
# before launching Claude, so memory, sessions, and settings survive across reboots.
|
||||
#
|
||||
# On first run with no existing persistent data, migrates from the current live locations:
|
||||
# /root/.claude → PERSIST_DIR/.claude (memory, sessions, settings)
|
||||
# /root/.local/share/claude → PERSIST_DIR/local/share/claude (installed binaries)
|
||||
# Subsequent runs skip the migration and only create the symlinks.
|
||||
#
|
||||
# Standalone script — no common.sh dependency. Safe to run directly from terminal
|
||||
# or from array_started.sh.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# claude_startup.sh
|
||||
# Set up persistent symlinks only — default, used by array_started.sh on boot.
|
||||
#
|
||||
# claude_startup.sh --launch
|
||||
# Set up persistent symlinks and launch Claude interactively.
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
PERSIST_DIR="/mnt/user/appdata/claude-code"
|
||||
CLAUDE_DATA="$PERSIST_DIR/.claude"
|
||||
CLAUDE_BIN="$PERSIST_DIR/local/share/claude"
|
||||
|
||||
LAUNCH=false
|
||||
[[ "$1" == "--launch" ]] && LAUNCH=true
|
||||
|
||||
# Standalone — no common.sh dependency
|
||||
_log() { echo " ✅ $*"; }
|
||||
_warn() { echo " ⚠️ $*"; }
|
||||
_err() { echo " ❌ $*" >&2; }
|
||||
|
||||
echo ""
|
||||
echo "━━━ Claude Code Startup ━━━"
|
||||
echo ""
|
||||
|
||||
# ── Array must be mounted ─────────────────────────────────────────────────────────────────────
|
||||
if ! mountpoint -q /mnt/user 2>/dev/null; then
|
||||
_err "Array not mounted — /mnt/user not available"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ── Create persistent dirs ────────────────────────────────────────────────────────────────────
|
||||
mkdir -p "$CLAUDE_DATA" "$CLAUDE_BIN"
|
||||
|
||||
# ── Migrate .claude on first run ──────────────────────────────────────────────────────────────
|
||||
if [[ ! -L /root/.claude && -d /root/.claude ]]; then
|
||||
_warn "First run — migrating /root/.claude → $CLAUDE_DATA"
|
||||
cp -a /root/.claude/. "$CLAUDE_DATA/"
|
||||
rm -rf /root/.claude
|
||||
_log "Migrated .claude (memory, sessions, settings)"
|
||||
elif [[ -z "$(ls -A "$CLAUDE_DATA" 2>/dev/null)" && -d /root/.claude ]]; then
|
||||
_warn "Persistent storage empty — copying current .claude data"
|
||||
cp -a /root/.claude/. "$CLAUDE_DATA/"
|
||||
_log "Copied .claude data to persistent storage"
|
||||
fi
|
||||
|
||||
# ── Migrate Claude binaries on first run ──────────────────────────────────────────────────────
|
||||
if [[ ! -L /root/.local/share/claude && -d /root/.local/share/claude ]]; then
|
||||
_warn "First run — migrating Claude binaries → $CLAUDE_BIN"
|
||||
cp -a /root/.local/share/claude/. "$CLAUDE_BIN/"
|
||||
_log "Migrated Claude binaries"
|
||||
fi
|
||||
|
||||
# ── Create symlinks ───────────────────────────────────────────────────────────────────────────
|
||||
# Remove any real directories first — ln -sfn silently creates inside a dir instead of
|
||||
# replacing it, which produces a circular symlink on subsequent runs after migration.
|
||||
mkdir -p /root/.local/share /root/.local/bin
|
||||
|
||||
[[ -d /root/.claude && ! -L /root/.claude ]] && rm -rf /root/.claude
|
||||
ln -sfn "$CLAUDE_DATA" /root/.claude
|
||||
_log ".claude → $CLAUDE_DATA"
|
||||
|
||||
[[ -d /root/.local/share/claude && ! -L /root/.local/share/claude ]] && rm -rf /root/.local/share/claude
|
||||
ln -sfn "$CLAUDE_BIN" /root/.local/share/claude
|
||||
_log "claude binary → $CLAUDE_BIN"
|
||||
|
||||
# ── Symlink CLAUDE.md ─────────────────────────────────────────────────────────────────────────
|
||||
# Lives on /boot so it survives reboots without appdata. Symlinked into /root so Claude
|
||||
# picks it up automatically from the working directory on every session.
|
||||
CLAUDE_MD="/boot/config/plugins/varaverk/CLAUDE.md"
|
||||
if [[ -f "$CLAUDE_MD" ]]; then
|
||||
ln -sfn "$CLAUDE_MD" /root/CLAUDE.md
|
||||
_log "CLAUDE.md → $CLAUDE_MD"
|
||||
else
|
||||
_warn "CLAUDE.md not found at $CLAUDE_MD — skipping symlink"
|
||||
fi
|
||||
|
||||
# ── Point the claude binary at the latest installed version ───────────────────────────────────
|
||||
LATEST=$(ls "$CLAUDE_BIN/versions/" 2>/dev/null | sort -V | tail -1)
|
||||
if [[ -z "$LATEST" ]]; then
|
||||
_err "No Claude versions found in $CLAUDE_BIN/versions/"
|
||||
_err "Install Claude Code first: npm install -g @anthropic-ai/claude-code"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
ln -sfn "$CLAUDE_BIN/versions/$LATEST" /root/.local/bin/claude
|
||||
_log "claude v$LATEST ready"
|
||||
|
||||
echo ""
|
||||
|
||||
# ── Setup-only mode (used by array_started.sh or other callers) ─────────────────────────────────
|
||||
if [[ "$LAUNCH" == false ]]; then
|
||||
_log "Setup complete — run 'claude' to start"
|
||||
echo ""
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ── Launch ────────────────────────────────────────────────────────────────────────────────────
|
||||
export PATH="$HOME/.local/bin:$PATH"
|
||||
exec claude
|
||||
+146
@@ -0,0 +1,146 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ================================= Claude Code Startup ========================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Restores Claude Code's persistent data after an Unraid reboot.
|
||||
# /root is RAM — wiped on every boot. This script re-creates symlinks so
|
||||
# Claude's memory, sessions, settings, and binary survive across reboots.
|
||||
#
|
||||
# Storage mode is read from the host conf file:
|
||||
#
|
||||
# HOST*_STORAGE_MODE_INTERNAL=true → Internal (boot) mode
|
||||
# .claude data → /boot/config/claude
|
||||
# binary → /boot/config/claude-bin
|
||||
# No array dependency — runs even before array mounts.
|
||||
#
|
||||
# HOST*_STORAGE_MODE_INTERNAL=false → Appdata mode
|
||||
# .claude data → /mnt/user/appdata/claude-code/.claude
|
||||
# binary → /mnt/user/appdata/claude-code/local/share/claude
|
||||
# Requires array to be mounted.
|
||||
#
|
||||
# On first run in either mode, migrates any existing live data to persistent
|
||||
# storage. Subsequent runs only re-create the symlinks.
|
||||
#
|
||||
# Standalone script — no common.sh dependency. Safe to run directly from
|
||||
# terminal or from array_started.sh.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# claude_startup.sh
|
||||
# Set up persistent symlinks only — default, used by array_started.sh on boot.
|
||||
#
|
||||
# claude_startup.sh --launch
|
||||
# Set up persistent symlinks and launch Claude interactively.
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
LAUNCH=false
|
||||
[[ "$1" == "--launch" ]] && LAUNCH=true
|
||||
|
||||
_log() { echo " ✅ $*"; }
|
||||
_warn() { echo " ⚠️ $*"; }
|
||||
_err() { echo " ❌ $*" >&2; }
|
||||
|
||||
echo ""
|
||||
echo "━━━ Claude Code Startup ━━━"
|
||||
echo ""
|
||||
|
||||
# ── Detect storage mode ───────────────────────────────────────────────────────────────────────
|
||||
CONF_DIR="/boot/config/plugins/varaverk/Configurations"
|
||||
STORAGE_INTERNAL=false
|
||||
for _conf in "$CONF_DIR"/host*.conf; do
|
||||
[[ -f "$_conf" ]] || continue
|
||||
if grep -q "_STORAGE_MODE_INTERNAL=true" "$_conf" 2>/dev/null; then
|
||||
STORAGE_INTERNAL=true
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ "$STORAGE_INTERNAL" == true ]]; then
|
||||
CLAUDE_DATA="/boot/config/claude"
|
||||
CLAUDE_BIN="/boot/config/claude-bin"
|
||||
_log "Storage mode: internal boot"
|
||||
else
|
||||
PERSIST_DIR="/mnt/user/appdata/claude-code"
|
||||
CLAUDE_DATA="$PERSIST_DIR/.claude"
|
||||
CLAUDE_BIN="$PERSIST_DIR/local/share/claude"
|
||||
_log "Storage mode: appdata"
|
||||
fi
|
||||
|
||||
# ── Array check (appdata mode only) ──────────────────────────────────────────────────────────
|
||||
if [[ "$STORAGE_INTERNAL" == false ]]; then
|
||||
if ! mountpoint -q /mnt/user 2>/dev/null; then
|
||||
_err "Array not mounted — /mnt/user not available (required for appdata mode)"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Create persistent dirs ────────────────────────────────────────────────────────────────────
|
||||
mkdir -p "$CLAUDE_DATA" "$CLAUDE_BIN"
|
||||
|
||||
# ── Migrate .claude on first run ──────────────────────────────────────────────────────────────
|
||||
if [[ ! -L /root/.claude && -d /root/.claude ]]; then
|
||||
_warn "First run — migrating /root/.claude → $CLAUDE_DATA"
|
||||
cp -a /root/.claude/. "$CLAUDE_DATA/"
|
||||
rm -rf /root/.claude
|
||||
_log "Migrated .claude (memory, sessions, settings)"
|
||||
elif [[ -z "$(ls -A "$CLAUDE_DATA" 2>/dev/null)" && -d /root/.claude ]]; then
|
||||
_warn "Persistent storage empty — copying current .claude data"
|
||||
cp -a /root/.claude/. "$CLAUDE_DATA/"
|
||||
_log "Copied .claude data to persistent storage"
|
||||
fi
|
||||
|
||||
# ── Migrate Claude binaries on first run ──────────────────────────────────────────────────────
|
||||
if [[ ! -L /root/.local/share/claude && -d /root/.local/share/claude ]]; then
|
||||
_warn "First run — migrating Claude binaries → $CLAUDE_BIN"
|
||||
cp -a /root/.local/share/claude/. "$CLAUDE_BIN/"
|
||||
_log "Migrated Claude binaries"
|
||||
fi
|
||||
|
||||
# ── Create symlinks ───────────────────────────────────────────────────────────────────────────
|
||||
mkdir -p /root/.local/share /root/.local/bin
|
||||
|
||||
[[ -d /root/.claude && ! -L /root/.claude ]] && rm -rf /root/.claude
|
||||
ln -sfn "$CLAUDE_DATA" /root/.claude
|
||||
_log ".claude → $CLAUDE_DATA"
|
||||
|
||||
[[ -d /root/.local/share/claude && ! -L /root/.local/share/claude ]] && rm -rf /root/.local/share/claude
|
||||
ln -sfn "$CLAUDE_BIN" /root/.local/share/claude
|
||||
_log "claude binary → $CLAUDE_BIN"
|
||||
|
||||
# ── Symlink CLAUDE.md ─────────────────────────────────────────────────────────────────────────
|
||||
CLAUDE_MD="/boot/config/plugins/varaverk/CLAUDE.md"
|
||||
if [[ -f "$CLAUDE_MD" ]]; then
|
||||
ln -sfn "$CLAUDE_MD" /root/CLAUDE.md
|
||||
_log "CLAUDE.md → $CLAUDE_MD"
|
||||
else
|
||||
_warn "CLAUDE.md not found at $CLAUDE_MD — skipping symlink"
|
||||
fi
|
||||
|
||||
# ── Point the claude binary at the latest installed version ───────────────────────────────────
|
||||
LATEST=$(ls "$CLAUDE_BIN/versions/" 2>/dev/null | sort -V | tail -1)
|
||||
if [[ -z "$LATEST" ]]; then
|
||||
_err "No Claude versions found in $CLAUDE_BIN/versions/"
|
||||
_err "Install Claude Code first: npm install -g @anthropic-ai/claude-code"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
ln -sfn "$CLAUDE_BIN/versions/$LATEST" /root/.local/bin/claude
|
||||
_log "claude v$LATEST ready"
|
||||
|
||||
echo ""
|
||||
|
||||
if [[ "$LAUNCH" == false ]]; then
|
||||
_log "Setup complete — run 'claude' to start"
|
||||
echo ""
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ── Launch ────────────────────────────────────────────────────────────────────────────────────
|
||||
export PATH="$HOME/.local/bin:$PATH"
|
||||
exec claude
|
||||
Reference in New Issue
Block a user