Varaverk/host2.conf

#!/bin/bash
# ==============================================================================================
# ========================== HOST2 CONFIGURATION — unRAID-Jayred365 ===========================
# ==============================================================================================
# HOST2-specific variables — credentials, container names, share paths, failover lists.
# Sourced after master.conf — values here extend shared profile arrays and add HOST2-specific
# identity, credentials, and container configuration.
#
# Sparse checkout (git) ensures HOST1 never receives this file.
# HOST1 never sees HOST2 credentials — clean separation at the file level.
#
# DO NOT put shared config here — thresholds, toggles, profiles belong in master.conf.
# DO NOT put HOST1 variables here — they belong in host1.conf.
#
# ── STATUS ────────────────────────────────────────────────────────────────────────────────────
# HOST2 is currently being rebuilt — most sections scaffolded, fill in when back online.
# When ready: set FALLBACK_ENABLED=true and DAILY_RSYNC_ENABLED=true in master.conf.
#
# ── INDEX ─────────────────────────────────────────────────────────────────────────────────────
#
#   ── IDENTITY & CONNECTIVITY ────────────────────────────────────────────────────────────────
#   IDENTITY                hostname, SSH key
#   EMBY                    container name, URL, API key
#   NOTIFICATIONS           Discord webhook
#   PARTNERSHIP             auth containers, backup paths
#
#   ── RSYNC ──────────────────────────────────────────────────────────────────────────────────
#   DAILY SYNC SHARES       media shares HOST2 owns and pushes to HOST1
#   WEEKLY SYNC SHARES      appdata shares synced weekly (Sunday 2:30am)
#   CRITICAL SYNC SHARES    appdata shares synced every 15 minutes
#   BACKUP VERIFY           shares for checksum verification against remote
#   HOST2 RSYNC PROFILE     host2-appdata profile for HOST2-specific appdata syncs
#
#   ── DOCKER ─────────────────────────────────────────────────────────────────────────────────
#   DOCKER DAILY RESTART    containers restarted daily
#   DOCKER WEEKLY RESTART   containers restarted weekly
#   DOCKER WATCHDOG         memory limits, health URLs, required containers, ignore list
#   DOCKER NETWORK CONNECT  networks and containers for docker_network_connect.sh
#
#   ── FALLBACK ───────────────────────────────────────────────────────────────────────────────
#   DDNS                    DDNS containers managed by HOST2
#   INTERNET LOSS           containers stopped when internet is lost
#   FALLBACK TIERS          what HOST2 runs for HOST1 per tier
#   TIER DELAYS             how long HOST2 must be down before each tier activates on HOST1
#   RSYNC WRITEBACK         HOST2 appdata synced back on handback
#
#   ── MEDIA ──────────────────────────────────────────────────────────────────────────────────
#   MEDIA PERMISSIONS       share list for media_shares_permissions.sh
#   MEDIA CLEANER           folder lists for media_cleaner.sh
#
#   ── MONITORS ───────────────────────────────────────────────────────────────────────────────
#   CERTIFICATE MONITOR     domains checked for SSL expiry
#   SMART HEALTH            drives to skip in SMART monitoring
#   ZFS REPORT              pools to exclude from ZFS health report
#
#   ── TRANSCODES ─────────────────────────────────────────────────────────────────────────────
#   TRANSCODES              ramdisk size, thresholds, SSD path, server array
#
#   ── ARR STACK ──────────────────────────────────────────────────────────────────────────────
#   SONARR                  URL, API key, path map
#   RADARR                  URL, API key, path map
#   ARR RECOVERY            per-arr recovery toggles (no Lidarr on HOST2)
#
#   ── SYSTEM WATCHDOG ────────────────────────────────────────────────────────────────────────
#   SYSTEM WATCHDOG         per-host check toggles and NIC configuration
#
#   ── RESOURCE MANAGER ───────────────────────────────────────────────────────────────────────
#   RESOURCE MANAGER        containers paused/stopped under memory pressure
#
# ==============================================================================================

# ==============================================================================================
# ── IDENTITY & CONNECTIVITY ───────────────────────────────────────────────────────────────────
# ==============================================================================================

# ━━━ Identity ━━━
# HOST2 hostname lives in master.conf (not a credential — safe for all servers).
# SSH key used for all server-to-server operations — rsync, failover container commands.
# Must be in /root/.ssh/ and authorised in HOST1's /root/.ssh/authorized_keys.
    HOST2_SSH_KEY="/root/.ssh/Jayred365-rsync-key"
    HOST2_OWNER="jayred365"
    HOST2_OWNER_EMAIL=""              # fill in when HOST2 is back online

# ━━━ Emby ━━━
# Referenced by transcode_manager.sh, emby_session_report.sh, emby_database_repair.sh,
# weekly_sync_maintenance.sh, and HOST2_TRANSCODE_SERVERS below.
# API key: Emby Dashboard → API Keys → + New Key
    HOST2_EMBY_CONTAINER="Emby-Jayred365"
    HOST2_EMBY_URL="http://localhost:8096"
    HOST2_EMBY_API_KEY="your-host2-emby-api-key"

# ━━━ Notifications ━━━
# Discord webhook — leave blank to disable.
# Per-host so HOST1 and HOST2 can post to different channels or only one server notifies.
    HOST2_DISCORD_WEBHOOK=""

# ━━━ Partnership ━━━
# HOST2 is the mirror — HOST1 is always the owner unless --transfer has been run.
# See README-Partnership.md and master.conf PARTNERSHIP section for full lifecycle docs.

# Auth containers reconfigured on onboard/offboard.
# Format: "ContainerName|WebUIPort"
# On onboard → WebUI pointed at owner's Tailscale IP (mirror clicks NPM, gets owner's auth)
# On offboard → WebUI pointed back at localhost
    HOST2_PARTNERSHIP_AUTH_WEBUIS=(
        # fill in when HOST2 is back online
        # "NginxProxyManager|81"
    )

# Containers to stop on this server before the owner deploys the auth stack during onboard.
# List whatever auth/proxy containers are currently running here.
    HOST2_PARTNERSHIP_REPLACE_CONTAINERS=(
        "NginxProxyManager"
        "Authelia"
        "Authelia-Secondary"
        "Mariadb-Authelia"
        "Mariadb-Authelia-Secondary"
        "Redis-Authelia"
        "Redis-Authelia-Secondary"
        "Lldap"
    )

# Arr containers to stop on this server before the owner deploys the arr stack during onboard.
    HOST2_PARTNERSHIP_ARR_REPLACE_CONTAINERS=(
        # "Sonarr"
        # "Radarr"
        # "Lidarr"
        # "Prowlarr"
        # "Bazarr"
    )

# Paths HOST1 should collect during the grace window after offboard.
# Notified on offboard — no auto-deletion, HOST1 must collect manually within PARTNERSHIP_GRACE_HOURS.
    HOST2_PARTNERSHIP_MIRROR_BACKUPS=(
        # fill in when HOST2 is back online
    )

# Containers parked on this server when partnership is active.
# Stopped on onboard (owner deploys its stack instead), restarted on offboard.
    HOST2_PARTNERSHIP_OWN_CONTAINERS=(
        # "Emby"
        # "NginxProxyManager"
    )

# This server's desired Emby admin account on the shared Emby instance.
# Set these — owner reads them during --onboard to create the account.
    HOST2_PARTNERSHIP_EMBY_ADMIN_USER=""            # desired Emby username
    HOST2_PARTNERSHIP_EMBY_ADMIN_PASS=""            # desired Emby password

# ==============================================================================================
# ── RSYNC ─────────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================

# ━━━ Daily Sync Shares ━━━
# Shares HOST2 pushes to all other nodes every night (1am via daily_sync_maintenance.sh).
# Mesh model: every node pushes every media share — no ownership, no mirrors.
# arr_sync ensures all arr libraries converge (union). rsync spreads files (additive, no --delete).
# arr_cleanup removes true orphans based on local arr state.
# Any node can download content to any share — it propagates to all nodes on the next cycle.
# Nextcloud excluded — personal data, not arr-managed, synced HOST1→HOST2 only as offsite backup.
# Uses DEFAULT_RSYNC_OPTS from master.conf — no profile needed.
# For shares needing container stops or custom options — add a profile in master.conf.
    HOST2_DAILY_SYNC_SHARES=(
        /mnt/user/Anime_Movies
        /mnt/user/Anime_Shows
        /mnt/user/Books
        /mnt/user/Intros
        /mnt/user/Kids_Movies
        /mnt/user/Kids_Tv_Shows
        /mnt/user/Movies
        /mnt/user/Music
        /mnt/user/Music_Videos
        /mnt/user/stand-up_comedy
        /mnt/user/Sports
        /mnt/user/Tv_Shows
        /mnt/user/Anime_Shows-Old
        /mnt/user/Anime_Movies-Old
    )

# Personal encrypted shares — synced for offsite backup, independent of media shares.
# ZFS encrypted at dataset level — remote receives encrypted blocks, cannot read content.
# See README-Rsync_Setup.md for ZFS encryption setup before uncommenting.
    HOST2_PERSONAL_SHARES=(
        # /mnt/user/HOST2-Personal     # uncomment after creating encrypted dataset
    )

# ━━━ Weekly Sync Shares ━━━
# Appdata shares synced during the weekly maintenance window (Sunday 2:30am).
# Containers stopped both sides before sync — full clean state guaranteed.
# Profiles drive container stops, excludes, and options — configured in master.conf RSYNC section.
    HOST2_WEEKLY_SYNC_SHARES=(
        # fill in when HOST2 is back online
        # "/mnt/user/Media_Server/Emby"
        # "/mnt/user/appdata-Fallback/Critical-Data"
    )

# ━━━ Intermediate Sync Shares ━━━
# Shares synced every 4 hours by intermediate_sync_maintenance.sh.
# Uses DEFAULT_RSYNC_OPTS (no --delete) — for sub-daily propagation of metadata or watch state.
# Full media share sync stays in the daily window. Leave empty to skip mid-day rsync entirely.
    HOST2_INTERMEDIATE_SYNC_SHARES=(
        # fill in when HOST2 is back online
        # Example: "/mnt/user/Emby_Metadata"
    )

# ━━━ Critical Sync Shares ━━━
# Appdata shares synced every 15 minutes by critical_sync_maintenance.sh.
# Format: "/path/to/share" or "/path/to/share|profile-name"
    HOST2_CRITICAL_SYNC_SHARES=(
        # fill in when HOST2 is back online
        # "/mnt/user/appdata-Fallback/Critical-Data|critical-fallback"
        # "/mnt/user/Media_Server/Emby|emby-fallback"
    )

# ━━━ Backup Verify ━━━
# Shares verified by backup_verify.sh — random file checksum comparison against remote.
# Leave empty to use HOST2_DAILY_SYNC_SHARES automatically.
# Sample size and minimum file size defined in master.conf.
    HOST2_BACKUP_VERIFY_SHARES=(
        # leave empty to use HOST2_DAILY_SYNC_SHARES automatically
    )

# ━━━ HOST2 Rsync Profile — host2-appdata ━━━
# HOST2-specific appdata sync profile — extends the shared PROFILE_* arrays in master.conf.
# Use for appdata unique to HOST2.
# Shared appdata (auth stack, Emby) use dedicated profiles defined in master.conf.
# Run manually: bash Rsync/rsync.sh /mnt/user/appdata-Fallback/HOST2-Appdata --profile=host2-appdata
    PROFILE_RSYNC_OPTS[host2-appdata]="-av --info=progress2 --bwlimit=${PROFILE_BW_LIMIT[host2-appdata]:-8000}"
    PROFILE_BW_LIMIT[host2-appdata]=8000
    PROFILE_RETRY_COUNT[host2-appdata]=3
    PROFILE_SLEEP[host2-appdata]=300
    PROFILE_CRITICAL_CONTAINER_NAMES[host2-appdata]=""     # fill in when HOST2 is back online
    PROFILE_DELAYED_CONTAINERS[host2-appdata]=""
    PROFILE_CONTAINER_DELAY[host2-appdata]=5
    PROFILE_EXCLUDE_DIRS[host2-appdata]="logs *.tmp"

# ==============================================================================================
# ── DOCKER ────────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================

# ━━━ Docker Daily Restart ━━━
# Containers restarted every day via DAILY_MAINTENANCE_SCRIPTS.
# Fill in when HOST2 is back online — add containers that degrade without daily restart.
    HOST2_DAILY_RESTART_CONTAINERS=(
        "NginxProxyManager"
        # add HOST2 daily restart containers here
    )

# ━━━ Docker Weekly Restart ━━━
# Less critical services restarted weekly via WEEKLY_MAINTENANCE_SCRIPTS (Sunday 2:30am).
# Containers already stopped for weekly sync — restart adds zero extra downtime.
    HOST2_WEEKLY_RESTART_CONTAINERS=(
        # add HOST2 weekly restart containers here
    )

# ━━━ Docker Watchdog ━━━
# Per-HOST2 container configuration for docker_watchdog.sh.
# Shared thresholds and toggles live in master.conf.

# Memory hard limits in MB — immediate restart if exceeded.
# Set at "container is clearly broken" not "container is busy".
# 20GB=20480  16GB=16384  12GB=12288  10GB=10240  8GB=8192  4GB=4096  2GB=2048  1GB=1024
    declare -A HOST2_WATCHDOG_CONTAINERS=(
        ["Emby"]=16384              # fill in correct limit when HOST2 is back online
    )

# HTTP health check URLs — checked every cycle, strike system before restart.
# Only add containers with a meaningful web interface to check.
    declare -A HOST2_WATCHDOG_CONTAINER_URLS=(
        ["Emby"]="http://localhost:8096"
    )

# Required containers — must always be running on HOST2.
# Strike system before restart — repeated failures go on skip list, auto-clears on recovery.
# Listed in dependency order — dependencies before dependents.
    HOST2_WATCHDOG_REQUIRED_CONTAINERS=(
        "NginxProxyManager"
        # add HOST2 required containers here when back online
    )

# Containers to skip in Tier 2 global scan — legitimately stopped or frequently restarting.
# Watchdog leaves these alone entirely — no restart attempts, no crash loop tracking.
    HOST2_WATCHDOG_SCAN_IGNORE=(
        # add HOST2 scan ignore containers here when back online
    )

# Dependency ordering — skip restarting a container if its dependency is also down.
# Prevents watchdog from restarting dependent services before their dependencies are up.
# SPACE-SEPARATED STRINGS — converted to array at runtime.
    declare -A HOST2_WATCHDOG_DEPENDENCIES=(
        # add HOST2 dependencies here when containers are defined
        # ["Authelia"]="Mariadb-Authelia Redis-Authelia"
    )

# Per-container appdata growth suppress ceilings in MB.
# ONLY needed in specific cases — growth rate detection covers all containers automatically.
# Use when a container legitimately has large stable data and you want to suppress false-positive
# growth alerts. Add entries here only when a container triggers warnings it shouldn't.
    declare -A HOST2_WATCHDOG_APPDATA_SIZES=(
        # add HOST2 suppress entries here only as needed
    )

# ━━━ Docker Network Connect ━━━
# Containers connected to custom networks at array start by docker_network_connect.sh.
# Networks created if they don't exist — idempotent, safe to re-run.
    HOST2_NETWORK_CONNECT_CONTAINERS=(
        # fill in when HOST2 is back online
    )

    HOST2_NETWORK_CONNECT_NETWORKS=(
        # fill in when HOST2 is back online
    )

# ==============================================================================================
# ── FALLBACK ──────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================

# ━━━ DDNS ━━━
# DDNS containers HOST2 manages — started/stopped by fallback.sh per DDNS absolute rules:
#   Internet loss → stop immediately
#   Failover → HOST1 starts HOST2's DDNS as Tier 1 (before any other containers)
#   Handback → stop HOST2's DDNS on HOST1 → rsync → start containers → start local DDNS last
    HOST2_DDNS_CONTAINERS=(
        "Gmer4Lfe.us"
    )

# ━━━ Internet Loss ━━━
# Containers stopped immediately on HOST2 when internet connection is lost.
# Prevents external-facing services from operating without connectivity.
    FALLBACK_HOST2_STOP_ON_NO_NET=(
        "Gmer4Lfe.us"
    )

# ━━━ Fallback Tiers — HOST2 Runs for HOST1 ━━━
# Containers HOST2 starts when HOST1 goes down.
# Tier 1 is always immediate — vital services cannot wait.
# Higher tiers activate after HOST1_TIER*_DELAY minutes (set in host1.conf).
    FALLBACK_HOST2_COVERS_HOST1_TIER1=(
        "Gmer4Lfe.com"
        "Emby"
        "VaultWarden-Gmer4Lfe"
        "Dispatcharr"
        "Dispatcharr-Basic"
        "Dispatcharr-Iptv-Users"
        "ErsatzTV-Emby"
    )

    FALLBACK_HOST2_COVERS_HOST1_TIER2=(
        "Postgres-NextCloud"
        "NextCloud"
        "PostgreSQL_Immich"
        "Immich-Gmer4Lfe"
    )

    FALLBACK_HOST2_COVERS_HOST1_TIER3=(
        "Gitea"
    )

    FALLBACK_HOST2_COVERS_HOST1_TIER4=(
        "Sonarr"
        "Radarr"
        "Lidarr"
        "Readarr"
        "Prowlarr"
        "Bazarr"
        "SABnzbd-Gmer4Lfe"
        "Qbittorrent-Gmer4Lfe"
        "LidaTube"
        "Pinchflat"
        "ChannelTube"
    )

# ━━━ Tier Delays — HOST2's Containers on HOST1 ━━━
# How long HOST2 must be down before each tier activates on HOST1 — in minutes.
# Tier 1 is always immediate — no delay var needed.
    HOST2_TIER2_DELAY=240           # 4 hours  — productivity services
    HOST2_TIER3_DELAY=720           # 12 hours — secondary services
    HOST2_TIER4_DELAY=1440          # 24 hours — arrs + downloaders

# ━━━ Rsync Writeback — HOST2 Appdata Back on Handback ━━━
# Syncs HOST2 appdata BACK to HOST2 when it comes back online after a failover.
# Containers stopped before writeback — clean source, no competing writes.
#
# HOST2_TIER1_WRITEBACK_DELAY: short outages skip Tier 1 writeback — primary state
# is more reliable than dirty sync data for brief outages.
    HOST2_TIER1_WRITEBACK_DELAY=60  # skip writeback if outage under 1hr

# Tier 4 automatically syncs HOST2_DAILY_SYNC_SHARES — only list paths NOT in that array.
    FALLBACK_HOST2_WRITEBACK_TIER1=(
        # "/mnt/user/appdata-Fallback/Jayred365-Emby"
    )

    FALLBACK_HOST2_WRITEBACK_TIER2=(
        # "/mnt/user/appdata-Fallback/Jayred365-Important"
    )

    FALLBACK_HOST2_WRITEBACK_TIER3=(
        # "location-placeholder"
    )

    FALLBACK_HOST2_WRITEBACK_TIER4=(
        "/mnt/user/appdata-Fallback/Arrs_Stack"     # arr databases — downloads queued during outage
    )

# ==============================================================================================
# ── MEDIA ─────────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================

# ━━━ Media Permissions ━━━
# Shares that media_shares_permissions.sh applies PERMISSIONS_MODE and PERMISSIONS_OWNER to.
# Runs first in DAILY_MAINTENANCE_SCRIPTS — arr cleanup depends on correct ownership.
    HOST2_MEDIA_PERMISSION_SHARES=(
        /mnt/user/Anime_Movies
        /mnt/user/Anime_Shows
    )

# ━━━ Media Cleaner ━━━
# Folder lists for media_cleaner.sh — two profiles: anime and media.
# File patterns shared across all servers — defined in master.conf.
# Called via DAILY_MAINTENANCE_SCRIPTS. Run manually: Media/media_cleaner.sh anime|media
    HOST2_ANIME_CLEAN_FOLDERS=(
        /mnt/user/Anime_Movies
        /mnt/user/Anime_Shows
    )

    HOST2_MEDIA_CLEAN_FOLDERS=(
        # fill in when HOST2 is back online
    )

# ==============================================================================================
# ── MONITORS ──────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================

# ━━━ Certificate Monitor ━━━
# Domains checked via direct openssl connection — not relying on NPM's certificate state.
# Checks the actual certificate served, not what NPM thinks it has.
# Thresholds (CERT_WARN_DAYS, CERT_CRIT_DAYS) defined in master.conf.
    HOST2_CERT_MONITOR_DOMAINS=(
        # fill in when HOST2 is back online
    )

# ━━━ SMART Health ━━━
# Drives skipped in SMART attribute monitoring — hardware is server-specific.
# Thresholds read from dynamix.cfg at runtime — fallbacks in master.conf.
    HOST2_SMART_IGNORE_DRIVES=(
        "sda"                       # boot USB — SMART not meaningful on flash drives
    )

# ━━━ ZFS Report ━━━
# Pools excluded from the weekly ZFS health report — reduces noise from single-disk array pools.
# Pool health thresholds defined in master.conf.
    HOST2_ZFS_REPORT_IGNORE_POOLS=(
        # fill in when HOST2 is back online
    )

# ==============================================================================================
# ── TRANSCODES ────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================

# Ramdisk size ceiling — tmpfs only uses RAM actually needed, not the full size upfront.
# Adjust HOST2_RAMDISK_WARN_GB and HOST2_RAMDISK_LOW_GB together if this changes.
    HOST2_RAMDISK_SIZE="8G"

# Usage thresholds — coupled to HOST2_RAMDISK_SIZE, adjust all three together if size changes.
# Hysteresis gap (6.8 - 5.5 = 1.3GB) prevents flip-flop between ramdisk and SSD.
    HOST2_RAMDISK_WARN_GB=6.8      # flip to SSD when ramdisk usage reaches this
    HOST2_RAMDISK_LOW_GB=5.5       # flip back to ramdisk when usage drops to this

# SSD fallback path — where transcodes land when ramdisk exceeds HOST2_RAMDISK_WARN_GB.
# Must be on cache pool — array disks too slow for active transcode writes.
    HOST2_TRANSCODE_SSD="/mnt/cache/Temp_Storage/Emby/Transcodes/"

# Media servers sharing the ramdisk transcode space on HOST2.
# Format: "ContainerName|URL|APIKey|Type" — Type: emby | jellyfin | plex
# Entries with placeholder API keys are skipped automatically.
# ⚠️ Tdarr does NOT belong here — keep Tdarr on SSD, not ramdisk.
    HOST2_TRANSCODE_SERVERS=(
        "${HOST2_EMBY_CONTAINER}|${HOST2_EMBY_URL}|${HOST2_EMBY_API_KEY}|emby"
    )

# ==============================================================================================
# ── ARR STACK ─────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Used by arr cleanup scripts and arrs_failed_stalled_recovery.sh.
# detect_hosts() selects HOST2 vars when running on HOST2.
# Lidarr does not run on HOST2 — HOST1_LIDARR_RECOVERY flag handles the exit cleanly.
#
# PATH MAPS — container path → host path translation.
# Arr stores file paths using container-internal paths — scripts need host paths to scan.
# Add one entry per root folder in arr Settings → Media Management → Root Folders.
    HOST2_FANART_API_KEY="Yd7147a43b692df0b364b94dc47efb81"
    HOST2_LASTFM_API_KEY="be6dc169c33ae263e690c30d18b7491d"

# ━━━ Sonarr ━━━
    HOST2_SONARR_URL="http://localhost:8989"
    HOST2_SONARR_API_KEY="130decd3db5b4c25afad64864cd03f9f"
    HOST2_SONARR_TV_ROOT="/mnt/user/Anime_Shows"

    declare -A HOST2_SONARR_PATH_MAP=(
        # fill in when HOST2 is back online
        # ["/tv"]="/mnt/user/Anime_Shows"
    )

# ━━━ Radarr ━━━
    HOST2_RADARR_URL="http://localhost:7878"
    HOST2_RADARR_API_KEY="d43a3ec6cf1549edb4af0cc63f98b2a9"
    HOST2_RADARR_MOVIES_ROOT="/mnt/user/Anime_Movies"

    declare -A HOST2_RADARR_PATH_MAP=(
        # fill in when HOST2 is back online
        # ["/anime-movies"]="/mnt/user/Anime_Movies"
    )

# ━━━ Arr Recovery Toggles ━━━
# false = skip that arr on this host — exits cleanly without error
    HOST2_SONARR_RECOVERY=true
    HOST2_RADARR_RECOVERY=true
    # HOST2_LIDARR_RECOVERY not set — Lidarr does not run on HOST2

# ==============================================================================================
# ── SYSTEM WATCHDOG ───────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Per-host check toggles and NIC config for system_watchdog.sh.
# Aliased by detect_hosts() — script uses unprefixed SYS_WATCHDOG_* names.
# HOST2: i5 10th gen 64GB — being rebuilt, lighter workload, no ZFS cache pools.
#
# Conservative defaults during rebuild — re-enable checks as HOST2 stabilises.
# Three-tier response — all critical checks enabled regardless of rebuild state:
#   Tier 1 (bypass strikes, reboot now):  docker daemon, rootfs full, kernel oops, FD, /boot
#   Tier 2 (bypass strikes with OOM):     RAM critical + OOM kills in cycle
#   Tier 3 (standard strike system):      selectively disabled during rebuild
#
# RAM tiers, OOM limits, and reboot loop settings in master.conf System Watchdog section.

# ━━━ Primary NIC ━━━
# Network interface for NIC state check — verify with: ip link show | grep "^[0-9]"
# Common values: eth0, bond0, br0, eno1
    HOST2_SYS_WATCHDOG_NIC="eth0"

# ━━━ Tier 1 — Critical Checks ━━━
# All critical checks always enabled — these protect against acute failure regardless of
# rebuild state. Disabling any is not recommended.

# Docker daemon unresponsive → try restart, reboot if restart fails.
    HOST2_SYS_WATCHDOG_CHECK_DOCKER_DAEMON=true

# rootfs at critical threshold (ROOTFS_CRITICAL_PCT=99) → reboot immediately.
    HOST2_SYS_WATCHDOG_CHECK_ROOTFS=true

# Kernel BUG/Oops in dmesg delta since last cycle → reboot immediately.
    HOST2_SYS_WATCHDOG_CHECK_KERNEL_OOPS=true

# File descriptor exhaustion at FD_CRITICAL_PCT (95%) → reboot immediately.
    HOST2_SYS_WATCHDOG_CHECK_FD=true

# /boot read-only detected → reboot immediately.
    HOST2_SYS_WATCHDOG_CHECK_BOOT=true

# ━━━ Tier 2 — Urgent OOM Check ━━━
# Both must be enabled for Tier 2 bypass to function.

# Track kernel OOM kills each cycle via /proc/vmstat oom_kill delta.
    HOST2_SYS_WATCHDOG_CHECK_OOM=true

# Free RAM check — 64GB RAM on HOST2, tiers adjusted relative to HOST1.
# Update master.conf SYS_WATCHDOG_MEM_* thresholds if HOST2 needs different values.
# Currently inheriting shared master.conf values — may want lower thresholds on 64GB.
    HOST2_SYS_WATCHDOG_CHECK_RAM=true

# ━━━ Tier 3 — Standard Checks (strike system) ━━━
# Several checks disabled during rebuild — enable progressively as HOST2 stabilises.
# Each check must fail SYS_WATCHDOG_STRIKE_LIMIT consecutive cycles before action.

# /var/log filesystem usage above SYS_WATCHDOG_LOG_PCT.
    HOST2_SYS_WATCHDOG_CHECK_LOG=true

# ZFS ARC memory check.
# DISABLED — HOST2 has no ZFS cache pools. Enable if ZFS pools are added later.
    HOST2_SYS_WATCHDOG_CHECK_ARC=false

# CPU temperature above SYS_WATCHDOG_CPU_TEMP_MAX (95°C).
    HOST2_SYS_WATCHDOG_CHECK_CPU_TEMP=true

# Load average above SYS_WATCHDOG_LOAD_MULTIPLIER × core count.
# DISABLED — rebuild operations cause legitimate load spikes. Enable after rebuild.
    HOST2_SYS_WATCHDOG_CHECK_LOAD=false

# Zombie process count above SYS_WATCHDOG_ZOMBIE_LIMIT (50).
    HOST2_SYS_WATCHDOG_CHECK_ZOMBIES=true

# docker_watchdog.sh persistent skip list check.
# DISABLED during rebuild — skip list may be unreliable mid-rebuild, avoid false reboots.
# Enable once HOST2 is fully operational and docker_watchdog.sh is running stably.
    HOST2_SYS_WATCHDOG_CHECK_CONTAINERS=false

# /tmp filesystem usage with auto-clear attempt.
    HOST2_SYS_WATCHDOG_CHECK_TMP=true

# Array disk error count delta in /proc/mdstat.
    HOST2_SYS_WATCHDOG_CHECK_MDSTAT=true

# Primary NIC operstate — uses HOST2_SYS_WATCHDOG_NIC above.
    HOST2_SYS_WATCHDOG_CHECK_NETWORK=true

# sshd running check — restart attempt before escalating.
    HOST2_SYS_WATCHDOG_CHECK_SSHD=true

# Runaway process detection.
# DISABLED — rebuild workloads may legitimately peg CPU. Enable after rebuild.
    HOST2_SYS_WATCHDOG_CHECK_RUNAWAY=false

# ==============================================================================================
# ── RESOURCE MANAGER ──────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Containers to manage under pressure — see master.conf RW_CRITICAL_CONTAINERS for exclusions.

# docker pause at medium pressure (RAM < RW_RAM_MEDIUM_GB or load > medium threshold)
# Suspended in-place — instant to pause/unpause, no state lost, no restart delay.
    HOST2_RW_PAUSE_CONTAINERS=(
        # fill in when HOST2 is back online
    )

# docker stop at hard pressure (RAM < RW_RAM_HARD_GB)
# Full stop — these are optional/heavy services that free significant RAM when stopped.
# resource_watchdog.sh restarts them when pressure fully clears (RAM >= RW_RAM_RECOVER_GB).
    HOST2_RW_STOP_CONTAINERS=(
        # fill in when HOST2 is back online
    )

# ==============================================================================================
# ──────────────────────── End Of HOST2 Variables ──────────────────────────────────────────────
# ==============================================================================================