From fb051b60c1b8445a7938a2adf3ea14d045244cb7 Mon Sep 17 00:00:00 2001 From: Gmer4Lfe Date: Thu, 28 May 2026 22:24:40 -0400 Subject: [PATCH] Varaverk: FallBack + Watchdog tabs; plugin path restructure to Plugin/unraid/ - FallBack tab: per-node tier inventory + active fallback card with duration, tier, handback strikes, running container status - Watchdog tab: live system health (RAM bar + thresholds, load, uptime, daemon), docker watchdog strikes + skip list + restart history, stability strikes + reboot log, resource pressure alert card, config inventory (mem limits, required, pause/stop lists) - Swapped partnership/arrs tab order; FallBack between partnership and watchdog - Plugin source tree moved from Plugin/usr/local/emhttp/plugins/varaverk/ to Plugin/unraid/ - Deployment/ conf templates added --- Deployment/conf_templates/host.conf.template | 769 +++++++++ Deployment/conf_templates/master.conf | 1417 +++++++++++++++++ Deployment/conf_upgrade.sh | 200 +++ Media/lidarr_cleanup.sh | 7 +- Media/lidarr_missing_art.sh | 2 +- Media/radarr_cleanup.sh | 7 +- Media/sonarr_cleanup.sh | 7 +- Plugin/dev_install.sh | 63 +- .../plugins/varaverk => unraid}/Varaverk.page | 5 +- Plugin/unraid/api/arrs.php | 4 + Plugin/unraid/api/board.php | 121 ++ .../varaverk => unraid}/api/clearlock.php | 0 .../varaverk => unraid}/api/conf_toggle.php | 0 .../varaverk => unraid}/api/confform.php | 0 .../varaverk => unraid}/api/config.php | 0 .../varaverk => unraid}/api/docker_action.php | 0 .../varaverk => unraid}/api/dryrun.php | 5 +- Plugin/unraid/api/fallback.php | 4 + .../varaverk => unraid}/api/flag_toggle.php | 0 .../plugins/varaverk => unraid}/api/log.php | 0 .../plugins/varaverk => unraid}/api/media.php | 0 .../varaverk => unraid}/api/monitor.php | 0 .../varaverk => unraid}/api/movescript.php | 0 Plugin/unraid/api/partnership.php | 4 + .../varaverk => unraid}/api/rawconf.php | 0 .../varaverk => unraid}/api/readscript.php | 0 Plugin/unraid/api/recent.php | 28 + .../varaverk => unraid}/api/reorderarray.php | 0 .../api/rsync_standalone.php | 0 .../plugins/varaverk => unraid}/api/run.php | 5 +- .../varaverk => unraid}/api/savefolders.php | 0 .../varaverk => unraid}/api/scheduler.php | 4 +- .../varaverk => unraid}/api/script.php | 0 Plugin/unraid/api/scriptinfo.php | 54 + .../varaverk => unraid}/api/settings.php | 0 .../varaverk => unraid}/api/snapshot.php | 0 .../varaverk => unraid}/api/status.php | 0 .../plugins/varaverk => unraid}/api/stop.php | 0 .../varaverk => unraid}/api/system.php | 0 Plugin/unraid/api/watchdog.php | 4 + .../varaverk => unraid}/css/varaverk.css | 120 +- .../event/disks_mounted/array_start_jobs | 16 + .../event/disks_mounted/rebuild_cron | 0 .../event/disks_unmounting/array_stop_jobs | 16 + .../varaverk => unraid}/icons/varaverk.png | Bin Plugin/unraid/include/arrs.php | 278 ++++ .../varaverk => unraid}/include/confform.php | 0 .../varaverk => unraid}/include/config.php | 34 +- .../include/docker_folders.php | 0 .../varaverk => unraid}/include/docs.php | 0 Plugin/unraid/include/fallback.php | 184 +++ .../varaverk => unraid}/include/media.php | 0 .../varaverk => unraid}/include/monitor.php | 11 +- Plugin/unraid/include/partnership.php | 178 +++ .../varaverk => unraid}/include/scheduler.php | 83 +- .../varaverk => unraid}/include/vms.php | 0 Plugin/unraid/include/watchdog.php | 341 ++++ .../varaverk => unraid}/js/varaverk.js | 0 Plugin/unraid/pages/arrs.php | 231 +++ .../varaverk => unraid}/pages/config.php | 0 .../varaverk => unraid}/pages/docs.php | 0 Plugin/unraid/pages/fallback.php | 243 +++ .../varaverk => unraid}/pages/monitor.php | 77 +- Plugin/unraid/pages/partnership.php | 290 ++++ .../varaverk => unraid}/pages/scheduler.php | 853 ++++++++-- Plugin/unraid/pages/watchdog.php | 370 +++++ Plugin/unraid/run_job.sh | 99 ++ .../emhttp/plugins/varaverk/api/board.php | 88 - .../event/disks_mounted/array_start_jobs | 13 - .../event/disks_unmounting/array_stop_jobs | 13 - .../local/emhttp/plugins/varaverk/run_job.sh | 49 - Transcodes/transcode_cleanup.sh | 17 +- git_pull_execute.sh | 44 + 73 files changed, 5896 insertions(+), 462 deletions(-) create mode 100644 Deployment/conf_templates/host.conf.template create mode 100644 Deployment/conf_templates/master.conf create mode 100755 Deployment/conf_upgrade.sh rename Plugin/{usr/local/emhttp/plugins/varaverk => unraid}/Varaverk.page (74%) create mode 100644 Plugin/unraid/api/arrs.php create mode 100644 Plugin/unraid/api/board.php rename Plugin/{usr/local/emhttp/plugins/varaverk => unraid}/api/clearlock.php (100%) rename Plugin/{usr/local/emhttp/plugins/varaverk => unraid}/api/conf_toggle.php (100%) rename Plugin/{usr/local/emhttp/plugins/varaverk => unraid}/api/confform.php (100%) rename Plugin/{usr/local/emhttp/plugins/varaverk => unraid}/api/config.php (100%) rename Plugin/{usr/local/emhttp/plugins/varaverk => unraid}/api/docker_action.php (100%) rename Plugin/{usr/local/emhttp/plugins/varaverk => unraid}/api/dryrun.php (75%) create mode 100644 Plugin/unraid/api/fallback.php rename Plugin/{usr/local/emhttp/plugins/varaverk => unraid}/api/flag_toggle.php (100%) rename Plugin/{usr/local/emhttp/plugins/varaverk => unraid}/api/log.php (100%) rename Plugin/{usr/local/emhttp/plugins/varaverk => unraid}/api/media.php (100%) rename Plugin/{usr/local/emhttp/plugins/varaverk => unraid}/api/monitor.php (100%) rename Plugin/{usr/local/emhttp/plugins/varaverk => unraid}/api/movescript.php (100%) create mode 100644 Plugin/unraid/api/partnership.php rename Plugin/{usr/local/emhttp/plugins/varaverk => unraid}/api/rawconf.php (100%) rename Plugin/{usr/local/emhttp/plugins/varaverk => unraid}/api/readscript.php (100%) create mode 100644 Plugin/unraid/api/recent.php rename Plugin/{usr/local/emhttp/plugins/varaverk => unraid}/api/reorderarray.php (100%) rename Plugin/{usr/local/emhttp/plugins/varaverk => unraid}/api/rsync_standalone.php (100%) rename Plugin/{usr/local/emhttp/plugins/varaverk => unraid}/api/run.php (76%) rename Plugin/{usr/local/emhttp/plugins/varaverk => unraid}/api/savefolders.php (100%) rename Plugin/{usr/local/emhttp/plugins/varaverk => unraid}/api/scheduler.php (91%) rename Plugin/{usr/local/emhttp/plugins/varaverk => unraid}/api/script.php (100%) create mode 100644 Plugin/unraid/api/scriptinfo.php rename Plugin/{usr/local/emhttp/plugins/varaverk => unraid}/api/settings.php (100%) rename Plugin/{usr/local/emhttp/plugins/varaverk => unraid}/api/snapshot.php (100%) rename Plugin/{usr/local/emhttp/plugins/varaverk => unraid}/api/status.php (100%) rename Plugin/{usr/local/emhttp/plugins/varaverk => unraid}/api/stop.php (100%) rename Plugin/{usr/local/emhttp/plugins/varaverk => unraid}/api/system.php (100%) create mode 100644 Plugin/unraid/api/watchdog.php rename Plugin/{usr/local/emhttp/plugins/varaverk => unraid}/css/varaverk.css (86%) create mode 100755 Plugin/unraid/event/disks_mounted/array_start_jobs rename Plugin/{usr/local/emhttp/plugins/varaverk => unraid}/event/disks_mounted/rebuild_cron (100%) create mode 100755 Plugin/unraid/event/disks_unmounting/array_stop_jobs rename Plugin/{usr/local/emhttp/plugins/varaverk => unraid}/icons/varaverk.png (100%) create mode 100644 Plugin/unraid/include/arrs.php rename Plugin/{usr/local/emhttp/plugins/varaverk => unraid}/include/confform.php (100%) rename Plugin/{usr/local/emhttp/plugins/varaverk => unraid}/include/config.php (60%) rename Plugin/{usr/local/emhttp/plugins/varaverk => unraid}/include/docker_folders.php (100%) rename Plugin/{usr/local/emhttp/plugins/varaverk => unraid}/include/docs.php (100%) create mode 100644 Plugin/unraid/include/fallback.php rename Plugin/{usr/local/emhttp/plugins/varaverk => unraid}/include/media.php (100%) rename Plugin/{usr/local/emhttp/plugins/varaverk => unraid}/include/monitor.php (98%) create mode 100644 Plugin/unraid/include/partnership.php rename Plugin/{usr/local/emhttp/plugins/varaverk => unraid}/include/scheduler.php (91%) rename Plugin/{usr/local/emhttp/plugins/varaverk => unraid}/include/vms.php (100%) create mode 100644 Plugin/unraid/include/watchdog.php rename Plugin/{usr/local/emhttp/plugins/varaverk => unraid}/js/varaverk.js (100%) create mode 100644 Plugin/unraid/pages/arrs.php rename Plugin/{usr/local/emhttp/plugins/varaverk => unraid}/pages/config.php (100%) rename Plugin/{usr/local/emhttp/plugins/varaverk => unraid}/pages/docs.php (100%) create mode 100644 Plugin/unraid/pages/fallback.php rename Plugin/{usr/local/emhttp/plugins/varaverk => unraid}/pages/monitor.php (96%) create mode 100644 Plugin/unraid/pages/partnership.php rename Plugin/{usr/local/emhttp/plugins/varaverk => unraid}/pages/scheduler.php (76%) create mode 100644 Plugin/unraid/pages/watchdog.php create mode 100644 Plugin/unraid/run_job.sh delete mode 100644 Plugin/usr/local/emhttp/plugins/varaverk/api/board.php delete mode 100755 Plugin/usr/local/emhttp/plugins/varaverk/event/disks_mounted/array_start_jobs delete mode 100755 Plugin/usr/local/emhttp/plugins/varaverk/event/disks_unmounting/array_stop_jobs delete mode 100644 Plugin/usr/local/emhttp/plugins/varaverk/run_job.sh diff --git a/Deployment/conf_templates/host.conf.template b/Deployment/conf_templates/host.conf.template new file mode 100644 index 0000000..a763285 --- /dev/null +++ b/Deployment/conf_templates/host.conf.template @@ -0,0 +1,769 @@ +#!/bin/bash +# ============================================================================================== +# ========================== HOST1 CONFIGURATION — unRAID-Gmer4Lfe ============================ +# ============================================================================================== +# HOST1-specific variables — credentials, container names, share paths, failover lists. +# Sourced after master.conf — values here extend shared profile arrays and add HOST1-specific +# identity, credentials, and container configuration. +# +# Sparse checkout (git) ensures HOST2 never receives this file. +# HOST2 never sees HOST1 credentials — clean separation at the file level. +# +# DO NOT put shared config here — thresholds, toggles, profiles belong in master.conf. +# DO NOT put HOST2 variables here — they belong in host2.conf. +# +# ── INDEX ───────────────────────────────────────────────────────────────────────────────────── +# +# ── IDENTITY & CONNECTIVITY ──────────────────────────────────────────────────────────────── +# IDENTITY hostname, SSH key +# EMBY container name, URL, API key +# NOTIFICATIONS Discord webhook +# PARTNERSHIP auth containers, backup paths +# +# ── RSYNC ────────────────────────────────────────────────────────────────────────────────── +# DAILY SYNC SHARES media shares HOST1 owns and pushes to HOST2 +# WEEKLY SYNC SHARES appdata shares synced weekly (Sunday 2:30am) +# CRITICAL SYNC SHARES appdata shares synced every 30 minutes +# BACKUP VERIFY shares for checksum verification against remote +# HOST1 RSYNC PROFILE host1-appdata profile for HOST1-specific appdata syncs +# +# ── DOCKER ───────────────────────────────────────────────────────────────────────────────── +# DOCKER DAILY RESTART containers restarted daily +# DOCKER WEEKLY RESTART containers restarted weekly +# DOCKER WATCHDOG memory limits, health URLs, required containers, ignore list +# DOCKER NETWORK CONNECT networks and containers for docker_network_connect.sh +# +# ── FALLBACK ─────────────────────────────────────────────────────────────────────────────── +# DDNS DDNS containers managed by HOST1 +# INTERNET LOSS containers stopped when internet is lost +# FALLBACK TIERS what HOST1 runs for HOST2 per tier +# TIER DELAYS how long HOST1 must be down before each tier activates on HOST2 +# RSYNC WRITEBACK HOST1 appdata synced back on handback +# +# ── MEDIA ────────────────────────────────────────────────────────────────────────────────── +# MEDIA PERMISSIONS share list for media_shares_permissions.sh +# MEDIA CLEANER folder lists for media_cleaner.sh +# +# ── MONITORS ─────────────────────────────────────────────────────────────────────────────── +# CERTIFICATE MONITOR domains checked for SSL expiry +# SMART HEALTH drives to skip in SMART monitoring +# ZFS REPORT pools to exclude from ZFS health report +# +# ── TRANSCODES ───────────────────────────────────────────────────────────────────────────── +# TRANSCODES ramdisk size, thresholds, SSD path, server array +# +# ── ARR STACK ────────────────────────────────────────────────────────────────────────────── +# DOWNLOADERS slskd, SABnzbd, qBittorrent credentials and URLs +# LIDARR URL, API key, path map +# SONARR URL, API key, path map +# RADARR URL, API key, path map +# ARR RECOVERY per-arr recovery toggles +# +# ── SYSTEM WATCHDOG ──────────────────────────────────────────────────────────────────────── +# SYSTEM WATCHDOG per-host check toggles and NIC configuration +# +# ── RESOURCE MANAGER ─────────────────────────────────────────────────────────────────────── +# RESOURCE MANAGER containers paused/stopped under memory pressure +# +# ============================================================================================== + +# ============================================================================================== +# ── IDENTITY & CONNECTIVITY ─────────────────────────────────────────────────────────────────── +# ============================================================================================== + +# ━━━ Identity ━━━ +# HOST1 hostname lives in master.conf (not a credential — safe for all servers). +# SSH key used for all server-to-server operations — rsync, failover container commands. +# Must be in /root/.ssh/ and authorised in HOST2's /root/.ssh/authorized_keys. + HOST1_SSH_KEY="" # /root/.ssh/your_key_name + HOST1_OWNER="" # your Gitea/GitHub username + HOST1_OWNER_EMAIL="" # your email + +# ━━━ Emby ━━━ +# Referenced by transcode_manager.sh, emby_session_report.sh, emby_database_repair.sh, +# weekly_sync_maintenance.sh, and HOST1_TRANSCODE_SERVERS below. +# API key: Emby Dashboard → API Keys → + New Key + HOST1_EMBY_CONTAINER="Emby" + HOST1_EMBY_URL="http://localhost:8096" + HOST1_EMBY_API_KEY="" # Emby Dashboard → API Keys → + New Key + +# ━━━ Jellyfin ━━━ +# API key: Jellyfin Dashboard → Administration → API Keys → + New Key + HOST1_JELLYFIN_CONTAINER="Jellyfin" + HOST1_JELLYFIN_URL="http://localhost:8096" + HOST1_JELLYFIN_API_KEY="" # Jellyfin Dashboard → API Keys → + New Key + +# ━━━ Gitea ━━━ +# Personal access token for gitea_ssh_setup.sh — registers this server's SSH public key +# with Gitea so git operations use key auth instead of passwords. +# Create in Gitea: Settings → Applications → Generate Token → scope: write:user + HOST1_GITEA_API_TOKEN="" # Gitea Settings → Applications → Generate Token + +# ━━━ Notifications ━━━ +# Discord webhook — leave blank to disable. +# Per-host so HOST1 and HOST2 can post to different channels or only one server notifies. + HOST1_DISCORD_WEBHOOK="" # Discord channel → Integrations → Webhooks + +# ━━━ Partnership ━━━ +# HOST1 is always the owner (source of truth) unless --transfer has been run. +# See README-Partnership.md and master.conf PARTNERSHIP section for full lifecycle docs. + +# Auth containers reconfigured on onboard/offboard. +# Format: "ContainerName|WebUIPort" +# On onboard → WebUI pointed at owner's Tailscale IP (mirror clicks NPM, gets owner's auth) +# On offboard → WebUI pointed back at localhost + HOST1_PARTNERSHIP_AUTH_WEBUIS=( + "NginxProxyManager|81" + "Lldap-Gmer4Lfe|17170" + "Authelia|9091" + "Authelia-Secondary|9092" + ) + +# XML templates (from this server's templates-user/) pushed to mirror during onboard. +# These become the mirror's active auth stack, backed by the rsync-synced appdata. +# Update filename if Lldap is renamed to drop the host suffix. + HOST1_PARTNERSHIP_AUTH_STACK=( + # Dependencies first — Mariadb/Redis must be healthy before Authelia starts + "my-Mariadb-Authelia.xml" + "my-Mariadb-Authelia-Secondary.xml" + "my-Redis-Authelia.xml" + "my-Redis-Authelia-Secondary.xml" + # Auth apps — deployed after their deps are confirmed healthy + "my-Authelia.xml" + "my-Authelia-Secondary.xml" + "my-NginxProxyManager.xml" + "my-Lldap-Gmer4Lfe.xml" + # Source of truth — must be available on HOST2 independently of the auth stack + "my-Gitea.xml" + ) + +# XML templates pushed to mirror for the arr stack during onboard. +# Deps (e.g. databases) first if any — same ordering rule as auth stack. + HOST1_PARTNERSHIP_ARR_STACK=( + # "my-Sonarr.xml" + # "my-Radarr.xml" + # "my-Lidarr.xml" + # "my-Prowlarr.xml" + # "my-Bazarr.xml" + ) + +# Paths HOST2 should collect during the grace window after offboard. +# Notified on offboard — no auto-deletion, HOST2 must collect manually within PARTNERSHIP_GRACE_HOURS. + HOST1_PARTNERSHIP_MIRROR_BACKUPS=( + # "/mnt/user/appdata-Fallback/Jayred365-Emby" + ) + +# Containers parked on this server when partnership is active. +# Stopped on onboard (owner deploys its stack instead), restarted on offboard. + HOST1_PARTNERSHIP_OWN_CONTAINERS=( + # "Emby" + # "NginxProxyManager" + ) + +# Emby admin provisioning — toggle is owner-only, credentials are per-host. +# Owner enables/disables the feature. Each host sets the account they want on the shared Emby. +# On onboard: owner reads mirror's HOST*_PARTNERSHIP_EMBY_ADMIN_* and creates that account. +# On offboard: account is deleted. Username collision → onboard exits with error. + HOST1_PARTNERSHIP_PROVISION_EMBY_ADMIN=false # owner controls whether Emby is shared + HOST1_PARTNERSHIP_EMBY_PORT=8096 + HOST1_PARTNERSHIP_EMBY_ADMIN_USER="" # this server's desired Emby username + HOST1_PARTNERSHIP_EMBY_ADMIN_PASS="" # this server's desired Emby password + +# ============================================================================================== +# ── RSYNC ───────────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +# ━━━ Daily Sync Shares ━━━ +# Shares HOST1 pushes to all other nodes every night (1am via daily_sync_maintenance.sh). +# Mesh model: every node pushes every media share — no ownership, no mirrors. +# arr_sync ensures all arr libraries converge (union). rsync spreads files (additive, no --delete). +# arr_cleanup removes true orphans based on local arr state. +# Any node can download content to any share — it propagates to all nodes on the next cycle. +# Nextcloud is intentionally one-directional (HOST1→HOST2 offsite backup — not arr-managed). +# Uses DEFAULT_RSYNC_OPTS from master.conf — no profile needed. +# For shares needing container stops or custom options — add a profile in master.conf. + HOST1_DAILY_SYNC_SHARES=( + /mnt/user/Books + /mnt/user/Intros + /mnt/user/Kids_Movies + /mnt/user/Kids_Tv_Shows + /mnt/user/Movies + /mnt/user/Music + /mnt/user/Music_Videos + /mnt/user/Nextcloud + /mnt/user/stand-up_comedy + /mnt/user/Sports + /mnt/user/Tv_Shows + /mnt/user/Anime_Shows-Old + /mnt/user/Anime_Movies-Old + /mnt/user/Anime_Movies + /mnt/user/Anime_Shows + ) + +# Personal encrypted shares — synced for offsite backup, independent of media shares. +# ZFS encrypted at dataset level — remote receives encrypted blocks, cannot read content. +# See README-Rsync_Setup.md for ZFS encryption setup before uncommenting. + HOST1_PERSONAL_SHARES=( + # /mnt/user/HOST1-Personal # uncomment after creating encrypted dataset + ) + +# ━━━ Weekly Sync Shares ━━━ +# Appdata shares synced during the weekly maintenance window (Sunday 2:30am). +# Containers stopped both sides before sync — full clean state guaranteed. +# Profiles drive container stops, excludes, and options — configured in master.conf RSYNC section. +# Order matters — Emby first (larger transfer), then Critical-Data (auth stack). + HOST1_WEEKLY_SYNC_SHARES=( + "/mnt/user/Media_Server/Emby" # emby profile — full clean mirror + "/mnt/user/appdata-Fallback/Critical-Data" # critical-data profile — auth stack + ) + +# ━━━ Intermediate Sync Shares ━━━ +# Shares synced every 4 hours by intermediate_sync_maintenance.sh. +# Uses DEFAULT_RSYNC_OPTS (no --delete) — for sub-daily propagation of metadata or watch state. +# Full media share sync stays in the daily window. Leave empty to skip mid-day rsync entirely. + HOST1_INTERMEDIATE_SYNC_SHARES=( + # Add shares here to enable mid-day rsync + # Example: "/mnt/user/Emby_Metadata" + ) + +# ━━━ Critical Sync Shares ━━━ +# Appdata shares synced every 30 minutes by critical_sync_maintenance.sh. +# Format: "/path/to/share" or "/path/to/share|profile-name" +# Order matters — Critical-Data first (auth stack), then Emby dirty sync. + HOST1_CRITICAL_SYNC_SHARES=( + "/mnt/user/appdata-Fallback/Critical-Data|critical-fallback" # auth dirty sync — stays running + "/mnt/user/Media_Server/Emby|emby-fallback" # Emby dirty sync — stays running + ) + +# ━━━ Backup Verify ━━━ +# Shares verified by backup_verify.sh — random file checksum comparison against remote. +# Leave empty to use HOST1_DAILY_SYNC_SHARES automatically. +# Sample size and minimum file size defined in master.conf. + HOST1_BACKUP_VERIFY_SHARES=( + # leave empty to use HOST1_DAILY_SYNC_SHARES automatically + ) + +# ━━━ HOST1 Rsync Profile — host1-appdata ━━━ +# HOST1-specific appdata sync profile — extends the shared PROFILE_* arrays in master.conf. +# Use for appdata unique to HOST1 (Organizrv2, VaultWarden, UptimeKuma etc.) +# Shared appdata (auth stack, Emby) use dedicated profiles defined in master.conf. +# Run manually: bash Rsync/rsync.sh /mnt/user/appdata-Fallback/HOST1-Appdata --profile=host1-appdata + PROFILE_RSYNC_OPTS[host1-appdata]="-av --info=progress2 --bwlimit=${PROFILE_BW_LIMIT[host1-appdata]:-8000}" + PROFILE_BW_LIMIT[host1-appdata]=8000 + PROFILE_RETRY_COUNT[host1-appdata]=3 + PROFILE_SLEEP[host1-appdata]=300 + PROFILE_CRITICAL_CONTAINER_NAMES[host1-appdata]="Organizrv2-Gmer4Lfe UptimeKuma-Gmer4Lfe VaultWarden-Gmer4Lfe" + PROFILE_DELAYED_CONTAINERS[host1-appdata]="" + PROFILE_CONTAINER_DELAY[host1-appdata]=5 + PROFILE_EXCLUDE_DIRS[host1-appdata]="logs *.tmp" + +# ============================================================================================== +# ── DOCKER ──────────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +# ━━━ Docker Daily Restart ━━━ +# Containers restarted every day via DAILY_MAINTENANCE_SCRIPTS. +# Dispatcharr degrades over time without restart — daily is intentional, not just housekeeping. +# Order matters — auth stack first, then media services. + HOST1_DAILY_RESTART_CONTAINERS=( + "NginxProxyManager" + "Lldap-Gmer4Lfe" + "Authelia" + "Authelia-Secondary" + "Dispatcharr-Iptv-Users" + "Dispatcharr" # Live TV scheduler — degrades without daily restart + "Dispatcharr-Basic" + "ErsatzTV-Emby" + "Slskd" # Soulseek connection drops after extended uptime; restart refreshes share index + ) + +# ━━━ Docker Weekly Restart ━━━ +# Less critical services restarted weekly via WEEKLY_MAINTENANCE_SCRIPTS (Sunday 2:30am). +# Containers already stopped for weekly sync — restart adds zero extra downtime. + HOST1_WEEKLY_RESTART_CONTAINERS=( + "NextCloud" + "Organizrv2-Gmer4Lfe" + "AdGuard-Home" + "Immich-Gmer4Lfe" + ) + +# ━━━ Docker Watchdog ━━━ +# Per-HOST1 container configuration for docker_watchdog.sh. +# Shared thresholds and toggles live in master.conf. + +# Memory hard limits in MB — immediate restart if exceeded. +# Set at "container is clearly broken" not "container is busy". +# 20GB=20480 18GB=18432 16GB=16384 12GB=12288 8GB=8192 4GB=4096 2GB=2048 1GB=1024 + declare -A HOST1_WATCHDOG_CONTAINERS=( + ["Emby"]=18432 # 18GB — large library + active transcodes + ["LidaTube"]=6144 # 6GB — memory leak over time + ["Tdarr"]=6144 # 6GB — encoding is memory intensive + ["Code-Server"]=1024 # 1GB — should never need more + ) + +# HTTP health check URLs — checked every cycle, strike system before restart. +# Only add containers with a meaningful web interface to check. + declare -A HOST1_WATCHDOG_CONTAINER_URLS=( + ["Emby"]="http://localhost:8096" + ) + +# Required containers — must always be running on HOST1. +# Strike system before restart — repeated failures go on skip list, auto-clears on recovery. +# Listed in dependency order — dependencies before dependents. + HOST1_WATCHDOG_REQUIRED_CONTAINERS=( + "NginxProxyManager" + "Lldap-Gmer4Lfe" + "Mariadb-Authelia" + "Mariadb-Authelia-Secondary" + "Redis-Authelia" + "Redis-Authelia-Secondary" + "Authelia" + "Authelia-Secondary" + ) + +# Containers to skip in Tier 2 global scan — legitimately stopped or frequently restarting. +# Watchdog leaves these alone entirely — no restart attempts, no crash loop tracking. + HOST1_WATCHDOG_SCAN_IGNORE=( + "DashGate" + "PIA-WG-Config-Generator" + "Aperture" + "Aperture-Kids" + "pgvector-18-Apeture-Kids" + "Pgvector18-Aperture" + ) + +# Dependency ordering — skip restarting a container if its dependency is also down. +# Prevents watchdog from restarting Authelia before Mariadb is back up. +# SPACE-SEPARATED STRINGS — converted to array at runtime. + declare -A HOST1_WATCHDOG_DEPENDENCIES=( + ["Authelia"]="Mariadb-Authelia Redis-Authelia" + ["Authelia-Secondary"]="Mariadb-Authelia Redis-Authelia-Secondary" + ["NextCloud"]="Postgres-NextCloud" + ) + +# Per-container appdata growth suppress ceilings in MB. +# ONLY needed in specific cases — growth rate detection covers all containers automatically. +# Use this when a container legitimately has large stable data and you want to guarantee +# it never triggers a false-positive growth alert. Growth warnings are suppressed while the +# container's dir stays below this ceiling; above it, warnings resume as normal. +# 50GB=51200 25GB=25600 20GB=20480 15GB=15360 10GB=10240 5GB=5120 + declare -A HOST1_WATCHDOG_APPDATA_SIZES=( + ["Tdarr"]="25600" # 25GB — transcode cache grows legitimately during active jobs + ["7dtd"]="20480" # 20GB — game server world data, expected to be large + ) + +# ━━━ Network Watchdog ━━━ +# Host-specific connectivity config for Watchdogs/System/network_watchdog.sh. + HOST1_NETWORK_WATCHDOG_DDNS_DOMAIN="gmer4lfe.com" + HOST1_NETWORK_WATCHDOG_DDNS_CONTAINER="Gmer4Lfe.com" + HOST1_NETWORK_WATCHDOG_NPM_URL="https://gmer4lfe.com" + +# ━━━ Docker Network Connect ━━━ +# Containers connected to custom networks at array start by docker_network_connect.sh. +# Networks created if they don't exist — idempotent, safe to re-run. + HOST1_NETWORK_CONNECT_CONTAINERS=( + "memcached" + "Npm-CrowdSec" + ) + + HOST1_NETWORK_CONNECT_NETWORKS=( + "high-availability" + ) + +# ============================================================================================== +# ── FALLBACK ────────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +# ━━━ DDNS ━━━ +# DDNS containers HOST1 manages — started/stopped by fallback.sh per DDNS absolute rules: +# Internet loss → stop immediately +# Failover → HOST2 starts HOST1's DDNS as Tier 1 (before any other containers) +# Handback → stop HOST1's DDNS on HOST2 → rsync → start containers → start local DDNS last + HOST1_DDNS_CONTAINERS=( + "Gmer4Lfe.com" + ) + +# ━━━ Internet Loss ━━━ +# Containers stopped immediately on HOST1 when internet connection is lost. +# Prevents external-facing services from operating without connectivity. + FALLBACK_HOST1_STOP_ON_NO_NET=( + "Gmer4Lfe.com" + ) + +# ━━━ Fallback Tiers — HOST1 Runs for HOST2 ━━━ +# Containers HOST1 starts when HOST2 goes down. +# Tier 1 is always immediate — vital services cannot wait. +# Higher tiers activate after HOST2_TIER*_DELAY minutes (set in host2.conf). + FALLBACK_HOST1_COVERS_HOST2_TIER1=( + "Gmer4Lfe.us" + "VaultWarden-Jayred365" + ) + + FALLBACK_HOST1_COVERS_HOST2_TIER2=( + # "container-placeholder" + ) + + FALLBACK_HOST1_COVERS_HOST2_TIER3=( + # "container-placeholder" + ) + + FALLBACK_HOST1_COVERS_HOST2_TIER4=( + # "container-placeholder" + ) + +# ━━━ Tier Delays — HOST1's Containers on HOST2 ━━━ +# How long HOST1 must be down before each tier activates on HOST2 — in minutes. +# Tier 1 is always immediate — no delay var needed. + HOST1_TIER2_DELAY=240 # 4 hours — NextCloud, Immich + HOST1_TIER3_DELAY=720 # 12 hours — secondary services + HOST1_TIER4_DELAY=1440 # 24 hours — arrs + downloaders + +# ━━━ Rsync Writeback — HOST1 Appdata Back on Handback ━━━ +# Syncs HOST1 appdata BACK to HOST1 when it comes back online after a failover. +# Containers stopped before writeback — clean source, no competing writes. +# +# HOST1_TIER1_WRITEBACK_DELAY: short outages skip Tier 1 writeback — primary state +# is more reliable than dirty sync data for brief outages. + HOST1_TIER1_WRITEBACK_DELAY=60 # skip Emby writeback if outage under 1hr + +# Tier 4 automatically syncs HOST1_DAILY_SYNC_SHARES — only list paths NOT in that array. + FALLBACK_HOST1_WRITEBACK_TIER1=( + "/mnt/user/Media_Server/Emby" # watch states built up during outage + ) + + FALLBACK_HOST1_WRITEBACK_TIER2=( + "/mnt/user/appdata-Fallback/Important-Data" # NextCloud + Postgres — files added during outage + ) + + FALLBACK_HOST1_WRITEBACK_TIER3=( + # "location-placeholder" + ) + + FALLBACK_HOST1_WRITEBACK_TIER4=( + "/mnt/user/appdata-Fallback/Arrs_Stack" # arr databases — downloads queued during outage + ) + +# ============================================================================================== +# ── MEDIA ───────────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +# ━━━ Media Permissions ━━━ +# Shares that media_shares_permissions.sh applies PERMISSIONS_MODE and PERMISSIONS_OWNER to. +# Runs first in DAILY_MAINTENANCE_SCRIPTS — arr cleanup depends on correct ownership. + HOST1_MEDIA_PERMISSION_SHARES=( + /mnt/user/Anime_Movies + /mnt/user/Anime_Movies-Old + /mnt/user/Anime_Shows + /mnt/user/Anime_Shows-Old + /mnt/user/appcache + /mnt/user/Books + /mnt/user/Downloads + /mnt/user/Games + /mnt/user/Intros + /mnt/user/Kids_Movies + /mnt/user/Kids_Tv_Shows + /mnt/user/Movie_Recordings + /mnt/user/Movies + /mnt/user/Music + /mnt/user/Music_Videos + /mnt/user/Photo + /mnt/user/Sports + /mnt/user/stand-up_comedy + /mnt/user/Temp_Storage + /mnt/user/Tv_Recordings + /mnt/user/Tv_Shows + /mnt/user/YouTube + ) + +# ━━━ Media Cleaner ━━━ +# Folder lists for media_cleaner.sh — two profiles: anime and media. +# File patterns shared across all servers — defined in master.conf. +# Called via DAILY_MAINTENANCE_SCRIPTS. Run manually: Media/media_cleaner.sh anime|media + HOST1_ANIME_CLEAN_FOLDERS=( + /mnt/user/Anime_Movies + /mnt/user/Anime_Movies-Old + /mnt/user/Anime_Shows + /mnt/user/Anime_Shows-Old + ) + + HOST1_MEDIA_CLEAN_FOLDERS=( + /mnt/user/Kids_Movies + /mnt/user/Kids_Tv_Shows + /mnt/user/Movies + /mnt/user/Music + /mnt/user/Sports + /mnt/user/stand-up_comedy + /mnt/user/Tv_Shows + ) + +# ============================================================================================== +# ── MONITORS ────────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +# ━━━ Certificate Monitor ━━━ +# Domains checked via direct openssl connection — not relying on NPM's certificate state. +# Checks the actual certificate served, not what NPM thinks it has. +# Thresholds (CERT_WARN_DAYS, CERT_CRIT_DAYS) defined in master.conf. + HOST1_CERT_MONITOR_DOMAINS=( + "Gmer4Lfe.com" + "Gmer4Lfe.us" + ) + +# ━━━ SMART Health ━━━ +# Drives skipped in SMART attribute monitoring — hardware is server-specific. +# Thresholds read from dynamix.cfg at runtime — fallbacks in master.conf. + HOST1_SMART_IGNORE_DRIVES=( + "sda" # boot USB — SMART not meaningful on flash drives + ) + +# ━━━ ZFS Report ━━━ +# Pools excluded from the weekly ZFS health report — reduces noise from single-disk array pools. +# These are individual array disks formatted as ZFS — converting to XFS over time via unBalance. +# Pool health thresholds defined in master.conf. + HOST1_ZFS_REPORT_IGNORE_POOLS=( + "disk5" + "disk6" + "disk8" + "disk9" + "disk10" + ) + +# ============================================================================================== +# ── TRANSCODES ──────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +# Ramdisk size ceiling — tmpfs only uses RAM actually needed, not the full size upfront. +# Real-world: 9 streams peaked at ~5.5GB — 10G gives generous headroom on 128GB RAM. + HOST1_RAMDISK_SIZE="10G" + +# Usage thresholds — coupled to HOST1_RAMDISK_SIZE, adjust all three together if size changes. +# Hysteresis gap (8.5 - 7 = 1.5GB) prevents flip-flop between ramdisk and SSD. + HOST1_RAMDISK_WARN_GB=8.5 # flip to SSD when ramdisk usage reaches this + HOST1_RAMDISK_LOW_GB=7 # flip back to ramdisk when usage drops to this + +# SSD fallback path — where transcodes land when ramdisk exceeds HOST1_RAMDISK_WARN_GB. +# Must be on cache pool — array disks too slow for active transcode writes. + HOST1_TRANSCODE_SSD="/mnt/cache/Temp_Storage/Emby/Transcodes/" + +# Media servers sharing the ramdisk transcode space on HOST1. +# Format: "ContainerName|URL|APIKey|Type" — Type: emby | jellyfin | plex +# Entries with placeholder API keys are skipped automatically. +# ⚠️ Tdarr does NOT belong here — keep Tdarr on SSD, not ramdisk. + HOST1_TRANSCODE_SERVERS=( + "${HOST1_EMBY_CONTAINER}|${HOST1_EMBY_URL}|${HOST1_EMBY_API_KEY}|emby" + "${HOST1_JELLYFIN_CONTAINER}|${HOST1_JELLYFIN_URL}|${HOST1_JELLYFIN_API_KEY}|jellyfin" + ) + +# ============================================================================================== +# ── ARR STACK ───────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# Used by arr cleanup scripts and arrs_failed_stalled_recovery.sh. +# detect_hosts() selects HOST1 vars when running on HOST1. +# +# PATH MAPS — container path → host path translation. +# Arr stores file paths using container-internal paths — scripts need host paths to scan. +# Add one entry per root folder in arr Settings → Media Management → Root Folders. + +# ━━━ Downloaders ━━━ +# Used by downloaders_reset.sh — runs every 30min via CRITICAL_MAINTENANCE_SCRIPTS. +# Clears stuck states, purges old history, prepares each client for a clean cycle. + +# slskd — clears stuck searches, dead transfers, purges expired failed imports. +# SLSKD_FAILED_IMPORTS_DIR: where Soularr moves albums Lidarr rejected. + HOST1_SLSKD_URL="http://localhost:8980" + HOST1_SLSKD_API_KEY="" # slskd Settings → API key + HOST1_SLSKD_FAILED_IMPORTS_DIR="/mnt/user/Temp_Storage/Slskd/completed/failed_imports" + +# SABnzbd + HOST1_SABNZBD_URL="http://localhost:8180" + HOST1_SABNZBD_API_KEY="" # SABnzbd Config → General → API Key + +# qBittorrent — deleteFiles=false removes torrent from qBit but leaves files on disk. +# Radarr/Sonarr manage actual files independently. + HOST1_QBIT_URL="http://localhost:8080" + HOST1_QBIT_USERNAME="root" + HOST1_QBIT_PASSWORD="" # qBittorrent WebUI password + +# ━━━ Lidarr — HOST1 only ━━━ +# HOST2 does not run Lidarr — HOST1_LIDARR_RECOVERY flag handles the exit cleanly. + HOST1_LIDARR_URL="http://localhost:8686" + HOST1_LIDARR_API_KEY="" # Lidarr Settings → General → API Key + HOST1_LIDARR_MUSIC_ROOT="/mnt/user/Music-New" + HOST1_FANART_API_KEY="" # fanart.tv account → API Key + HOST1_LASTFM_API_KEY="" # last.fm API → Create API Account + + declare -A HOST1_LIDARR_PATH_MAP=( + ["/ext-music"]="/mnt/user/Music-New" + ) + +# ━━━ Sonarr ━━━ + HOST1_SONARR_URL="http://localhost:8989" + HOST1_SONARR_API_KEY="" # Sonarr Settings → General → API Key + HOST1_SONARR_TV_ROOT="/mnt/user/Tv_Shows" + +# Note: stand-up_comedy in both Sonarr + Radarr — TV specials and movie specials, one folder + declare -A HOST1_SONARR_PATH_MAP=( + ["/tv"]="/mnt/user/Tv_Shows" + ["/ext-standup-comedy"]="/mnt/user/stand-up_comedy" + ["/kids tv"]="/mnt/user/Kids_Tv_Shows" + ["/ext-anime-shows"]="/mnt/user/Anime_Shows-Old" + ) + +# ━━━ Radarr ━━━ + HOST1_RADARR_URL="http://localhost:7878" + HOST1_RADARR_API_KEY="" # Radarr Settings → General → API Key + HOST1_TMDB_API_KEY="" # themoviedb.org → Settings → API + HOST1_RADARR_MOVIES_ROOT="/mnt/user/Movies" + +# Note: stand-up_comedy in both Radarr + Sonarr — movie specials and TV specials, one folder + declare -A HOST1_RADARR_PATH_MAP=( + ["/movies"]="/mnt/user/Movies" + ["/kids movies"]="/mnt/user/Kids_Movies" + ["/ext-stand-up-comedy"]="/mnt/user/stand-up_comedy" + ["/anime-movies"]="/mnt/user/Anime_Movies-Old" + ) + +# ━━━ Arr Recovery Toggles ━━━ +# false = skip that arr on this host — exits cleanly without error + HOST1_SONARR_RECOVERY=true + HOST1_RADARR_RECOVERY=true + HOST1_LIDARR_RECOVERY=true # HOST1 only — exits cleanly on HOST2 + +# ============================================================================================== +# ── SYSTEM WATCHDOG ─────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# Per-host check toggles and NIC config for system_watchdog.sh. +# Aliased by detect_hosts() — script uses unprefixed SYS_WATCHDOG_* names. +# HOST1: TR1950X 128GB — full media server, active transcoding, ZFS cache pools. +# +# Three-tier response — all critical checks enabled by default on HOST1: +# Tier 1 (bypass strikes, reboot now): docker daemon, rootfs full, kernel oops, FD, /boot +# Tier 2 (bypass strikes with OOM): RAM critical + OOM kills in cycle +# Tier 3 (standard strike system): everything else +# +# RAM tiers, OOM limits, and reboot loop settings in master.conf System Watchdog section. + +# ━━━ Primary NIC ━━━ +# Network interface for NIC state check — verify with: ip link show | grep "^[0-9]" +# Common values: eth0, bond0, br0, eno1 + HOST1_SYS_WATCHDOG_NIC="eth0" + +# ━━━ Tier 1 — Critical Checks ━━━ +# These bypass the strike system — a single hit triggers immediate reboot. +# Disabling any of these is not recommended — they protect against acute system failure. + +# Docker daemon unresponsive → try restart, reboot if restart fails. +# Without a working daemon docker_watchdog.sh is blind and containers cannot be managed. + HOST1_SYS_WATCHDOG_CHECK_DOCKER_DAEMON=true + +# rootfs at critical threshold (ROOTFS_CRITICAL_PCT=99) → reboot immediately. +# At 99% rootfs writes fail silently — logs stop, Docker errors out, SSH may stop working. +# Standard 95% threshold still uses strike system — only 99%+ is critical tier. + HOST1_SYS_WATCHDOG_CHECK_ROOTFS=true + +# Kernel BUG/Oops in dmesg delta since last cycle → reboot immediately. +# A kernel oops means the kernel ran with a corrupted state — stability is not guaranteed. + HOST1_SYS_WATCHDOG_CHECK_KERNEL_OOPS=true + +# File descriptor exhaustion at FD_CRITICAL_PCT (95%) → reboot immediately. +# At 95% FD: new connections fail, Docker can't spawn processes, SSH drops. + HOST1_SYS_WATCHDOG_CHECK_FD=true + +# /boot read-only detected → reboot immediately. +# Unexpected read-only /boot means state files and config writes are silently failing. +# Fallback state, watchdog reboot log, and lock files all go stale silently. + HOST1_SYS_WATCHDOG_CHECK_BOOT=true + +# ━━━ Tier 2 — Urgent OOM Check ━━━ +# Bypass strikes when RAM is critically low AND OOM kill rate confirms active crisis. +# Both must be enabled for Tier 2 bypass to function — disable either to always use strikes. + +# Track kernel OOM kills each cycle via /proc/vmstat oom_kill delta. +# Also provides diagnostic context in reboot messages (which processes were killed). + HOST1_SYS_WATCHDOG_CHECK_OOM=true + +# Free RAM check — required for both Tier 2 bypass and RAM tier logic. +# Tiers: MEM_WARN_GB(10) → notify | MEM_SHUTDOWN_GB(6) → stop containers | MEM_GB(4) → strikes + HOST1_SYS_WATCHDOG_CHECK_RAM=true + +# ━━━ Tier 3 — Standard Checks (strike system) ━━━ +# Each check must fail SYS_WATCHDOG_STRIKE_LIMIT consecutive cycles before action is taken. +# Single spikes are ignored — sustained problems trigger reboot. + +# /var/log filesystem usage above SYS_WATCHDOG_LOG_PCT. +# Log spam (Docker log storms, syslog loops) fills rootfs — indicates something broken. + HOST1_SYS_WATCHDOG_CHECK_LOG=true + +# ZFS ARC memory pinned above SYS_WATCHDOG_ARC_PINNED_PCT after cache drop. +# Enabled on HOST1 — ZFS cache pools actively used. Disable on hosts without ZFS. + HOST1_SYS_WATCHDOG_CHECK_ARC=true + +# CPU temperature above SYS_WATCHDOG_CPU_TEMP_MAX (95°C). +# Sustained high temp causes kernel throttling or panic. Requires lm-sensors. + HOST1_SYS_WATCHDOG_CHECK_CPU_TEMP=true + +# Load average above SYS_WATCHDOG_LOAD_MULTIPLIER × core count. +# DISABLED on HOST1 — Tdarr and Emby cause legitimate sustained load spikes during encoding. +# Enable on idle servers or adjust SYS_WATCHDOG_LOAD_MULTIPLIER if load is always high. + HOST1_SYS_WATCHDOG_CHECK_LOAD=false + +# Zombie process count above SYS_WATCHDOG_ZOMBIE_LIMIT (50). +# Large zombie counts indicate serious process management failure — something is stuck. + HOST1_SYS_WATCHDOG_CHECK_ZOMBIES=true + +# Check docker_watchdog.sh persistent skip list — required containers on skip list. +# Cross-watchdog coordination: if docker_watchdog gave up, system_watchdog escalates. +# ENABLED — HOST1 fully built and operational, skip list is meaningful. + HOST1_SYS_WATCHDOG_CHECK_CONTAINERS=true + +# /tmp filesystem usage above SYS_WATCHDOG_TMP_PCT with auto-clear attempt. +# Script tries to clear aged /tmp files first — only strikes if clear fails. +# Lock files, rsync temp files, and Docker ops use /tmp — 100% means lock failures. + HOST1_SYS_WATCHDOG_CHECK_TMP=true + +# Array disk error count delta in /proc/mdstat — accumulating errors = disk failing now. +# Triggers on SYS_WATCHDOG_MDSTAT_ERROR_LIMIT new errors in one cycle. + HOST1_SYS_WATCHDOG_CHECK_MDSTAT=true + +# Primary NIC operstate — detects NIC going down (physical or driver failure). +# Uses HOST1_SYS_WATCHDOG_NIC above. Strike system — brief flaps don't trigger reboot. + HOST1_SYS_WATCHDOG_CHECK_NETWORK=true + +# sshd running check — attempts restart before escalating. +# sshd down = no remote access. Script tries rc.sshd start, notifies, strikes on failure. + HOST1_SYS_WATCHDOG_CHECK_SSHD=true + +# Runaway process detection — single process above SYS_WATCHDOG_RUNAWAY_CPU_PCT sustained. +# DISABLED — Tdarr encoding and Emby transcoding legitimately peg CPU for extended periods. +# Enable only if HOST1 has no CPU-intensive workloads. + HOST1_SYS_WATCHDOG_CHECK_RUNAWAY=false + +# ============================================================================================== +# ── RESOURCE MANAGER ────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# Containers to manage under pressure — see master.conf RW_CRITICAL_CONTAINERS for exclusions. + +# docker pause at medium pressure (RAM < RW_RAM_MEDIUM_GB or load > medium threshold) +# Suspended in-place — instant to pause/unpause, no state lost, no restart delay. + HOST1_RW_PAUSE_CONTAINERS=( + "Huntarr" # arr search automation — safe to suspend + "Cleanuparr" # download cleanup — safe to suspend + "Healarr" # arr health checks — safe to suspend + "Soularr" # Slskd automation — background only + "ChannelTube" # YouTube archiver — background only + "Pinchflat" # YouTube archiver — background only + ) + +# docker stop at hard pressure (RAM < RW_RAM_HARD_GB) +# Full stop — these are optional/heavy services that free significant RAM when stopped. +# resource_watchdog.sh restarts them when pressure fully clears (RAM >= RW_RAM_RECOVER_GB). + HOST1_RW_STOP_CONTAINERS=( + "LocalAI" # GPU/CPU heavy — largest RAM consumer when idle + "7DaysToDie" # game server — optional + "V-Rising" # game server — optional + "Code-Server" # IDE — not needed during pressure events + ) + +# ============================================================================================== +# ──────────────────────── End Of HOST1 Variables ────────────────────────────────────────────── +# ============================================================================================== \ No newline at end of file diff --git a/Deployment/conf_templates/master.conf b/Deployment/conf_templates/master.conf new file mode 100644 index 0000000..f8d5bce --- /dev/null +++ b/Deployment/conf_templates/master.conf @@ -0,0 +1,1417 @@ +#!/bin/bash +# ============================================================================================== +# ================================= MASTER CONFIGURATION ======================================= +# ============================================================================================== +# Shared configuration for the unRAID script ecosystem. +# Contains all settings that apply to every server — thresholds, toggles, profiles, job lists. +# +# ── HOW THE THREE-FILE SYSTEM WORKS ────────────────────────────────────────────────────────── +# Scripts source all three files at startup: +# source master.conf ← shared config (this file) +# source host1.conf ← HOST1 credentials, shares, container lists +# source host2.conf ← HOST2 credentials, shares, container lists +# +# Sparse checkout (git) ensures each server only pulls its own host*.conf. +# HOST2 never sees HOST1 credentials. HOST1 never sees HOST2 credentials. +# +# What belongs here: thresholds, toggles, intervals, profiles, job lists +# What belongs in HOST*: hostnames, SSH keys, API keys, passwords, share paths, +# container names, failover lists, watchdog containers +# +# ── HOW THIS FILE WORKS ─────────────────────────────────────────────────────────────────────── +# Every script sources all three conf files and common.sh at startup. +# Change a value here and it affects all scripts that use it — no hunting through files. +# To disable something: comment it out with # rather than deleting it. +# To add a new rsync profile: add a key to each PROFILE_* array in the RSYNC section. +# To add or remove orchestrator jobs: edit the arrays in the ORCHESTRATORS section. +# +# ── INDEX ───────────────────────────────────────────────────────────────────────────────────── +# +# Section Description +# ─────────────────────────────────────────────────────────────────────────────────────────── +# SHARED HOST CONFIGURATION DATA_DIR and shared connection settings +# PARTNERSHIP Mirror relationship lifecycle — onboard/offboard/transfer +# LOGGING Enable or disable verbose logging +# NOTIFICATIONS unRAID native and Discord webhook settings +# GIT / REPO Gitea repository and SSH settings +# +# ── ORCHESTRATORS ────────────────────────────────────────────────────────────────────────── +# ARRAY START Scripts launched at array start (array_started.sh) +# ARRAY STOP Scripts run at planned shutdown (array_stopping.sh) +# WATCHDOG ORCHESTRATOR Per-minute watchdog runner (watchdog_orchestrator.sh) +# SYSTEM WATCHDOG Sub-scripts called by watchdog_orchestrator.sh +# CRITICAL SYNC MAINTENANCE 30-minute jobs + sync shares + partnership check (critical_sync_maintenance.sh) +# INTERMEDIATE SYNC MAINTENANCE Arr sync + optional mid-day rsync (intermediate_sync_maintenance.sh) +# DAILY SYNC MAINTENANCE Job list + media shares (daily_sync_maintenance.sh) +# WEEKLY SYNC MAINTENANCE Job list + sync shares + update toggles (weekly_sync_maintenance.sh) +# MONTHLY MAINTENANCE Uptime-triggered heavy tasks — ZFS scrub, SMART tests (monthly_maintenance.sh) +# +# ── RSYNC ────────────────────────────────────────────────────────────────────────────────── +# RSYNC ENABLE/DISABLE Two-tier toggle system — global gate + per-orchestrator +# RSYNC DEFAULTS Global fallback rsync options and limits +# REMOTE HEALTH CHECKS Rootfs threshold for pre-flight abort +# RSYNC PROFILE SYSTEM Per-profile overrides for appdata syncs +# +# ── FALLBACK ─────────────────────────────────────────────────────────────────────────────── +# FALLBACK Mutual container failover shared settings +# FALLBACK TEST Simulated outage settings for fallback_test.sh +# +# ── DOCKER ESSENTIALS ────────────────────────────────────────────────────────────────────── +# DOWNLOADERS RESET Retention and thresholds for slskd, SABnzbd, qBittorrent +# DOCKER DAILY RESTART Containers restarted daily +# DOCKER WEEKLY RESTART Containers restarted weekly +# DOCKER WATCHDOG Thresholds and toggles for container monitoring +# DOCKER NETWORK CONNECT Networks to ensure + containers to connect +# +# ── UNRAID ESSENTIALS ────────────────────────────────────────────────────────────────────── +# INOTIFY TUNING inotify limits — raised at array start by inotify_tuning.sh +# SYSTEM TUNING MONITOR Tracks inotify + php-fpm usage over time +# REBOOT User warning delay before scheduled reboot +# MOVER Mover stop timeout +# SYSLOG FILTER Docker veth noise filter file path +# PHP-FPM PHP-FPM max children config +# CLEAR LOGS System log file paths +# WEBGUI WATCHDOG WebGUI nginx + emhttp monitoring and restart +# +# ── MEDIA ────────────────────────────────────────────────────────────────────────────────── +# MEDIA PERMISSIONS Share list, mode and owner for permissions script +# MEDIA CLEANER Anime and media folder lists and file patterns +# ARR CLEANUP Lidarr, Sonarr, Radarr orphan file cleanup settings +# ARR FAILED/STALLED RECOVERY Auto blocklist + re-search settings +# +# ── TRANSCODES ───────────────────────────────────────────────────────────────────────────── +# TRANSCODE MANAGER Ramdisk and SSD fallback transcode management +# TRANSCODE SERVER ARRAY Multi-server session monitoring (Emby, Jellyfin, Plex) +# +# ── MONITORS ─────────────────────────────────────────────────────────────────────────────── +# CERTIFICATE MONITOR SSL certificate expiry monitoring +# BACKUP VERIFY Random sample checksum verification against remote +# SMART HEALTH Drive SMART attribute monitoring +# ZFS MEMORY SNAPSHOT Weekly ZFS health and memory diagnostic report +# BANDWIDTH MONITOR Daily rsync transfer logging and weekly summary +# HEALTH DIGEST Aggregated system health digest — always/smart/weekly +# EMBY SESSION REPORT Weekly Emby usage statistics via API +# +# ── SYSTEM WATCHDOG ──────────────────────────────────────────────────────────────────────── +# SYSTEM WATCHDOG Continuous system health monitoring — last line of defense +# +# ============================================================================================== + +# ============================================================================================== +# ── HOST IDENTITIES ─────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# Hostnames for every server in the ecosystem — not credentials, safe for all machines. +# Must match the exact unRAID hostname AND Tailscale device name (case sensitive). +# detect_hosts() in common.sh matches the local hostname against these to set MY_ID / REMOTE_ID. +# Tailscale IP resolution uses these names — no hardcoded IPs needed. +# To add a new server: add HOST3="unRAID-NewServer" here + create host3.conf. + HOST1="unRAID-Gmer4Lfe" + HOST2="unRAID-Jayred365" + +# ============================================================================================== +# ── SHARED HOST CONFIGURATION ───────────────────────────────────────────────────────────────── +# ============================================================================================== +# DATA_DIR is the same path on all servers — persistent script state and statistics. +# Array share — survives reboots, no flash drive wear. +# Created automatically if it doesn't exist. +# Only truly critical files (fallback state, watchdog reboot log) stay on /boot/config. + DATA_DIR="/mnt/user/appdata/unraid_scripts/data" + +# ── Version Parity ── +# Controls behaviour when local and remote unRAID versions differ. +# Major version mismatch always aborts regardless of this setting. +# "warn" — log warning and continue (safe for minor/patch differences) +# "abort" — refuse to continue (strict — ensures both sides always match) + UNRAID_VERSION_MISMATCH_ACTION="warn" + +# ── Arr Sync ── +# arr_sync.sh syncs Lidarr/Sonarr/Radarr libraries across all nodes bidirectionally. +# Runs before rsync — all nodes agree on tracked library before files are transferred. +# Remote API keys are read live from each node's config.xml via SSH — never stored here. + ARR_SYNC_ENABLED=true + ARR_SYNC_BLOCKLIST="${DATA_DIR}/arr_sync_blocklist.tsv" + ARR_SYNC_CONNECT_TIMEOUT=10 # seconds — SSH connect timeout per node + ARR_SYNC_API_TIMEOUT=60 # seconds — curl timeout for library fetches + DOCKER_APPDATA_BASE="/mnt/user/appdata" + ARR_SYNC_LIDARR_PORT=8686 + ARR_SYNC_SONARR_PORT=8989 + ARR_SYNC_RADARR_PORT=7878 + +# ── Remote Docker Daemon ── +# Strike system for remote Docker daemon health checks. +# Scripts that issue remote container commands check the remote daemon first. +# Below limit → skip operation this run | At limit → notify critical + exit + REMOTE_DOCKER_STRIKE_LIMIT=3 # consecutive failures before critical notify + +# ============================================================================================== +# ── PARTNERSHIP ─────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# Manages the relationship lifecycle between two unRAID servers. +# HOST1 is always the owner (source of truth) — HOST2 is always the mirror. +# PARTNERSHIP_OWNER_HOST flips to "HOST2" after a --transfer operation. +# All identity vars (hostnames, SSH keys) live in host*.conf. +# Hostnames already match Tailscale device names — IP resolution is automatic. +# +# State files on /boot/config — survives reboots, available before array starts: +# /boot/config/partnership_HOST1.db ← HOST1 writes only +# /boot/config/partnership_HOST2.db ← HOST2 writes only +# Propagated via SSH — no rsync needed +# +# critical_sync_maintenance.sh runs --check every 30min: +# Reads both state files via SSH +# Detects offboard requests → finalises from owner side +# Increments offline counter → auto-offboards after threshold +# Silent when healthy ✅ +# +# See README-Partnership.md for full lifecycle documentation. + + PARTNERSHIP_ENABLED=false + PARTNERSHIP_OWNER_HOST="HOST1" # "HOST1" or "HOST2" — flips on --transfer + +# Auth containers reconfigured on onboard/offboard — defined per host in host*.conf. +# Format: "ContainerName|WebUIPort" +# HOST1_PARTNERSHIP_AUTH_WEBUIS / HOST2_PARTNERSHIP_AUTH_WEBUIS +# On onboard → WebUI pointed at owner's Tailscale IP +# On offboard → WebUI pointed back at localhost + +# Paths to collect during the grace window after offboard — defined per host in host*.conf. +# HOST1_PARTNERSHIP_MIRROR_BACKUPS / HOST2_PARTNERSHIP_MIRROR_BACKUPS +# Notified on offboard — no auto-deletion, manual collection. + +# Timing — single var controls both Tailscale removal and backup access expiry. +# Both expire at the same time — keeping backups accessible beyond Tailscale removal is pointless. + PARTNERSHIP_GRACE_HOURS=6 # hours after offboard before Tailscale removal + # backup access expires at the same time + PARTNERSHIP_OFFLINE_THRESHOLD=30 # days either server unreachable before auto-offboard + # works both directions independently + +# Tailscale removal on offboard. + PARTNERSHIP_REMOVE_TAILSCALE=true # remove mirror from Tailscale tailnet on offboard + # false = skip removal (manual or testing) + +# Tailscale API — required when PARTNERSHIP_REMOVE_TAILSCALE=true. +# Stays in shared conf — only owner uses it, and owner is always running this script. +# API key: https://login.tailscale.com/admin/settings/keys → Devices write scope + TAILSCALE_API_KEY="" # tskey-api-... + TAILSCALE_TAILNET="" # your tailnet name (e.g. yourname.github) + +# Transfer safety. + PARTNERSHIP_TRANSFER_CONFIRM="i-understand-this-transfers-ownership" + PARTNERSHIP_TRANSFER_STRIKES=3 # consecutive health checks required + PARTNERSHIP_TRANSFER_MAX_ATTEMPTS=20 # max health check attempts before giving up + +# Onboard settings. + PARTNERSHIP_ONBOARD_VERIFY=true # verify WebUI reachable after reconfiguration + PARTNERSHIP_ONBOARD_NOTIFY=true # notify both servers on completion + PARTNERSHIP_SYNC_INTERVAL=15 # minutes — informational, actual schedule in cron + +# SSH key lifecycle — managed by Partnership/ssh_setup.sh. +# Key named after this server: hostname lowercased, unraid- prefix stripped. +# unRAID-Gmer4Lfe → /root/.ssh/gmer4lfe_rsync_automation +# Run Partnership/partnership_onboard.sh to generate, copy, and update conf automatically. + SSH_MAX_STRIKES=5 # consecutive SSH auth failures before critical notify + SSH_STRIKE_RESET_HRS=24 # hours since last failure before strike counter resets + +# FolderView3 plugin integration — chodeus/VladoPortos/scolcipitato. +# Creates a "{Partner}-Failover" folder on onboard, removes + cleans containers on offboard. +# Folder name derived from remote hostname (strip unraid- prefix case-insensitively). +# Plugin config: /boot/config/plugins/folder.view3/docker.json + PARTNERSHIP_FOLDERVIEW3=true # create/remove FolderView3 folder on onboard/offboard + PARTNERSHIP_FOLDERVIEW3_URL="" # CA plugin URL — leave empty to skip auto-install + +# ============================================================================================== +# ── LOGGING ─────────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +# Controls verbose [LOG] output across all scripts. +# true = show detailed [LOG] lines — useful for debugging or first-time setup +# false = show only user-facing output — cleaner for scheduled runs + ENABLE_LOGGING=false + +# ============================================================================================== +# ── NOTIFICATIONS ───────────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +# unRAID native notification system — integrates with the bell icon in the WebGUI. +# normal = job completed successfully / warning = something failed or needs attention + NOTIFY_UNRAID=true + +# Discord webhook URL — defined per host in host*.conf. +# HOST1_DISCORD_WEBHOOK / HOST2_DISCORD_WEBHOOK +# Allows different webhooks per server, or only one server notifying. + +# ============================================================================================== +# ── GIT / REPO ──────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +# Gitea self-hosted repository — used by git_pull_execute.sh. +# Detects Gitea container location at runtime — works through failover automatically. +# Falls back to GITEA_DOMAIN if local and Tailscale both fail. + GITEA_CONTAINER="Gitea" + GITEA_REPO_PATH="Varaverk/Unraid_Scripts.git" + GITEA_DOMAIN="" # e.g. git.yourdomain.com — requires NPM + DNS + TARGET_DIR="/mnt/user/appdata/unraid_scripts" + GITEA_SSH_KEY="/root/.ssh/unraid_gitea" + SSH_PORT=221 # Gitea SSH port (default 22, Gitea often uses 221/222) + GITEA_HTTP_PORT=3000 # Gitea web/API port — used by gitea_ssh_setup.sh + +# ============================================================================================== +# ── ORCHESTRATORS ───────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# All orchestrator job lists live here — edit arrays to add/remove scripts. +# No changes to orchestrator scripts needed when adding or removing jobs. +# Sections ordered by run frequency: array events first, then shortest interval to longest. + +# ━━━ Array Start ━━━ +# Scripts launched by array_started.sh when the array comes online. +# Launched in order — each as a background process. +# One-shot scripts (ramdisk, syslog, fpm, inotify, network) run and exit naturally. +# Continuous scripts (failover) run until array stops. +# Watchdogs (resource_watchdog, docker_watchdog, system_watchdog) are cronned via +# watchdog_orchestrator.sh — NOT launched here. + ARRAY_START_SCRIPTS=( + "Transcodes/ramdisk_setup.sh" # creates ramdisk + symlink before Emby starts + "unRAID_Essentials/docker_syslog_filter.sh" # suppress veth noise before logs fill + "unRAID_Essentials/php_fpm_max_children.sh" # WebGUI performance tuning + "unRAID_Essentials/inotify_tuning.sh" # bump inotify limits — containers miss events if exhausted + "Docker_Essentials/docker_network_connect.sh" # ensure networks exist + connect containers + "Fallback/fallback.sh" # mutual failover — continuous + ) + +# ━━━ Array Stop ━━━ +# Scripts run by array_stopping.sh for a planned shutdown — stops everything cleanly in order. +# Run sequentially (foreground) — each must complete before the next starts. +# Order matters: user scripts first (prevents new ops), then data movement, then containers. + ARRAY_STOP_SCRIPTS=( + "unRAID_Essentials/user_scripts_stop.sh" # stop background scripts before they start new ops + "Fallback/fallback.sh --stop" # gracefully stop fallback (not caught by user_scripts_stop) + "unRAID_Essentials/rsync_stop.sh --rsync-only" # kill rsync; skip container recovery (handled below) + "unRAID_Essentials/mover_stop.sh" # stop mover after rsync (they conflict on same files) + "Docker_Essentials/docker_container_stop.sh" # stop all containers last + ) + +# ━━━ Watchdog Orchestrator ━━━ +# watchdog_orchestrator.sh runs WATCHDOG_ORCHESTRATOR_SCRIPTS in order each cron cycle. +# Schedule: * * * * * (every minute) +# NOT in ARRAY_START_SCRIPTS — has its own cron entry. +# Order matters — resource first (frees pressure), docker second (heals with freed resources), +# system third (storage + webgui component health), stability last (last line of defense). + WATCHDOG_ORCHESTRATOR_SCRIPTS=( + "Watchdogs/resource_watchdog.sh" # reduce system pressure before healing attempts + "Watchdogs/docker_watchdog.sh" # heal containers with freed resources + "Watchdogs/system_watchdog.sh" # system component health — storage + webgui + "Watchdogs/stability_watchdog.sh" # reboot if all else fails — last line of defense + ) + +# ━━━ System Watchdog ━━━ +# system_watchdog.sh runs SYSTEM_WATCHDOG_SCRIPTS sequentially each cycle. +# Called by watchdog_orchestrator.sh — not scheduled directly. + SYSTEM_WATCHDOG_SCRIPTS=( + "Watchdogs/System/storage_watchdog.sh" # pool growth + runaway log detection + "Watchdogs/System/webgui_watchdog.sh" # WebGUI availability — nginx → php-fpm → emhttp + "Watchdogs/System/network_watchdog.sh" # internet, DDNS, Tailscale, NPM proxy + ) + +# ━━━ Critical Sync Maintenance ━━━ +# critical_sync_maintenance.sh runs every 30 minutes. +# Order: CRITICAL_MAINTENANCE_SCRIPTS (jobs) → CRITICAL_SYNC_SHARES (rsync) → partnership --check +# partnership --check always runs last regardless of rsync gate. +# Comment out entries to disable without removing. + CRITICAL_MAINTENANCE_SCRIPTS=( + "Docker_Essentials/downloaders_reset.sh" # clear stuck download states every 30min + ) + +# Shares synced every 30 minutes — defined per host in host*.conf. +# HOST1_CRITICAL_SYNC_SHARES / HOST2_CRITICAL_SYNC_SHARES +# Format: "/path/to/share" or "/path/to/share|profile-name" +# Order matters — Critical-Data first (auth stack), then Emby dirty sync. + +# ━━━ Intermediate Sync Maintenance ━━━ +# intermediate_sync_maintenance.sh runs every 4 hours — arr library sync, artwork fetch, +# and optional mid-day rsync for any shares that need sub-daily propagation. +# Schedule: 0 */4 * * * + # INTERMEDIATE_SYNC_SHARES is host-specific — configure HOST*_INTERMEDIATE_SYNC_SHARES in host*.conf. + + INTERMEDIATE_MAINTENANCE_SCRIPTS=( + "Media/arrs_failed_stalled_recovery.sh" # blocklist + re-search failed/stalled arr queue items + ) + # arr_sync.sh runs as a fixed first step in intermediate_sync_maintenance.sh — not listed here. + # It is controlled by ARR_SYNC_ENABLED (see Arr Sync section above). + +# ━━━ Daily Sync Maintenance ━━━ +# daily_sync_maintenance.sh runs media share sync first, then iterates +# DAILY_MAINTENANCE_SCRIPTS for all jobs. +# Schedule: 0 1 * * * (1am daily) + DAILY_MAINTENANCE_SCRIPTS=( + "git_pull_execute.sh" # pull latest scripts — always runs first + "Media/media_shares_permissions.sh" # apply permissions — runs before cleaners + "Media/media_cleaner.sh anime" # remove junk from anime shares + "Media/media_cleaner.sh media" # remove junk from media shares + #"Media/lidarr_cleanup.sh" # remove orphaned music files — enable when ready + #"Media/sonarr_cleanup.sh" # remove orphaned TV files — enable when ready + "Media/radarr_cleanup.sh" # remove orphaned movie files + "Media/lidarr_missing_art.sh" # fetch missing album/artist artwork (HOST1 only — self-guards) + "Media/radarr_tmdb_removed.sh" # remove movies dropped from TMDb + "Media/sonarr_tvdb_removed.sh" # remove series dropped from TVDB + "Docker_Essentials/docker_update.sh" # pull container image updates before restart + "Docker_Essentials/docker_daily_restart.sh" # daily container restarts — runs last + ) + +# Pull latest images for DAILY_RESTART_CONTAINERS before the daily restart. +# Containers keep running during pull — no extra downtime. +# Set false to skip updates while still running the daily restart. + DAILY_CONTAINER_UPDATES=true + +# Media shares synced daily by daily_sync_maintenance.sh. +# Defined per-host in host*.conf — HOST1_DAILY_SYNC_SHARES and HOST2_DAILY_SYNC_SHARES. +# Mesh model: every node pushes every media share. rsync has no --delete so pushes are additive. +# arr_sync (union) ensures all arr libraries converge first. arr_cleanup removes true orphans. +# Any node can download content to any share — it propagates to all nodes on the next cycle. +# Adding HOST3: list every media share in HOST3_DAILY_SYNC_SHARES. No ownership to track. +# These shares use DEFAULT_RSYNC_OPTS — no profile entry needed. +# For shares needing custom options or container stops — create a profile in RSYNC section. + +# Personal encrypted shares defined per-host in host*.conf. +# ZFS encrypted at dataset level — remote receives encrypted blocks, cannot read content. +# See README-Rsync_Setup.md for ZFS encryption setup before uncommenting. + +# ━━━ Weekly Sync Maintenance ━━━ +# weekly_sync_maintenance.sh stops containers both sides → pulls updates → +# syncs WEEKLY_SYNC_SHARES → restarts → then iterates WEEKLY_MAINTENANCE_SCRIPTS. +# Schedule: 30 2 * * 0 (Sunday 2:30am) + WEEKLY_MAINTENANCE_SCRIPTS=( + "Docker_Essentials/docker_weekly_restart.sh" # weekly container restarts after sync + "Docker_Essentials/docker_update_remaining.sh" # pull updates for all other containers + "unRAID_Essentials/clear_logs.sh" # purge aged logs — Sunday only, low priority + "Media/playback_aware_lidarr_discovery.sh" # behavior-driven music discovery using weekly Emby playback history + "Media/playback_aware_radarr_discovery.sh" # behavior-driven movie discovery using TMDB recommendations + "Media/playback_aware_sonarr_discovery.sh" # behavior-driven TV discovery using TMDB recommendations + ) + +# Pull updates for all running containers NOT in daily/weekly restart lists. +# Ensures every deployed container receives at least one image pull per week. +# Set false to skip — docker_weekly_restart.sh still runs regardless. + WEEKLY_REMAINING_UPDATES=true + +# Shares synced during the weekly maintenance window — defined per host in host*.conf. +# HOST1_WEEKLY_SYNC_SHARES / HOST2_WEEKLY_SYNC_SHARES +# Containers stopped both sides before sync — full clean state guaranteed. +# Profiles drive container stops, excludes, and options — configure in RSYNC section. +# Order matters — Emby first (larger), then Critical-Data (auth stack). + +# Container update toggles for the weekly sync window. +# Containers already stopped for sync — updates pull at no extra downtime. +# Both false → sync only, no updates. + WEEKLY_SYNC_UPDATES=true # pull container updates locally during weekly window + WEEKLY_SYNC_UPDATES_REMOTE=true # pull container updates on remote via SSH + +# ━━━ Monthly Maintenance ━━━ +# monthly_maintenance.sh fires only when BOTH gates pass: +# 1. Server uptime >= MONTHLY_UPTIME_THRESHOLD_DAYS days +# 2. Last run was >= MONTHLY_RUN_INTERVAL_DAYS days ago (or never run) +# Cron: 0 3 * * * (daily 3am check — script self-gates, calling daily is safe) +# NOT in WATCHDOG_ORCHESTRATOR_SCRIPTS — has its own cron entry. +# Add scripts that require a long-stable settled system — scrubs, extended drive tests. +# State file on /boot/config — survives reboots (interval gate independent of uptime gate). + MONTHLY_MAINTENANCE_SCRIPTS=( + #"Tools/zfs_pool_scrub.sh" # ZFS pool integrity scrub — not yet built + #"Tools/smart_long_test.sh" # SMART extended drive health test — not yet built + ) + + MONTHLY_UPTIME_THRESHOLD_DAYS=30 # minimum uptime in days before maintenance fires + MONTHLY_RUN_INTERVAL_DAYS=30 # minimum days since last run before running again + MONTHLY_LAST_RUN_FILE="/boot/config/monthly_maintenance_last_run.db" + +# ━━━ Sunday Morning Coffee Report ━━━ +# Orchestrator that runs all Sunday monitor scripts in sequence. +# Schedule: 0 7 * * 0 (Sunday 7am — after weekly_sync_maintenance.sh finishes at ~3am) +# Each script runs independently and notifies on its own findings. + COFFEE_REPORT_SCRIPTS=( + "Monitors/zfs_memory_snapshot.sh" # ZFS pool health + ARC + Docker memory snapshot + "Monitors/smart_health.sh" # drive SMART attributes — reallocated, pending, temp + "Monitors/cert_monitor.sh" # SSL certificate expiry for all configured domains + "Monitors/backup_verify.sh" # rsync mirror integrity via independent MD5 checksums + "Monitors/bandwidth_monitor.sh" # weekly rsync transfer totals and per-share breakdown + "Monitors/emby_session_report.sh" # Emby usage — streams, users, library, transcode ratio + "Monitors/weekly_health_digest.sh" # aggregated digest — watchdogs, fallback, skip list + ) + +# ============================================================================================== +# ── RSYNC ───────────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +# ━━━ Rsync Enable/Disable ━━━ +# Two-tier toggle system — Tier 1 overrides Tier 2. +# +# Tier 1 — Global gate: +# RSYNC_ENABLED=false → ALL rsync stops everywhere, no exceptions +# Use when: remote completely offline, major maintenance, disaster recovery +# +# Tier 2 — Per-orchestrator (only applies when Tier 1 is true): +# Fine grained control — disable specific orchestrators while keeping others +# Use when: rebuilding secondary, testing, per-window bandwidth management +# +# Example — HOST2 data rebuild: +# RSYNC_ENABLED=true ← rsync works, individual scripts run fine +# CRITICAL_RSYNC_ENABLED=true ← 30min auth stack sync still runs +# INTERMEDIATE_RSYNC_ENABLED=false ← skip 4h arr/mid-day rsync during rebuild +# DAILY_RSYNC_ENABLED=false ← skip daily HDD syncs during rebuild +# WEEKLY_RSYNC_ENABLED=true ← Emby + Critical-Data still sync (NVMe) +# FALLBACK_RSYNC_ENABLED=true ← handback writeback still works +# → Run individual: bash Rsync/rsync.sh /mnt/user/Movies +# → When ready: INTERMEDIATE_RSYNC_ENABLED=true DAILY_RSYNC_ENABLED=true + RSYNC_ENABLED=true # Tier 1 — global gate, overrides everything below + CRITICAL_RSYNC_ENABLED=false # Tier 2 — disabled during HOST2 rebuild, re-enable when ready + INTERMEDIATE_RSYNC_ENABLED=true # Tier 2 — intermediate_sync_maintenance.sh rsync section + DAILY_RSYNC_ENABLED=false # Tier 2 — HOST2 rebuild in progress, re-enable when ready + WEEKLY_RSYNC_ENABLED=true # Tier 2 — weekly_sync_maintenance.sh rsync section + FALLBACK_RSYNC_ENABLED=true # Tier 2 — fallback.sh writeback jobs on handback + +# ━━━ Rsync Defaults ━━━ +# Global fallback values used when no profile match is found. +# Media shares in HOST*_DAILY_SYNC_SHARES always use these globals — no profile needed. +# Appdata shares match profiles by directory basename (lowercased). + BW_LIMIT=12500 # KB/s — 12500 ≈ 100Mbit + RETRY_COUNT=3 # retry attempts before giving up + SLEEP=300 # seconds between retry attempts + CRITICAL_CONTAINER_NAMES=() # containers stopped on REMOTE before rsync — profiles override + DELAYED_CONTAINERS=() # containers needing delay before starting — profiles override + CONTAINER_DELAY=5 # seconds before starting delayed containers + EXCLUDE_DIRS=() # directories excluded from transfer — profiles override + +# --inplace writes directly to destination — delta against existing file, better for large media +# --partial keep partial file on interrupted transfer so next run resumes, not re-transfers +# --timeout kill stalled transfers instead of hanging indefinitely +# --numeric-ids use UIDs/GIDs numerically — prevents ownership mismatches between servers +# --delete intentionally omitted — arr_cleanup.sh enforces media truth post-rsync. +# Media shares use this default; rsync spreads files only, never removes them. +# Note: --no-whole-file removed — redundant over SSH (delta transfer is already the default). +# Note: arr_cleanup only catches true orphans under the union model — intentional removals +# require arr_sync.sh --blocklist-add first, then manual file deletion. + DEFAULT_RSYNC_OPTS=(-av --info=progress2 --human-readable --bwlimit="$BW_LIMIT" --inplace --partial --timeout=60 --numeric-ids) + +# ━━━ Remote Health Checks ━━━ +# Pre-flight — aborts if remote rootfs (/) usage is at or above this percentage. +# When remote array is down, rsync writes land on rootfs and fill it rapidly. + ROOTFS_WARN=75 + +# ━━━ Rsync Profile System ━━━ +# Profiles allow per-share rsync behaviour without touching script logic. +# Profile key matched by basename of directory passed to rsync.sh (lowercased). +# Override with --profile=name flag. +# +# IMPORTANT: PROFILE_RSYNC_OPTS does NOT inherit DEFAULT_RSYNC_OPTS. +# List ALL desired options explicitly when defining a profile. +# +# Current profiles: +# arrs_stack — arr databases — lower bandwidth, containers stopped for consistency +# critical-data — auth stack — full stop both sides, Authelia delayed start +# called by weekly_sync_maintenance.sh — full clean sync weekly +# critical-fallback — dirty sync — auth stays running both sides, WAL excluded +# called by critical_sync_maintenance.sh every 30min +# host1-appdata — HOST1 server-specific appdata — defined in host1.conf +# host2-appdata — HOST2 server-specific appdata — defined in host2.conf +# important-data — NextCloud + Postgres — NextCloud delayed start after Postgres +# emby — weekly clean sync — both Emby stopped, full mirror +# called by weekly_sync_maintenance.sh only — do NOT schedule separately +# emby-fallback — dirty sync — Emby stays running, WAL excluded +# called by critical_sync_maintenance.sh every 30min + + declare -A PROFILE_RSYNC_OPTS=( + [arrs_stack]="-av --info=progress2 --human-readable --bwlimit=$BW_LIMIT --delete --inplace" + [critical-data]="-av --human-readable --bwlimit=$BW_LIMIT --delete" + [critical-fallback]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file" + [important-data]="-av --human-readable --bwlimit=$BW_LIMIT" + [emby]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file" + [emby-fallback]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file" + ) + +# Per-profile bandwidth limits in KB/s + declare -A PROFILE_BW_LIMIT=( + [arrs_stack]=5000 # lower — runs alongside other syncs + [critical-data]=9500 # high — small dataset, sync fast + [critical-fallback]=9500 # high — small dataset, sync fast + [important-data]=9500 # high — database sync + [emby]=8000 # medium — large full mirror + [emby-fallback]=9500 # high — small critical dataset + ) + +# Retry attempts per profile + declare -A PROFILE_RETRY_COUNT=( + [arrs_stack]=3 + [critical-data]=3 + [critical-fallback]=3 + [important-data]=3 + [emby]=3 + [emby-fallback]=3 + ) + +# Seconds between retry attempts + declare -A PROFILE_SLEEP=( + [arrs_stack]=300 + [critical-data]=300 + [critical-fallback]=120 # shorter — frequent dirty sync, retry faster + [important-data]=300 + [emby]=300 + [emby-fallback]=120 # shorter — frequent dirty sync, retry faster + ) + +# Containers stopped on BOTH LOCAL and REMOTE before rsync. +# Local stops first — flushes databases cleanly. Remote stops next — prevents writes. +# Only running containers get restarted — stopped containers stay stopped. +# SPACE-SEPARATED STRINGS — converted to array at runtime + declare -A PROFILE_CRITICAL_CONTAINER_NAMES=( + [arrs_stack]="Sonarr Lidarr Readarr Radarr Prowlarr Bazarr Pinchflat" + [critical-data]="Mariadb-Authelia Mariadb-Authelia-Secondary Redis-Authelia Redis-Authelia-Secondary Lldap-Gmer4Lfe NginxProxyManager Authelia Authelia-Secondary" + [critical-fallback]="" # dirty sync — auth stays running both sides + [important-data]="Postgres-NextCloud NextCloud" + [emby]="Emby" + [emby-fallback]="" # dirty sync — Emby stays running both sides + ) + +# Containers needing a delay after rsync before starting. +# SPACE-SEPARATED STRINGS — converted to array at runtime + declare -A PROFILE_DELAYED_CONTAINERS=( + [arrs_stack]="" + [critical-data]="Authelia Authelia-Secondary" # wait for Mariadb + Redis + [critical-fallback]="" + [important-data]="NextCloud" # wait for Postgres + [emby]="" + [emby-fallback]="" + ) + +# Seconds before starting delayed containers + declare -A PROFILE_CONTAINER_DELAY=( + [arrs_stack]=5 + [critical-data]=15 # Mariadb + Redis need time to accept connections + [critical-fallback]=5 + [important-data]=10 # Postgres needs time before NextCloud + [emby]=5 + [emby-fallback]=5 + ) + +# Directories excluded from rsync per profile. +# SPACE-SEPARATED STRINGS — converted to array at runtime + declare -A PROFILE_EXCLUDE_DIRS=( + [arrs_stack]="logs *.tmp" + [critical-data]="logs *.tmp *.log nginx/temp nginx/cache __pycache__ notification.txt" + [critical-fallback]="logs *.tmp *.log nginx/temp nginx/cache __pycache__ notification.txt *.db-wal *.db-shm" + [important-data]="logs *.tmp" + [emby]="logs transcodes cache crash*" + [emby-fallback]="logs transcodes cache metadata *.db-wal *.db-shm crash* plugins root" + ) + +# Remote restart after dirty sync — restart these on remote IF they were running before sync. +# Same logic as stop/start — was stopped = stays stopped, was running = gets restarted. +# Used by dirty sync profiles (critical-fallback, emby-fallback) so remote picks up changes. +# SPACE-SEPARATED STRINGS — converted to array at runtime + declare -A PROFILE_REMOTE_RESTART_CONTAINERS=( + [critical-fallback]="NginxProxyManager Authelia Authelia-Secondary Lldap-Gmer4Lfe Mariadb-Authelia Mariadb-Authelia-Secondary Redis-Authelia Redis-Authelia-Secondary" + [emby-fallback]="Emby" + ) + +# Note: disk check is auto-detected from disks.ini — no PROFILE_SKIP_DISK_CHECK needed. +# check_remote_disks() reads fsType per disk and handles XFS, ZFS, and cache pools automatically. + +# ============================================================================================== +# ── FALLBACK ────────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# Mutual container failover between two unRAID servers. +# Each server runs Fallback/fallback.sh independently via array_started.sh. +# All decisions based on two pings: remote reachable + internet reachable. +# +# States: NORMAL | FALLBACK | NO_INTERNET | DARK +# +# DDNS rules — absolute: +# Internet loss → stop own DDNS immediately +# Failover → start remote DDNS first (Tier 1) +# Handback → stop remote DDNS → rsync → start containers → start local DDNS last +# +# Per-host container lists and tier delays live in host*.conf. +# Shared settings (intervals, state file, thresholds) live here. + + EXTERNAL_IP="8.8.8.8" + FALLBACK_CHECK_INTERVAL=30 # seconds between fallback state checks + FALLBACK_HANDBACK_STRIKES=3 # consecutive healthy checks before initiating handback (3×30s = 90s) + FALLBACK_STATE_FILE="/boot/config/fallback_state.db" + FALLBACK_ENABLED=true # HOST2 back online + # false = suppresses "not running" warnings in status scripts + +# ━━━ Failover Test ━━━ +# Controlled simulation of a failover event — run manually via fallback_test.sh. + FALLBACK_TEST_BLOCK_WAIT=150 # seconds to wait after blocking connectivity + FALLBACK_TEST_HANDBACK_WAIT=360 # seconds to wait before initiating handback + +# ============================================================================================== +# ── DOCKER ESSENTIALS ───────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +# ━━━ Downloaders Reset ━━━ +# Runs every 30 minutes via CRITICAL_MAINTENANCE_SCRIPTS. +# Clears stuck states, purges old history, prepares each download client for a clean cycle. +# Per-host URLs and API keys live in host*.conf. + DOWNLOADER_RETENTION_DAYS=7 # days — purge history older than this + +# qBittorrent failsafe — removes torrents older than threshold regardless of ratio +# deleteFiles=false — removes from qBit but leaves files on disk (arr manages files) + QBIT_FAILSAFE_MIN_DAYS=180 # days — minimum age before failsafe deletion + QBIT_FAILSAFE_MIN_RATIO=0 # 0 = age only, no ratio requirement + +# ━━━ Docker Daily Restart ━━━ +# Containers restarted every day via DAILY_MAINTENANCE_SCRIPTS. +# Per-host lists live in host*.conf: +# HOST1_DAILY_RESTART_CONTAINERS +# HOST2_DAILY_RESTART_CONTAINERS +# detect_hosts() sets DAILY_RESTART_CONTAINERS to the correct host array at runtime. + +# ━━━ Docker Weekly Restart ━━━ +# Less critical services restarted weekly via WEEKLY_MAINTENANCE_SCRIPTS (Sunday 2:30am). +# Containers already stopped for weekly sync — restart adds zero extra downtime. +# Per-host lists live in host*.conf: +# HOST1_WEEKLY_RESTART_CONTAINERS +# HOST2_WEEKLY_RESTART_CONTAINERS + +# ━━━ Docker Watchdog ━━━ +# Continuous two-tier self-healing container monitoring. +# Started by array_started.sh — runs until array stops. +# Re-sources all three conf files each cycle — add/remove containers without restarting watchdog. +# +# Tier 1 — strict monitoring of explicitly configured containers: +# Memory hard limits — immediate restart if exceeded +# CPU thresholds — strike system, restart after CPU_FAIL_LIMIT sustained strikes +# HTTP responsiveness — strike system, restart after RESP_FAIL_LIMIT failed checks +# Required containers — must always be running, strike + skip list with auto-clear +# +# Tier 2 — global health scan of ALL running containers: +# Unhealthy status — Docker HEALTHCHECK unhealthy → restart +# OOM killed — kernel killed → restart + notify +# Crash loop detection — RestartCount climbing → notify, critical above limit +# Dead containers — remove and restart +# Unexpected exits — non-zero exit code → restart +# +# All per-host container lists live in host*.conf: +# HOST*_WATCHDOG_CONTAINERS — memory hard limits per container +# HOST*_WATCHDOG_CONTAINER_URLS — HTTP health check URLs +# HOST*_WATCHDOG_REQUIRED_CONTAINERS — must always be running +# HOST*_WATCHDOG_SCAN_IGNORE — skip in Tier 2 scan +# HOST*_WATCHDOG_DEPENDENCIES — dependency ordering for restart decisions + +# Strike state file — /tmp resets on reboot (correct — no stale strikes after reboot) + WATCHDOG_STATE_FILE="/tmp/container_watchdog_state.db" + +# CPU thresholds — normalised against total core count at runtime + SOFT_CPU_THRESHOLD=80 # warn at this % of total system CPU + HARD_CPU_THRESHOLD=85 # strike at this % of total system CPU + CPU_FAIL_LIMIT=2 # consecutive hard CPU strikes before container restart + +# Memory soft threshold — warn when container reaches this % of its hard limit + SOFT_MEM_THRESHOLD=80 + +# HTTP responsiveness + RESP_FAIL_LIMIT=2 # consecutive failed checks before restart + CURL_TIMEOUT=5 # seconds per check before timeout + +# Tier 2 master toggle + WATCHDOG_SCAN_ALL=true # false = only WATCHDOG_CONTAINERS + required containers + +# Individual Tier 2 check toggles + WATCHDOG_RESTART_UNHEALTHY=true + WATCHDOG_RESTART_DEAD=true + WATCHDOG_RESTART_CRASHED=true + WATCHDOG_NOTIFY_OOM=true + WATCHDOG_NOTIFY_CRASHLOOP=true + +# Crash loop threshold + WATCHDOG_CRASH_LIMIT=5 # RestartCount above this = critical crash loop + +# Startup grace period — skip restarts while system is still booting + WATCHDOG_STARTUP_GRACE=600 # seconds after boot before watchdog acts on failures + +# Restart loop protection — prevents watchdog from endlessly restarting a broken container + WATCHDOG_CONTAINER_RESTART_LIMIT=3 + WATCHDOG_CONTAINER_RESTART_WINDOW=1 # rolling window in hours + WATCHDOG_CONTAINER_RESTART_LOG="$DATA_DIR/container_restart_history.db" + +# Notification batching — one summary per cycle instead of one ping per event + WATCHDOG_BATCH_NOTIFY=true + +# Appdata size monitoring — two-part catch-all for runaway growth and oversized log files. +# +# Part 1 — Growth rate (zero-config): +# Reads per-container dir totals each cycle via du, compares to previous cycle. +# Any container growing more than WATCHDOG_APPDATA_GROWTH_GB triggers a focused *.log scan +# inside that container. No per-container config required — new containers are covered +# automatically. Baseline built on first run after boot; growth detection starts cycle 2. +# +# Part 2 — Absolute log size: +# Finds *.log / *.log.* files over WATCHDOG_APPDATA_LOG_MAX_GB across all appdata paths. +# Catches logs that have already stabilised at a large size and are no longer actively growing. +# +# Strike system (reuses existing watchdog infrastructure): +# Strike 1 — warn + notify: condition first detected +# Strike 2 — warn + escalated notify: still present next cycle +# Strike 3 (WATCHDOG_APPDATA_STRIKE_LIMIT) — action cycle: +# If WATCHDOG_APPDATA_TRUNCATE_LOGS=true: truncate *.log files in-place, clear strikes +# If false: critical notify only, strikes held until condition resolves +# Condition resolves (growth stops / log drops below threshold) → strikes auto-clear +# +# HOST*_WATCHDOG_APPDATA_SIZES (in host*.conf) suppresses growth warnings for a +# container until its dir exceeds the configured ceiling. Only needed when a container +# legitimately has large stable data and you want to guarantee it never triggers a false alarm. + WATCHDOG_CHECK_APPDATA=true + WATCHDOG_APPDATA_PATHS=("/mnt/docker-unraid/appdata") + WATCHDOG_APPDATA_GROWTH_GB=2 # flag containers growing more than this per cycle + WATCHDOG_APPDATA_LOG_MAX_GB=2 # flag *.log files exceeding this size (absolute) + WATCHDOG_APPDATA_TRUNCATE_LOGS=false # set true to auto-truncate oversized *.log files on action cycle + WATCHDOG_APPDATA_STRIKE_LIMIT=3 # cycles before action fires (matches existing watchdog pattern) + WATCHDOG_APPDATA_GROWTH_FILE="/tmp/watchdog_appdata_growth.db" # /tmp — resets on reboot ✅ + STORAGE_WATCHDOG_STATE_FILE="/tmp/storage_watchdog_state.db" # /tmp — resets on reboot ✅ + +# ━━━ Docker Network Connect ━━━ +# Ensures custom networks exist and connects containers at array start. +# Runs once via ARRAY_START_SCRIPTS — idempotent, safe to re-run. +# Container and network lists are host-specific — defined in host*.conf: +# HOST1_NETWORK_CONNECT_CONTAINERS / HOST2_NETWORK_CONNECT_CONTAINERS +# HOST1_NETWORK_CONNECT_NETWORKS / HOST2_NETWORK_CONNECT_NETWORKS + +# ============================================================================================== +# ── UNRAID ESSENTIALS ───────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +# ━━━ inotify Tuning ━━━ +# Linux inotify limits — applied at every array start by inotify_tuning.sh. +# Default unRAID values are very low — with many Docker containers watching files +# you can silently exhaust the limit causing containers to miss file events. +# Symptoms: containers miss file events, downloads not detected, library not updated. +# Live TV stutter is a known symptom of inotify exhaustion. +# These settings are lost on reboot — reapplied automatically at array start. + INOTIFY_MAX_INSTANCES=1024 # default: 128 — max inotify instances per user + # 1024 handles ~20-30 containers watching files simultaneously + INOTIFY_MAX_WATCHES=1048576 # default: 8192 — SHARED budget across ALL users/containers + # 524288 (512K) was previous value — raised to 1M (1048576) + # VSCode/Code-Server alone needs ~50K-200K for large workspaces + # with node_modules. All arr containers + Emby + VSCode share + # this budget. 1M is safe on 128GB RAM (~128MB kernel memory) + # If VSCode shows "unable to watch for file changes" → too low + INOTIFY_MAX_QUEUED_EVENTS=32768 # default: 16384 — max events queued before dropping + +# ━━━ System Tuning Monitor ━━━ +# Tracks inotify and php-fpm usage over time. +# Snapshot written every 6 hours by system_tuning_monitor.sh. +# Read by sunday_morning_coffee_report.sh for weekly peak/avg/warning summary. + INOTIFY_WARN_PCT=80 # warn if inotify instances exceed this % of limit + PHP_FPM_WARN_PCT=80 # warn if php-fpm workers exceed this % of max_children + TUNING_MONITOR_LOG="$DATA_DIR/system_tuning_history.db" + TUNING_LOG_RETENTION=30 # days before old entries are purged + +# ━━━ Reboot ━━━ +# Seconds of warning broadcast to logged-in users before server_reboot.sh reboots. +# Gives users time to save work — 300s = 5 minutes. + REBOOT_SLEEP=300 + +# ━━━ Mover ━━━ +# Seconds to wait before mover_stop.sh sends SIGTERM to the mover process. +# Gives mover time to finish current file transfer before being interrupted. + MOVER_STOP_TIMEOUT=300 + +# ━━━ Syslog Filter ━━━ +# Path for the rsyslog filter file that suppresses Docker veth interface noise. +# Docker creates a new veth interface for each container — generates hundreds of +# log lines per hour that have no diagnostic value. Filter removes them at source. + FILTER_FILE="/etc/rsyslog.d/ignore-docker-veth.conf" + +# ━━━ PHP-FPM ━━━ +# Higher max_children allows more concurrent PHP requests to the unRAID WebGUI. +# Default is very low — increasing it prevents WebGUI slowdowns under load. +# 250 is safe for servers with 32GB+ RAM. + PHP_CONF="/etc/php-fpm.d/www.conf" + PHP_MAX_CHILDREN=250 + +# ━━━ Clear Logs ━━━ +# System log files cleared weekly to prevent rootfs fill over time. +# These grow continuously — without clearing they eventually consume all rootfs space. +# clear_logs.sh runs in WEEKLY_MAINTENANCE_SCRIPTS — Sunday 2:30am. +# +# Size threshold approach — only clear if log exceeds threshold. +# Avoids destroying useful recent diagnostic context when logs are small. +# LOG_MIN_SIZE_MB: skip clearing if log is under this size (not worth clearing) +# LOG_DOCKER_MAX_MB: clear a container log only if it exceeds this size +# Docker logs grow fastest on active containers (Emby, SABnzbd, Sonarr) +# 100MB per container × 30 containers = 3GB before clearing kicks in + LOG_FILES=(/var/log/syslog /var/log/messages /var/log/dmesg) + LOG_MIN_SIZE_MB=10 # skip system log if under this size (already small) + LOG_DOCKER_MAX_MB=100 # clear Docker container log only if over this size (MB) + +# ━━━ Network Watchdog ━━━ +# Services-layer connectivity — internet reachability, DDNS sync, Tailscale, NPM proxy. +# Host-specific values (domain, container, NPM URL) live in host*.conf. + NETWORK_WATCHDOG_ENABLED=true + NETWORK_WATCHDOG_INTERNET_URL="https://1.1.1.1" + NETWORK_WATCHDOG_INTERNET_TIMEOUT=5 + NETWORK_WATCHDOG_CHECK_TAILSCALE=true + NETWORK_WATCHDOG_NPM_TIMEOUT=10 + NETWORK_WATCHDOG_NPM_STRIKE_LIMIT=2 + NETWORK_WATCHDOG_NPM_STATE_FILE="/tmp/network_watchdog_state.db" + +# ━━━ WebGUI Watchdog ━━━ +# Monitors unRAID WebGUI responsiveness — escalates through nginx restart → emhttp restart. +# Separate from docker_watchdog — this monitors the unRAID UI itself, not containers. + WEBGUI_URL="http://localhost" + WEBGUI_TIMEOUT=5 # seconds before curl gives up on WebGUI check + WEBGUI_NGINX_WAIT=15 # seconds after nginx restart before rechecking + WEBGUI_EMHTTP_WAIT=30 # seconds after emhttp restart before rechecking + +# ============================================================================================== +# ── MEDIA ───────────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +# ━━━ Media Permissions ━━━ +# Applied recursively by media_shares_permissions.sh. +# Runs first in DAILY_MAINTENANCE_SCRIPTS — arr cleanup depends on correct ownership. +# +# Why split directory vs file permissions: +# Directories need execute bit to enter — 755 allows owner+group+others to traverse +# Files should NOT be executable — 664 allows owner+group read/write, others read +# This is POSIX best practice for media servers, not a blanket 777 band-aid +# +# Why nobody:users ownership: +# All ecosystem containers run as PUID=99 (nobody) PGID=100 (users) on unRAID +# Files owned by nobody:users are accessible to all containers without world-write +# If this script fixes many files each run → a container has wrong PUID/PGID set +# Fix: add PUID=99 PGID=100 to the container's environment variables +# +# This script runs daily as a failsafe — even with correct container config: +# rsync may bring files with source ownership if not run with --chown +# Manual admin copies create root:root files +# New containers may not have PUID/PGID set yet + PERMISSIONS_DIR_MODE="755" # directories — traverse + list, no world-write + PERMISSIONS_FILE_MODE="664" # files — group read/write, no execute + PERMISSIONS_OWNER="nobody:users" + +# Share list defined per host in host*.conf: +# HOST1_MEDIA_PERMISSION_SHARES / HOST2_MEDIA_PERMISSION_SHARES + +# ━━━ Media Cleaner ━━━ +# Removes junk files from media shares — two profiles: anime and media. +# Called via DAILY_MAINTENANCE_SCRIPTS. Run manually: Media/media_cleaner.sh anime|media +# Folder lists defined per host in host*.conf: +# HOST1_ANIME_CLEAN_FOLDERS / HOST2_ANIME_CLEAN_FOLDERS +# HOST1_MEDIA_CLEAN_FOLDERS / HOST2_MEDIA_CLEAN_FOLDERS + + ANIME_FILE_PATTERNS=( + # ── Checksums ──────────────────────────────────────────────────────────────────────── + '*.sfv' '*.md5' '*.sha1' # verification files — useless post-download + + # ── Scene / download metadata ───────────────────────────────────────────────────────── + '*.url' '*.lnk' # scene links + '*.info' '*.diz' # scene description files + '*.nzb' # usenet download files + '*.torrent' # torrent files left by download clients + + # ── Archives and segments ───────────────────────────────────────────────────────────── + '*.rar' '*.zip' '*.7z' '*.ace' # archives — source files not needed after extract + '*.r00' '*.r01' '*.r02' '*.r03' # multi-part rar segments + '*.r04' '*.r05' '*.r06' '*.r07' + '*.r08' '*.r09' '*.srr' # scene repair files + '*.001' '*.002' '*.003' # split archive parts + '*.gz' '*.tar' '*.bz2' # linux archives + + # ── Scene samples and proofs ────────────────────────────────────────────────────────── + '*.sample*' '*.proof*' # scene samples — never needed in library + + # ── Executables and scripts ─────────────────────────────────────────────────────────── + '*.exe' '*.scr' '*.com' # Windows executables + '*.bat' '*.cmd' '*.vbs' '*.ps1' # Windows scripts + '*.msi' '*.dll' '*.sys' # Windows installers and system files + '*.sh' # shell scripts in media folders = suspicious + + # ── Incomplete downloads ────────────────────────────────────────────────────────────── + '*.!ut' '*.!qB' # uTorrent / qBittorrent incomplete markers + '*.crdownload' '*.opdownload' # Chrome / Opera incomplete downloads + '*.part' # partial download files + + # ── Sync conflicts ──────────────────────────────────────────────────────────────────── + '*sync-conflict*' # Syncthing conflict copies + ) + + MEDIA_FILE_PATTERNS=( + # ── Checksums ──────────────────────────────────────────────────────────────────────── + '*.sfv' '*.md5' '*.sha1' # verification files — useless post-download + + # ── Scene / download metadata ───────────────────────────────────────────────────────── + '*.url' '*.lnk' # scene links + '*.info' '*.diz' # scene description files + '*.nzb' # usenet download files + '*.torrent' # torrent files left by download clients + + # ── Archives and segments ───────────────────────────────────────────────────────────── + '*.rar' '*.zip' '*.7z' '*.ace' # archives — source files not needed after extract + '*.r00' '*.r01' '*.r02' '*.r03' # multi-part rar segments + '*.r04' '*.r05' '*.r06' '*.r07' + '*.r08' '*.r09' '*.srr' # scene repair files + '*.001' '*.002' '*.003' # split archive parts + '*.gz' '*.tar' '*.bz2' # linux archives + + # ── Scene samples and proofs ────────────────────────────────────────────────────────── + '*.sample*' '*.proof*' # scene samples — never needed in library + + # ── Executables and scripts ─────────────────────────────────────────────────────────── + '*.exe' '*.scr' '*.com' # Windows executables + '*.bat' '*.cmd' '*.vbs' '*.ps1' # Windows scripts + '*.msi' '*.dll' '*.sys' # Windows installers and system files + '*.sh' # shell scripts in media folders = suspicious + + # ── Incomplete downloads ────────────────────────────────────────────────────────────── + '*.!ut' '*.!qB' # uTorrent / qBittorrent incomplete markers + '*.crdownload' '*.opdownload' # Chrome / Opera incomplete downloads + '*.part' # partial download files + + # ── Sync conflicts ──────────────────────────────────────────────────────────────────── + '*sync-conflict*' # Syncthing conflict copies + + # ── Media-specific extras ───────────────────────────────────────────────────────────── + '*.iso' # disc images — extracted content is in library + '*.lrc' # lyrics files — handled by music apps not Emby + ) + +# ━━━ Arr Cleanup ━━━ +# Orphan file cleanup via Lidarr, Sonarr, and Radarr APIs. +# Per-host URLs, API keys, and path maps live in host*.conf. +# detect_hosts() selects correct host vars at runtime. +# +# API versions — update MAJOR version here when script is updated for a new arr version: +# Sonarr v4 → /api/v3/series → /api/v3/episodefile?seriesId=X +# Radarr v6 → /api/v3/movie → /api/v3/moviefile?movieId=X +# Lidarr v3 → /api/v1/artist → /api/v1/trackFile?artistId=X + SONARR_VERSION_MAJOR=4 + RADARR_VERSION_MAJOR=6 + LIDARR_VERSION_MAJOR=3 + +# Lidarr shared settings + LIDARR_LOCK_WARN_AGE=3600 # 1hr — large libraries take time, not stuck + LIDARR_ORPHAN_AGE=7 # days — files must be older than this before eligible for deletion + LIDARR_MAX_DELETE_GB=5 # require --i-know-what-im-doing if deletion exceeds this + LIDARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run + # protects against API returning partial data on a bad day + LIDARR_TRACKED_COUNT_FILE="$DATA_DIR/lidarr_tracked.count" + LIDARR_EXTENSIONS=("flac" "mp3" "m4a" "wav" "aac" "ogg" "opus" "wma") + LIDARR_PROTECTED_PATTERNS=( + # Metadata + "*.nfo" "*.tbn" + # Images — album art, artist images, Emby artwork + "*.jpg" "*.jpeg" "*.png" "*.webp" "*.svg" + "poster.*" "fanart.*" "backdrop.*" "clearlogo.*" + "banner.*" "thumb.*" "landscape.*" + "folder.*" "cover.*" "album.*" "artist.*" "disc.*" + # Lyrics + "*.lrc" + ) + +# Lidarr artwork fetcher settings + LIDARR_ART_MIN_SIZE=10000 # bytes — reject downloads smaller than this + LIDARR_ART_MAX_PARALLEL=4 # concurrent background download jobs + LIDARR_ART_RETRIES=2 # download retry attempts per image + LIDARR_ART_SLEEP_BETWEEN=0.2 # seconds between fanart.tv API calls +# HOST*_FANART_API_KEY / HOST*_LASTFM_API_KEY — set in host*.conf + +# Lidarr discovery settings (playback_aware_lidarr_discovery.sh) + LIDARR_DISCOVERY_THRESHOLD=70 # score to accept candidate (0-100) + LIDARR_DISCOVERY_LOOKBACK_DAYS=7 # Emby play history window in days + LIDARR_DISCOVERY_MIN_PLAYS=3 # min plays in window before evaluating an artist + LIDARR_DISCOVERY_USER_CAP_PCT=35 # max % any single user can contribute to play score (prevents one listener dominating) + LIDARR_DISCOVERY_MAX_ADDS=5 # max artists to add per run — quality over bulk + LIDARR_DISCOVERY_REJECT_COOLDOWN=30 # days before re-evaluating a rejected artist + LIDARR_DISCOVERY_HISTORY="$DATA_DIR/lidarr_discovery_history.db" + +# Sonarr discovery settings (playback_aware_sonarr_discovery.sh) + SONARR_DISCOVERY_THRESHOLD=52 # score to accept candidate (0-100) + SONARR_DISCOVERY_LOOKBACK_DAYS=14 # Emby episode play history window in days (shorter than movies — TV watched more frequently) + SONARR_DISCOVERY_MAX_SEEDS=5 # max seed series from Stage 1 + SONARR_DISCOVERY_MAX_ADDS=3 # max shows to add per run — TV is a larger commitment than movies + SONARR_DISCOVERY_MIN_VOTE_COUNT=50 # min TMDB votes (TV has fewer votes than movies at same popularity) + SONARR_DISCOVERY_MIN_RATING=65 # min TMDB vote_average × 10 (65 = 6.5/10) + SONARR_DISCOVERY_REJECT_COOLDOWN=60 # days before re-evaluating a rejected show + SONARR_DISCOVERY_USER_EPISODE_CAP=8 # max episodes any one user contributes to seed volume score + SONARR_DISCOVERY_MONITOR_MODE="all" # Sonarr monitor mode on add: all | future | first | latest | none + SONARR_DISCOVERY_HISTORY="$DATA_DIR/sonarr_discovery_history.db" + +# Radarr discovery shared settings + RADARR_DISCOVERY_THRESHOLD=52 # score to accept candidate (0-100) — lower than Lidarr since diverse seeds rarely overlap + RADARR_DISCOVERY_LOOKBACK_DAYS=30 # Emby watch history window in days (movies rewatched less often) + RADARR_DISCOVERY_MAX_SEEDS=5 # max seed movies from Stage 1 + RADARR_DISCOVERY_MAX_ADDS=5 # max movies to add per run + RADARR_DISCOVERY_MIN_VOTE_COUNT=100 # min TMDB votes to be considered a candidate + RADARR_DISCOVERY_MIN_RATING=60 # min TMDB vote_average × 10 (60 = 6.0/10) + RADARR_DISCOVERY_REJECT_COOLDOWN=60 # days before re-evaluating a rejected movie + RADARR_DISCOVERY_SEED_LIBRARIES=("Movies") # Emby libraries to draw seed movies from + RADARR_DISCOVERY_HISTORY="$DATA_DIR/radarr_discovery_history.db" + +# Emby → arr sync library allowlists +# Only these Emby library names will be considered by the sync tools. +# Names must match exactly as shown in Emby > Dashboard > Libraries. + SONARR_EMBY_LIBRARIES=("Anime" "Kids Shows" "Stand-Up Comedy" "TV shows") + RADARR_EMBY_LIBRARIES=("Movies" "Kid's Movies") + +# Sonarr shared settings + SONARR_ORPHAN_AGE=7 # days — files must be older than this before eligible for deletion + SONARR_MAX_DELETE_GB=10 # require --i-know-what-im-doing if deletion exceeds this + SONARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "ts" "wmv" "mov") + SONARR_PROTECTED_PATTERNS=( + # Subtitles + "*.srt" "*.sub" "*.ass" "*.ssa" "*.idx" "*.vtt" + # Metadata + "*.nfo" "*.tbn" + # Images + "*.jpg" "*.jpeg" "*.png" "*.webp" "*.svg" + "poster.*" "fanart.*" "backdrop.*" "clearlogo.*" + "banner.*" "thumb.*" "landscape.*" + # Kodi/Emby extras — not tracked by Sonarr API + "*-trailer.*" "*-featurette.*" "*-behindthescenes.*" + "*-interview.*" "*-scene.*" "*-short.*" "*-deleted.*" + "*-clip.*" "*-other.*" + # Theme songs — stored in show folder, not tracked by arr + "theme.mp3" "theme.flac" "theme.wav" "theme.m4a" "theme.mka" + ) + +# Radarr shared settings + RADARR_ORPHAN_AGE=7 # days — files must be older than this before eligible for deletion + RADARR_MAX_DELETE_GB=15 # require --i-know-what-im-doing if deletion exceeds this + RADARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "wmv" "mov") + RADARR_PROTECTED_PATTERNS=( + # Subtitles + "*.srt" "*.sub" "*.ass" "*.ssa" "*.idx" "*.vtt" + # Metadata + "*.nfo" "*.tbn" + # Images + "*.jpg" "*.jpeg" "*.png" "*.webp" "*.svg" + "poster.*" "fanart.*" "backdrop.*" "clearlogo.*" + "banner.*" "thumb.*" "landscape.*" + # Kodi/Emby extras — not tracked by Radarr API + "*-trailer.*" "*-featurette.*" "*-behindthescenes.*" + "*-interview.*" "*-scene.*" "*-short.*" "*-deleted.*" + "*-clip.*" "*-other.*" + # Theme songs — stored in movie folder, not tracked by arr + "theme.mp3" "theme.flac" "theme.wav" "theme.m4a" "theme.mka" + ) + +# Radarr/Sonarr TMDb/TVDB removed entry cleanup + RADARR_DROPPED_ADD_EXCLUSION=true # add removed movies to import exclusion list + SONARR_DROPPED_ADD_EXCLUSION=true # add removed series to import exclusion list + +# ━━━ Arr Failed/Stalled Recovery ━━━ +# Auto blocklist + re-search failed imports and stalled downloads. +# Runs every 6 hours — schedule: 0 */6 * * * +# +# Targets four problem types: +# importFailed — downloaded but arr couldn't import +# importPending — downloaded, stuck waiting to import (won't self-resolve) +# error status — serious failure not covered above +# stalled — download stuck with no connections or progress +# +# Items newer than ARR_IMPORT_RECOVERY_AGE are skipped — gives arr time to retry first. +# Per-host recovery toggles (HOST1_SONARR_RECOVERY etc.) live in host*.conf. + ARR_IMPORT_RECOVERY_AGE=6 # hours — skip items newer than this + # matches cron interval — items eligible after one missed cycle + +# ============================================================================================== +# ── TRANSCODES ──────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# Session-based storage allocator using filesystem symlink indirection. +# ffmpeg resolves the symlink ONCE at session start — existing sessions never affected. +# Symlink flips between ramdisk and SSD are transparent to active streams. +# +# ⚠️ Docker mount — must use shared propagation so symlink flips work inside container: +# --mount type=bind,source=/mnt/ram-transcode,target=/ext-ram-transcode,bind-propagation=shared +# Standard path mappings use rprivate — symlink changes are NOT visible inside container. + +# ━━━ Transcode Manager ━━━ +# tmpfs mount point — created at array start by ramdisk_setup.sh. +# Must exist before Emby starts so the symlink resolves correctly. + RAMDISK_PATH="/mnt/ramdisk_transcodes" + +# Ramdisk size and flip thresholds — defined per host in host*.conf. +# All three are coupled — if size changes, thresholds must change with it. +# HOST1_RAMDISK_SIZE / HOST2_RAMDISK_SIZE +# HOST1_RAMDISK_WARN_GB / HOST2_RAMDISK_WARN_GB ← flip to SSD at this usage +# HOST1_RAMDISK_LOW_GB / HOST2_RAMDISK_LOW_GB ← flip back to ramdisk at this usage + +# Symlink that Emby points at — this path NEVER changes regardless of ramdisk/SSD state. +# Emby resolves the symlink once per session at start — symlink flips are transparent. +# Must match the container path configured in Emby's Extra Parameters. + TRANSCODE_LINK="/mnt/ram-transcode" + +# SSD fallback location — defined per host in host*.conf (cache path differs per server): +# HOST1_TRANSCODE_SSD / HOST2_TRANSCODE_SSD + +# Minimum free GB on SSD before allowing flip from ramdisk to SSD. + RAMDISK_SSD_MIN_GB=20 + + TRANSCODE_MAX_AGE=20 # minutes — HLS segment age before cleanup eligibility + TRANSCODE_ORPHAN_AGE=30 # minutes — files with no matching active session + TRANSCODE_FLIP_WARN=3 # notify if symlink flips this many times in one hour + + TRANSCODE_OWNER="nobody:users" + TRANSCODE_CHMOD="755" + +# Operating mode — controls symlink direction behaviour. +# smart — auto-flips between ramdisk and SSD based on thresholds (default) +# ramdisk — always uses ramdisk, warns if RAMDISK_WARN_GB exceeded but holds +# ssd — always uses SSD, never flips to ramdisk + TRANSCODE_MANAGER_MODE="smart" + +# Daily statistics log — read by weekly_health_digest.sh for transcode summary. + TRANSCODE_DAILY_LOG="$DATA_DIR/transcode_daily.db" + TRANSCODE_LOG_RETENTION=90 # days before old entries purged + + TRANSCODE_CHECK_EMBY=true + +# ━━━ Transcode Server Array ━━━ +# All media servers sharing the ramdisk transcode space. +# Format: "ContainerName|URL|APIKey|Type" — Type: emby | jellyfin | plex +# Entries with placeholder API keys are skipped automatically. +# ⚠️ Tdarr does NOT belong here — keep Tdarr on SSD, not ramdisk. +# Defined per host in host*.conf — Emby container names and keys differ per server: +# HOST1_TRANSCODE_SERVERS / HOST2_TRANSCODE_SERVERS + +# ============================================================================================== +# ── MONITORS ────────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +# ━━━ Certificate Monitor ━━━ +# Checks SSL certificate expiry via direct openssl connection — no NPM dependency. +# Checks the actual certificate served by each domain, not what NPM thinks it has. +# Domains defined per host in host*.conf — each server monitors its own domains: +# HOST1_CERT_MONITOR_DOMAINS / HOST2_CERT_MONITOR_DOMAINS + CERT_WARN_DAYS=30 # warn when cert expires within this many days + CERT_CRIT_DAYS=7 # critical alert within this many days + CERT_TIMEOUT=10 # seconds per domain before giving up + +# ━━━ Backup Verify ━━━ +# Verifies rsync mirror health by comparing random file checksums between servers. +# Catches silent corruption or incomplete syncs that rsync itself wouldn't detect. +# Defined per host in host*.conf — leave empty to use HOST*_DAILY_SYNC_SHARES automatically: +# HOST1_BACKUP_VERIFY_SHARES / HOST2_BACKUP_VERIFY_SHARES + BACKUP_VERIFY_SAMPLE=10 # random files to check per share + BACKUP_VERIFY_MIN_SIZE=1M # minimum file size to include in sample + +# ━━━ SMART Health ━━━ +# Monitors drive SMART attributes — discovers all drives via /dev/sd* and /dev/nvme*. +# Thresholds read from /boot/config/plugins/dynamix/dynamix.cfg at runtime +# (hot/max/hotssd/maxssd) — these vars are fallback only if dynamix.cfg not found. + SMART_TEMP_WARN=45 # fallback — Celsius warn threshold + SMART_TEMP_CRIT=55 # fallback — Celsius critical threshold +# Drives to ignore defined per host in host*.conf — hardware is server-specific: +# HOST1_SMART_IGNORE_DRIVES / HOST2_SMART_IGNORE_DRIVES + +# ━━━ ZFS Memory Snapshot ━━━ +# Weekly ZFS pool health and memory diagnostic report — informational only. + ZFS_REPORT_LOG="/var/log/zfs-weekly-health.log" + ZFS_REPORT_ARC_WARN_PCT=90 # warn if ARC using more than this % of its max + ZFS_REPORT_FREE_WARN_GB=10 # warn if less than this GB free RAM + ZFS_REPORT_AVAIL_WARN_GB=20 # warn if less than this GB available on ZFS pool + ZFS_REPORT_DOCKER_TOP=10 # how many top Docker containers to show by memory +# Pool ignore list defined per host in host*.conf — pool names are server-specific: +# HOST1_ZFS_REPORT_IGNORE_POOLS / HOST2_ZFS_REPORT_IGNORE_POOLS + +# ━━━ Bandwidth Monitor ━━━ +# Called automatically by rsync.sh after each sync — one bounded write per run. +# Tracks transfer size, duration and profile per sync for weekly summary reporting. + BANDWIDTH_LOG="$DATA_DIR/bandwidth_history.db" + BANDWIDTH_LOG_RETENTION=90 # days before old entries purged + BANDWIDTH_WARN_GB=50 # flag syncs larger than this in weekly report + +# Stats files — written by cleanup and recovery scripts, read by coffee report. +# All in DATA_DIR — array always running when these are written. + ARR_CLEANUP_STATS="$DATA_DIR/arr_cleanup_stats.db" # lidarr/sonarr/radarr orphan stats + ARR_RECOVERY_STATS="$DATA_DIR/arr_recovery_stats.db" # blocklist + re-search stats + +# ━━━ Health Digest ━━━ +# Aggregated system health summary — reads existing state files, no new writes. +# Three profiles control when the digest is sent: +# always — sends every run regardless of findings +# smart — sends only when DIGEST_SMART_ON_* conditions are found +# weekly — sends once per week on DIGEST_DAY only + DIGEST_PROFILE="weekly" # always | smart | weekly + DIGEST_DAY="Sunday" + DIGEST_SMART_ON_WATCHDOG=true # send if any watchdog strikes are active + DIGEST_SMART_ON_FALLBACK=true # send if fallback state is not NORMAL + DIGEST_SMART_ON_CERT_WARN=true # send if any cert is under CERT_WARN_DAYS + DIGEST_SMART_ON_BANDWIDTH=true # send if any transfer exceeded BANDWIDTH_WARN_GB + +# ━━━ Emby Session Report ━━━ +# Weekly Emby usage statistics via API — no persistent writes, queries fresh each run. +# URL and API key pulled from HOST*_EMBY_URL and HOST*_EMBY_API_KEY in host*.conf. + EMBY_REPORT_DAYS=7 # days to include in the report period + EMBY_REPORT_TOP_N=10 # number of top content items to show + +# ============================================================================================== +# ── RESOURCE MANAGER ────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# Pressure reduction layer — keeps the system comfortable before things break. +# Called by watchdog_orchestrator.sh. Single-pass, not a continuous loop. +# +# ── PRESSURE LEVELS ─────────────────────────────────────────────────────────────────────────── +# Level 1 (soft) — throttle SABnzbd + qBit download speeds +# Level 2 (medium) — further throttle + docker pause background containers +# Level 3 (hard) — docker stop optional containers, signal docker_watchdog to defer +# +# ── PER-HOST CONTAINER LISTS ────────────────────────────────────────────────────────────────── +# HOST*_RW_PAUSE_CONTAINERS — docker pause at medium pressure (in host*.conf) +# HOST*_RW_STOP_CONTAINERS — docker stop at hard pressure (in host*.conf) + + RW_ENABLED=true + RW_STATE_FILE="/tmp/resource_watchdog_state.db" + +# ━━━ Pressure Thresholds ━━━ +# Graduated RAM response — resource_watchdog acts before system_watchdog reboots. +# RW_RAM_SOFT_GB > RW_RAM_MEDIUM_GB > RW_RAM_HARD_GB > SYS_WATCHDOG_MEM_GB always + RW_RAM_SOFT_GB=12 # throttle start — reduce background load + RW_RAM_MEDIUM_GB=8 # pause background containers + RW_RAM_HARD_GB=6 # stop optional containers (was SYS_WATCHDOG_MEM_SHUTDOWN_GB) + RW_RAM_RECOVER_GB=20 # RAM must reach this before restoring hard-stopped containers + +# Load average thresholds — multiplier × core count + RW_LOAD_SOFT_MULTIPLIER=2.0 # soft pressure: 2× cores sustained + RW_LOAD_MEDIUM_MULTIPLIER=3.0 # medium pressure: 3× cores sustained + +# Consecutive runs at lower pressure before de-escalating + RW_RECOVER_CYCLES=3 + +# ━━━ SABnzbd Throttle ━━━ +# Speed values: "50M" = 50 MB/s, "0" = unlimited + RW_SABNZBD_ENABLED=true + RW_SABNZBD_SPEED_SOFT="50M" + RW_SABNZBD_SPEED_MEDIUM="10M" + +# ━━━ qBittorrent Throttle ━━━ +# KB/s — 0 = unlimited + RW_QBIT_ENABLED=true + RW_QBIT_DL_SOFT=51200 # 50 MB/s + RW_QBIT_DL_MEDIUM=10240 # 10 MB/s + +# ━━━ Critical Containers ━━━ +# Never paused or stopped regardless of pressure level. +# Keep DNS, auth, media serving, and live TV always running. + RW_CRITICAL_CONTAINERS=( + "NginxProxyManager" # reverse proxy — internet access + "Authelia" # auth — nothing accessible without it + "Authelia-Secondary" + "Mariadb-Authelia" # Authelia dependency + "Mariadb-Authelia-Secondary" + "Redis-Authelia" # Authelia dependency + "Redis-Authelia-Secondary" + "AdGuard-Home" # DNS — all LAN resolution + "Emby" # media server — Live TV buffering + "Dispatcharr" # Live TV scheduler — loses state if stopped + "Dispatcharr-Basic" + "Dispatcharr-Iptv-Users" + ) + +# ============================================================================================== +# ── SYSTEM WATCHDOG ─────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# Single-pass system health check — last line of defense before a crash. +# Called by watchdog_orchestrator.sh every minute. NOT started by array_started.sh. +# Re-sources config at each orchestrator run — config changes take effect immediately. +# +# ── THREE-TIER RESPONSE SYSTEM ──────────────────────────────────────────────────────────────── +# CRITICAL — bypass ALL strikes, reboot immediately +# Docker daemon down, rootfs 100%, kernel oops, FD exhaustion, /boot read-only +# +# URGENT — bypass strikes only when OOM confirms active crisis +# RAM < MEM_GB AND OOM kills >= OOM_LIMIT in this cycle → reboot NOW +# Without OOM confirmation → normal strike system +# +# STANDARD — strike system (N consecutive failures → reboot) +# RAM tiers, high load, CPU temp, zombies, /var/log, /tmp, containers +# +# ── RAM ─────────────────────────────────────────────────────────────────────────────────────── +# SYS_WATCHDOG_MEM_GB — strike system → reboot (or OOM bypass) +# Warn/shutdown/recover RAM tiers are handled by resource_watchdog.sh + +# ━━━ State Files ━━━ + SYS_WATCHDOG_STATE_FILE="/tmp/system_watchdog_state.db" # /tmp — resets on reboot ✅ + SYS_WATCHDOG_FAILED_FILE="/boot/config/system_watchdog_failed.db" + SYS_WATCHDOG_REBOOT_LOG="/boot/config/system_watchdog_reboots.db" + SYS_WATCHDOG_OOM_FILE="/tmp/system_watchdog_oom.db" # /tmp — resets on reboot ✅ + +# ━━━ Strike and Reboot Loop Settings ━━━ +# Strike system: a check must fail this many consecutive cycles before action is taken. +# Single spikes (one bad reading) are ignored — sustained problems trigger reboot. + SYS_WATCHDOG_STRIKE_LIMIT=2 # consecutive failures before reboot trigger + +# Reboot loop protection — if system keeps rebooting something is seriously wrong. +# After REBOOT_LIMIT reboots in REBOOT_WINDOW_HRS → shutdown instead of reboot. + SYS_WATCHDOG_REBOOT_LIMIT=3 + SYS_WATCHDOG_REBOOT_WINDOW_HRS=12 + +# ━━━ RAM Reboot Threshold ━━━ +# Reboot trigger only — warn/shutdown/recover handled by resource_watchdog.sh +# RW_RAM_HARD_GB > SYS_WATCHDOG_MEM_GB always (RM acts before watchdog reboots) + SYS_WATCHDOG_MEM_GB=4 # strike system → reboot + +# ━━━ OOM Bypass Settings ━━━ +# OOM bypass: if RAM is critically low AND kernel OOM kills exceed this threshold +# in a single cycle → bypass strike system and reboot immediately. +# Rate-based: kills per 5-minute cycle, not absolute count. +# Rationale: 1-2 kills = docker_watchdog handles it ✅ +# 3+ kills while RAM critical = system dying faster than watchdogs can heal ✅ + SYS_WATCHDOG_OOM_LIMIT=3 # OOM kills in one cycle to trigger bypass + +# ━━━ Thresholds ━━━ +# Set at "about to become unstable" — not "things are a bit high". + +# rootfs (/) usage — two levels: strike at 95%, critical bypass at 99% + SYS_WATCHDOG_ROOTFS_PCT=95 # standard strike threshold + SYS_WATCHDOG_ROOTFS_CRITICAL_PCT=99 # bypass strikes — truly full, writes failing + +# /var/log usage — log spam indicates something broken + SYS_WATCHDOG_LOG_PCT=95 + +# /tmp usage — tmpfs fills from downloads, lock files fail at 100% + SYS_WATCHDOG_TMP_PCT=90 # warn + attempt clear + SYS_WATCHDOG_TMP_CRITICAL_PCT=98 # bypass strikes if clear failed + +# ZFS ARC pinned percentage — ARC not releasing after reclaim = memory stuck + SYS_WATCHDOG_ARC_PINNED_PCT=98 + SYS_WATCHDOG_ARC_RELEASE_PCT=95 + +# Load average multiplier — threshold = MULTIPLIER × CPU core count +# MULTIPLIER=3 on 16-core = load of 48 before triggering + SYS_WATCHDOG_LOAD_MULTIPLIER=3 + +# Zombie process count — large numbers = serious process management failure + SYS_WATCHDOG_ZOMBIE_LIMIT=50 + +# CPU temperature — sustained high temp causes throttling or kernel panic + SYS_WATCHDOG_CPU_TEMP_MAX=95 + +# File descriptor exhaustion — system-wide FD limit near exhaustion +# New connections fail silently, Docker can't spawn processes, SSH fails + SYS_WATCHDOG_FD_CRITICAL_PCT=95 # bypass strikes — critical tier + +# Runaway process — single non-container process consuming excessive CPU +# Multiple strikes before action — single spikes are normal + SYS_WATCHDOG_RUNAWAY_CPU_PCT=90 # % single process must sustain + SYS_WATCHDOG_RUNAWAY_STRIKES=3 # consecutive cycles before warning + +# Array disk errors — accumulating mdstat errors = disk failing NOW + SYS_WATCHDOG_MDSTAT_ERROR_LIMIT=5 # new errors in one cycle before acting + +# ━━━ Check Toggles ━━━ +# Per-host — moved to host*.conf +# Different servers may have different hardware, NICs, and check requirements +# See HOST*_SYS_WATCHDOG_CHECK_* in host*.conf + +# ━━━ Abort Toggles ━━━ +# Conditions that prevent reboot even when a threshold is hit. +# CRITICAL tier bypasses these — a truly critical condition reboots regardless. +# true = abort standard reboot if this condition is active +# false = reboot anyway + SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY=true # rebooting with bad pool risks data loss + SYS_WATCHDOG_ABORT_ON_PARITY=false # aborting parity better than crashing mid-check + SYS_WATCHDOG_ABORT_ON_MOVER=false # aborting move better than crashing mid-move + +# ============================================================================================== +# ──────────────────────── End Of User Variables ─────────────────────────────────────────────── +# ============================================================================================== \ No newline at end of file diff --git a/Deployment/conf_upgrade.sh b/Deployment/conf_upgrade.sh new file mode 100755 index 0000000..ada8b14 --- /dev/null +++ b/Deployment/conf_upgrade.sh @@ -0,0 +1,200 @@ +#!/bin/bash +# ============================================================================================== +# conf_upgrade.sh — Merge new conf structure into an existing conf while preserving user values. +# ============================================================================================== +# +# Usage: +# conf_upgrade.sh --template --target [--dry-run] [--backup] +# +# What it does: +# • Keys in template only → ADDED with template's placeholder/default — user fills in once +# • Keys in target only → REMOVED — deprecated in new version +# • Keys in both → KEPT — target's value always preserved, template's ignored +# • Comments / blank lines → always from template (structure follows new version) +# +# Supports all conf variable patterns: +# Simple scalar: KEY="value" +# Indexed array: KEY=(\n "item"\n) +# Associative array: declare -A KEY=(\n ["k"]="v"\n) +# +# Options: +# --template New version conf file (source of structure and defaults) +# --target Existing user conf (source of real values — preserved) +# --dry-run Show what would change without writing +# --backup Write a .bak copy of target before modifying +# +# ============================================================================================== + +set -uo pipefail + +# ── Arguments ──────────────────────────────────────────────────────────────────────────────── + +TEMPLATE="" +TARGET="" +DRY_RUN=false +BACKUP=false + +while [[ $# -gt 0 ]]; do + case "$1" in + --template) TEMPLATE="$2"; shift 2 ;; + --target) TARGET="$2"; shift 2 ;; + --dry-run) DRY_RUN=true; shift ;; + --backup) BACKUP=true; shift ;; + *) echo "Unknown option: $1" >&2; exit 1 ;; + esac +done + +[[ -z "$TEMPLATE" ]] && { echo "Error: --template required" >&2; exit 1; } +[[ -z "$TARGET" ]] && { echo "Error: --target required" >&2; exit 1; } +[[ -f "$TEMPLATE" ]] || { echo "Error: template not found: $TEMPLATE" >&2; exit 1; } +[[ -f "$TARGET" ]] || { echo "Error: target not found: $TARGET" >&2; exit 1; } + +# ── Parse target → KEY → full definition block ─────────────────────────────────────────────── + +declare -A HOST_MAP # KEY → complete definition line(s) from user's conf + +_parse_target() { + local in_block=false cur_key="" cur_block="" line + + while IFS= read -r line || [[ -n "$line" ]]; do + if [[ "$in_block" == true ]]; then + cur_block+="$line"$'\n' + # Closing ) — optional trailing whitespace and comment + if [[ "$line" =~ ^[[:space:]]*\)[[:space:]]*(#.*)?$ ]]; then + HOST_MAP["$cur_key"]="$cur_block" + in_block=false; cur_key=""; cur_block="" + fi + else + # declare -A KEY=( + if [[ "$line" =~ ^[[:space:]]*declare[[:space:]]+-[a-zA-Z]+[[:space:]]+([A-Z0-9_]+)[[:space:]]*=\( ]]; then + cur_key="${BASH_REMATCH[1]}"; in_block=true; cur_block="$line"$'\n' + # KEY=( + elif [[ "$line" =~ ^[[:space:]]*([A-Z0-9_]+)[[:space:]]*=\( ]]; then + cur_key="${BASH_REMATCH[1]}"; in_block=true; cur_block="$line"$'\n' + # KEY=value (simple scalar) + elif [[ "$line" =~ ^[[:space:]]*([A-Z0-9_]+)[[:space:]]*= ]]; then + HOST_MAP["${BASH_REMATCH[1]}"]="$line"$'\n' + fi + fi + done < "$TARGET" +} + +# ── Walk template — collect stats (must run in current shell so arrays persist) ────────────── + +declare -a ADDED=() KEPT=() REMOVED=() +declare -A TMPL_SEEN=() + +_collect_stats() { + local in_block=false cur_key="" line + + while IFS= read -r line || [[ -n "$line" ]]; do + if [[ "$in_block" == true ]]; then + if [[ "$line" =~ ^[[:space:]]*\)[[:space:]]*(#.*)?$ ]]; then + in_block=false + TMPL_SEEN["$cur_key"]=1 + if [[ -n "${HOST_MAP[$cur_key]+_}" ]]; then KEPT+=("$cur_key") + else ADDED+=("$cur_key"); fi + cur_key="" + fi + else + if [[ "$line" =~ ^[[:space:]]*declare[[:space:]]+-[a-zA-Z]+[[:space:]]+([A-Z0-9_]+)[[:space:]]*=\( ]]; then + cur_key="${BASH_REMATCH[1]}"; in_block=true + elif [[ "$line" =~ ^[[:space:]]*([A-Z0-9_]+)[[:space:]]*=\( ]]; then + cur_key="${BASH_REMATCH[1]}"; in_block=true + elif [[ "$line" =~ ^[[:space:]]*([A-Z0-9_]+)[[:space:]]*= ]]; then + local k="${BASH_REMATCH[1]}" + TMPL_SEEN["$k"]=1 + if [[ -n "${HOST_MAP[$k]+_}" ]]; then KEPT+=("$k") + else ADDED+=("$k"); fi + fi + fi + done < "$TEMPLATE" + + for key in "${!HOST_MAP[@]}"; do + [[ -z "${TMPL_SEEN[$key]+_}" ]] && REMOVED+=("$key") + done +} + +# ── Walk template — write merged output ────────────────────────────────────────────────────── +# Runs in a subshell (stdout redirected) — array mutations are intentionally discarded here. + +_write_merged() { + local in_block=false cur_key="" cur_block="" line + + while IFS= read -r line || [[ -n "$line" ]]; do + if [[ "$in_block" == true ]]; then + cur_block+="$line"$'\n' + if [[ "$line" =~ ^[[:space:]]*\)[[:space:]]*(#.*)?$ ]]; then + in_block=false + if [[ -n "${HOST_MAP[$cur_key]+_}" ]]; then printf '%s' "${HOST_MAP[$cur_key]}" + else printf '%s' "$cur_block"; fi + cur_key=""; cur_block="" + fi + else + if [[ "$line" =~ ^[[:space:]]*declare[[:space:]]+-[a-zA-Z]+[[:space:]]+([A-Z0-9_]+)[[:space:]]*=\( ]]; then + cur_key="${BASH_REMATCH[1]}"; in_block=true; cur_block="$line"$'\n' + elif [[ "$line" =~ ^[[:space:]]*([A-Z0-9_]+)[[:space:]]*=\( ]]; then + cur_key="${BASH_REMATCH[1]}"; in_block=true; cur_block="$line"$'\n' + elif [[ "$line" =~ ^[[:space:]]*([A-Z0-9_]+)[[:space:]]*= ]]; then + local k="${BASH_REMATCH[1]}" + if [[ -n "${HOST_MAP[$k]+_}" ]]; then printf '%s' "${HOST_MAP[$k]}" + else printf '%s\n' "$line"; fi + else + printf '%s\n' "$line" + fi + fi + done < "$TEMPLATE" +} + +# ── Run ─────────────────────────────────────────────────────────────────────────────────────── + +_parse_target +_collect_stats + +# ── Report ──────────────────────────────────────────────────────────────────────────────────── + +TARGET_NAME="$(basename "$TARGET")" +echo "" +echo "── conf_upgrade: $TARGET_NAME ──────────────────────────────────────────" + +if [[ ${#ADDED[@]} -gt 0 ]]; then + echo " ADDED (new — fill in your values where needed):" + for k in "${ADDED[@]}"; do echo " + $k"; done +fi + +if [[ ${#REMOVED[@]} -gt 0 ]]; then + echo " REMOVED (deprecated — no longer in this version):" + for k in "${REMOVED[@]}"; do echo " - $k"; done +fi + +echo " KEPT ${#KEPT[@]} existing vars — your values preserved" + +if [[ ${#ADDED[@]} -eq 0 && ${#REMOVED[@]} -eq 0 ]]; then + echo " Already up to date — no changes needed." + echo "────────────────────────────────────────────────────────────────────────" + echo "" + exit 0 +fi + +echo "────────────────────────────────────────────────────────────────────────" +echo "" + +# ── Apply ───────────────────────────────────────────────────────────────────────────────────── + +if [[ "$DRY_RUN" == true ]]; then + echo "(dry-run — no changes written)" + exit 0 +fi + +TMPOUT="$(mktemp)" +trap 'rm -f "$TMPOUT"' EXIT + +_write_merged > "$TMPOUT" + +if [[ "$BACKUP" == true ]]; then + cp "$TARGET" "${TARGET}.bak" + echo "Backup: ${TARGET}.bak" +fi + +cp "$TMPOUT" "$TARGET" +echo "Updated: $TARGET" diff --git a/Media/lidarr_cleanup.sh b/Media/lidarr_cleanup.sh index 4b93a87..21709cf 100644 --- a/Media/lidarr_cleanup.sh +++ b/Media/lidarr_cleanup.sh @@ -161,6 +161,9 @@ validate_unraid_cmd \ # Lock before detect_hosts — large library scans take time, wait mode appropriate [[ -n "${LIDARR_LOCK_WARN_AGE:-}" ]] && LOCK_WARN_AGE="$LIDARR_LOCK_WARN_AGE" acquire_lock "wait" +TMP_DIR="/tmp/lidarr_cleanup_$$" +mkdir -p "$TMP_DIR" +trap "_release_all_locks; rm -rf $TMP_DIR" EXIT if ! command -v docker &>/dev/null; then error "Docker command not found" @@ -402,10 +405,6 @@ fi info "Found $ARTIST_COUNT artists — fetching track files..." -TMP_DIR="/tmp/lidarr_cleanup_$$" -mkdir -p "$TMP_DIR" -trap "rm -rf $TMP_DIR" EXIT - TRACKED_FILE="$TMP_DIR/tracked_paths.txt" > "$TRACKED_FILE" diff --git a/Media/lidarr_missing_art.sh b/Media/lidarr_missing_art.sh index 16c685e..37bfe13 100644 --- a/Media/lidarr_missing_art.sh +++ b/Media/lidarr_missing_art.sh @@ -133,7 +133,7 @@ fi # ── Temp dir for subshell fetch/fail counters ───────────────────────────────────────────────── LIDARR_TMP=$(mktemp -d) -trap 'rm -rf "$LIDARR_TMP"' EXIT +trap '_release_all_locks; rm -rf "$LIDARR_TMP"' EXIT touch "$LIDARR_TMP/album_fetches" "$LIDARR_TMP/album_fails" \ "$LIDARR_TMP/artist_fetches" "$LIDARR_TMP/artist_fails" diff --git a/Media/radarr_cleanup.sh b/Media/radarr_cleanup.sh index 1ba54d3..15521b9 100644 --- a/Media/radarr_cleanup.sh +++ b/Media/radarr_cleanup.sh @@ -147,6 +147,9 @@ validate_unraid_cmd \ "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" acquire_lock "wait" +TMP_DIR="/tmp/radarr_cleanup_$$" +mkdir -p "$TMP_DIR" +trap "_release_all_locks; rm -rf $TMP_DIR" EXIT if ! command -v docker &>/dev/null; then error "Docker command not found" @@ -388,10 +391,6 @@ fi info "Found $MOVIE_COUNT movies — fetching movie files..." -TMP_DIR="/tmp/radarr_cleanup_$$" -mkdir -p "$TMP_DIR" -trap "rm -rf $TMP_DIR" EXIT - TRACKED_FILE="$TMP_DIR/tracked_paths.txt" > "$TRACKED_FILE" diff --git a/Media/sonarr_cleanup.sh b/Media/sonarr_cleanup.sh index f41fc2b..20967db 100644 --- a/Media/sonarr_cleanup.sh +++ b/Media/sonarr_cleanup.sh @@ -147,6 +147,9 @@ validate_unraid_cmd \ "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" acquire_lock "wait" +TMP_DIR="/tmp/sonarr_cleanup_$$" +mkdir -p "$TMP_DIR" +trap "_release_all_locks; rm -rf $TMP_DIR" EXIT if ! command -v docker &>/dev/null; then error "Docker command not found" @@ -388,10 +391,6 @@ fi info "Found $SERIES_COUNT series — fetching episode files..." -TMP_DIR="/tmp/sonarr_cleanup_$$" -mkdir -p "$TMP_DIR" -trap "rm -rf $TMP_DIR" EXIT - TRACKED_FILE="$TMP_DIR/tracked_paths.txt" > "$TRACKED_FILE" diff --git a/Plugin/dev_install.sh b/Plugin/dev_install.sh index fe09c46..c540878 100755 --- a/Plugin/dev_install.sh +++ b/Plugin/dev_install.sh @@ -1,30 +1,71 @@ #!/bin/bash -# dev_install.sh — symlinks plugin files into unRAID WebUI for local development. -# Run once after cloning. Re-run if the plugin directory is moved. +# dev_install.sh — symlinks the OS-appropriate plugin into the web server for local development. +# Run once after cloning. Re-run if the repo is moved. # Safe to re-run: removes stale symlink before recreating. +# +# Plugin layout: +# Plugin/unraid/ ← Unraid plugin (PHP, served by emhttp at /usr/local/emhttp/plugins/) +# Plugin/debian/ ← future +# Plugin// ← future +# +# Adding a new OS: +# 1. Create Plugin// with the OS-appropriate web app files +# 2. Add detection + TARGET path below set -euo pipefail PLUGIN_NAME="varaverk" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -SOURCE="$SCRIPT_DIR/usr/local/emhttp/plugins/$PLUGIN_NAME" -TARGET="/usr/local/emhttp/plugins/$PLUGIN_NAME" -if [ ! -d "$SOURCE" ]; then - echo "ERROR: Source not found: $SOURCE" +# ── Detect OS ──────────────────────────────────────────────────────────────── + +detect_os() { + if [[ -f /etc/unraid-version ]]; then echo "unraid" + elif [[ -f /etc/debian_version ]]; then echo "debian" + elif [[ -f /etc/arch-release ]]; then echo "arch" + else echo "unknown"; fi +} + +OS="${1:-$(detect_os)}" # accept override: ./dev_install.sh unraid + +# ── OS-specific install target ──────────────────────────────────────────────── + +case "$OS" in + unraid) + SOURCE="$SCRIPT_DIR/unraid" + TARGET="/usr/local/emhttp/plugins/$PLUGIN_NAME" + ;; + debian|arch) + echo "OS '$OS' detected but plugin target path not yet defined." + echo "Add the TARGET= line for this OS in dev_install.sh." + exit 1 + ;; + unknown|*) + echo "Unsupported OS — could not detect from /etc/*-version markers." + echo "Usage: $0 (e.g. $0 unraid)" + exit 1 + ;; +esac + +# ── Install ─────────────────────────────────────────────────────────────────── + +if [[ ! -d "$SOURCE" ]]; then + echo "ERROR: Plugin source not found: $SOURCE" exit 1 fi -if [ -L "$TARGET" ]; then +if [[ -L "$TARGET" ]]; then echo "Removing existing symlink: $TARGET" rm "$TARGET" -elif [ -d "$TARGET" ]; then +elif [[ -d "$TARGET" ]]; then echo "ERROR: $TARGET exists as a real directory — remove it manually first." exit 1 fi ln -s "$SOURCE" "$TARGET" -echo "Linked: $TARGET -> $SOURCE" echo "" -echo "Plugin available in unRAID WebUI under Utilities → Varaverk." -echo "Changes in $SOURCE take effect immediately (no restart needed)." +echo "OS: $OS" +echo "Source: $SOURCE" +echo "Target: $TARGET" +echo "" +echo "Plugin linked. Changes in Plugin/$OS/ take effect immediately." diff --git a/Plugin/usr/local/emhttp/plugins/varaverk/Varaverk.page b/Plugin/unraid/Varaverk.page similarity index 74% rename from Plugin/usr/local/emhttp/plugins/varaverk/Varaverk.page rename to Plugin/unraid/Varaverk.page index 668395d..a339f54 100644 --- a/Plugin/usr/local/emhttp/plugins/varaverk/Varaverk.page +++ b/Plugin/unraid/Varaverk.page @@ -9,8 +9,9 @@ $pluginDir = "$docroot/plugins/$plugin"; // Determine active tab $tab = $_GET['tab'] ?? 'monitor'; -$validTabs = ['monitor', 'scheduler']; +$validTabs = ['monitor', 'scheduler', 'partnership', 'fallback', 'watchdog', 'arrs']; if (!in_array($tab, $validTabs)) $tab = 'monitor'; +$tabLabels = ['monitor' => 'Monitor', 'scheduler' => 'Scheduler', 'partnership' => 'Partnership', 'fallback' => 'FallBack', 'watchdog' => 'Watchdog', 'arrs' => 'Arrs']; ?> @@ -21,7 +22,7 @@ if (!in_array($tab, $validTabs)) $tab = 'monitor'; diff --git a/Plugin/unraid/api/arrs.php b/Plugin/unraid/api/arrs.php new file mode 100644 index 0000000..f65f3f2 --- /dev/null +++ b/Plugin/unraid/api/arrs.php @@ -0,0 +1,4 @@ + true]; + +// ── Active locks ────────────────────────────────────────────────────────── +$lockDir = '/tmp/unraid_locks'; +$locks = []; +if (is_dir($lockDir)) { + foreach (glob($lockDir . '/*.lock') ?: [] as $lf) { + $name = basename($lf, '.lock'); + $age = time() - (int)filemtime($lf); + $content = trim(file_get_contents($lf) ?: ''); + // content is "PID:scriptname" — extract PID + $pid = preg_match('/^(\d+)/', $content, $pm) ? $pm[1] : ''; + // Skip if PID is still alive (it's legitimately running) + if ($pid && file_exists("/proc/$pid")) continue; + $locks[] = ['name' => $name, 'file' => basename($lf), 'age' => $age]; + } +} +$out['locks'] = $locks; + +// ── Recent errors ────────────────────────────────────────────────────────── +// Scan recursively so subdirectory logs (Orchestrators/, Media/, etc.) are included. +// e.script = relative path without extension = job ID base, e.g. "Orchestrators/transcode_management" +$errors = []; +if (is_dir(LOG_DIR)) { + $cutoff = time() - 7 * 86400; + $logBase = rtrim(LOG_DIR, '/') . '/'; + try { + $ri = new RecursiveIteratorIterator( + new RecursiveDirectoryIterator(LOG_DIR, RecursiveDirectoryIterator::SKIP_DOTS) + ); + foreach ($ri as $lf) { + if ($lf->getExtension() !== 'log') continue; + if ($lf->getMTime() < $cutoff) continue; + $rel = ltrim(str_replace($logBase, '', $lf->getPathname()), '/'); + $script = preg_replace('/\.log$/', '', $rel); + + // Only report scripts whose last recorded run exited as warn or error. + // This eliminates dry-run false positives (exit 0 = ok) and success-run + // summaries that happen to contain words like "Failed: 0". + $statFile = $logBase . $script . '.json'; + if (file_exists($statFile)) { + $stat = json_decode(@file_get_contents($statFile), true); + $status = $stat['status'] ?? ''; + if ($status !== 'warn' && $status !== 'error') continue; + } else { + continue; // no stat file — never ran through run_job.sh, skip + } + + $lines = array_slice(@file($lf->getPathname()) ?: [], -200); + $lastErr = null; + foreach (array_reverse($lines) as $raw) { + $clean = preg_replace('/\033\[[0-9;]*[mK]/', '', rtrim($raw)); + if (!$clean) continue; + if (preg_match('/\[(?:ERROR|WARN|CRITICAL|FAILED)\]/i', $clean) || + preg_match('/\b(?:ERROR|CRITICAL):\s/i', $clean) || + str_contains($clean, '✗') || + (str_contains($clean, '⚠') && !str_contains($clean, '♥'))) { + $lastErr = mb_substr($clean, 0, 220); + break; + } + } + // Fall back to last non-blank line if no error pattern found in log + if ($lastErr === null) { + foreach (array_reverse($lines) as $raw) { + $clean = preg_replace('/\033\[[0-9;]*[mK]/', '', rtrim($raw)); + if ($clean && !str_starts_with($clean, '──')) { $lastErr = mb_substr($clean, 0, 220); break; } + } + } + if ($lastErr !== null) + $errors[] = ['script' => $script, 'line' => $lastErr, 'ts' => (int)$lf->getMTime()]; + } + } catch (Exception $e) {} + usort($errors, fn($a, $b) => $b['ts'] - $a['ts']); +} +$out['errors'] = array_slice($errors, 0, 20); + +// ── Partner reachability ─────────────────────────────────────────────────── +$cacheFile = '/tmp/vv_partner_cache.json'; +$cacheTtl = 30; +$partnerData = null; + +if (file_exists($cacheFile) && (time() - (int)filemtime($cacheFile)) < $cacheTtl) { + $partnerData = json_decode(file_get_contents($cacheFile), true); +} else { + // Discover partner hostname dynamically from master.conf (works for any number of hosts) + $vars = vv_conf_vars(); + $mine = vv_get_hostname(); + $partnerHost = null; + foreach ($vars as $k => $v) { + if (preg_match('/^HOST\d+$/', $k) && $v !== '' && strcasecmp($v, $mine) !== 0) { + $partnerHost = $v; // first non-self host is the partner + break; + } + } + + if ($partnerHost) { + // Resolve via Tailscale — mirrors common.sh resolve_tailscale_ip(). + // Uses `tailscale ip -4` so it survives IP changes; never relies on local DNS. + $partnerIp = vv_resolve_tailscale_ip($partnerHost); + $target = $partnerIp ?: $partnerHost; + $start = microtime(true); + $result = shell_exec('ping -c1 -W2 ' . escapeshellarg($target) . ' 2>&1'); + $elapsed = (int)round((microtime(true) - $start) * 1000); + $reached = str_contains((string)$result, '1 received') + || str_contains((string)$result, '1 packets received'); + $partnerData = [ + 'host' => $partnerHost, + 'reachable' => $reached, + 'latency' => $reached ? $elapsed : null, + ]; + @file_put_contents($cacheFile, json_encode($partnerData)); + } +} +$out['partner'] = $partnerData; + +echo json_encode($out); diff --git a/Plugin/usr/local/emhttp/plugins/varaverk/api/clearlock.php b/Plugin/unraid/api/clearlock.php similarity index 100% rename from Plugin/usr/local/emhttp/plugins/varaverk/api/clearlock.php rename to Plugin/unraid/api/clearlock.php diff --git a/Plugin/usr/local/emhttp/plugins/varaverk/api/conf_toggle.php b/Plugin/unraid/api/conf_toggle.php similarity index 100% rename from Plugin/usr/local/emhttp/plugins/varaverk/api/conf_toggle.php rename to Plugin/unraid/api/conf_toggle.php diff --git a/Plugin/usr/local/emhttp/plugins/varaverk/api/confform.php b/Plugin/unraid/api/confform.php similarity index 100% rename from Plugin/usr/local/emhttp/plugins/varaverk/api/confform.php rename to Plugin/unraid/api/confform.php diff --git a/Plugin/usr/local/emhttp/plugins/varaverk/api/config.php b/Plugin/unraid/api/config.php similarity index 100% rename from Plugin/usr/local/emhttp/plugins/varaverk/api/config.php rename to Plugin/unraid/api/config.php diff --git a/Plugin/usr/local/emhttp/plugins/varaverk/api/docker_action.php b/Plugin/unraid/api/docker_action.php similarity index 100% rename from Plugin/usr/local/emhttp/plugins/varaverk/api/docker_action.php rename to Plugin/unraid/api/docker_action.php diff --git a/Plugin/usr/local/emhttp/plugins/varaverk/api/dryrun.php b/Plugin/unraid/api/dryrun.php similarity index 75% rename from Plugin/usr/local/emhttp/plugins/varaverk/api/dryrun.php rename to Plugin/unraid/api/dryrun.php index 0039477..dce4766 100644 --- a/Plugin/usr/local/emhttp/plugins/varaverk/api/dryrun.php +++ b/Plugin/unraid/api/dryrun.php @@ -19,16 +19,15 @@ $logFile = vv_job_log_path($id); $logDir = dirname($logFile); if (!is_dir($logDir)) mkdir($logDir, 0755, true); -file_put_contents($logFile, "\n── " . date('Y-m-d H:i:s') . " [DRY RUN] ──────────────────────\n", FILE_APPEND); - $location = trim($_POST['location'] ?? ''); if ($location && (!str_starts_with($location, '/') || str_contains($location, '..') || preg_match('/[\x00\n\r]/', $location))) { echo json_encode(['ok' => false, 'error' => 'Invalid location']); exit; } +$runner = dirname(__DIR__) . '/run_job.sh'; $flags = vv_job_flags($id); $locArg = $location ? ' ' . escapeshellarg('--location=' . $location) : ''; -exec('nohup env DRY_RUN=1 bash ' . escapeshellarg($script) . ($flags ? " $flags" : '') . $locArg . ' >> ' . escapeshellarg($logFile) . ' 2>&1 > ' . escapeshellarg($logFile) . ' 2>&1 true]); diff --git a/Plugin/unraid/api/fallback.php b/Plugin/unraid/api/fallback.php new file mode 100644 index 0000000..aeeea10 --- /dev/null +++ b/Plugin/unraid/api/fallback.php @@ -0,0 +1,4 @@ +getExtension() !== 'json') continue; + $d = @json_decode(@file_get_contents($file->getPathname()), true); + if (!is_array($d) || empty($d['start']) || empty($d['status'])) continue; + if ($d['status'] === 'running') continue; + $id = (string)($d['id'] ?? ''); + $runs[] = [ + 'id' => $id, + 'label' => basename(str_replace('.sh', '', $id)), + 'status' => $d['status'], + 'start' => (int)$d['start'], + 'dur' => isset($d['end']) ? max(0, (int)$d['end'] - (int)$d['start']) : 0, + ]; + } +} catch (Exception $e) {} + +usort($runs, fn($a, $b) => $b['start'] - $a['start']); +echo json_encode(['ok' => true, 'runs' => array_slice($runs, 0, 24)]); diff --git a/Plugin/usr/local/emhttp/plugins/varaverk/api/reorderarray.php b/Plugin/unraid/api/reorderarray.php similarity index 100% rename from Plugin/usr/local/emhttp/plugins/varaverk/api/reorderarray.php rename to Plugin/unraid/api/reorderarray.php diff --git a/Plugin/usr/local/emhttp/plugins/varaverk/api/rsync_standalone.php b/Plugin/unraid/api/rsync_standalone.php similarity index 100% rename from Plugin/usr/local/emhttp/plugins/varaverk/api/rsync_standalone.php rename to Plugin/unraid/api/rsync_standalone.php diff --git a/Plugin/usr/local/emhttp/plugins/varaverk/api/run.php b/Plugin/unraid/api/run.php similarity index 76% rename from Plugin/usr/local/emhttp/plugins/varaverk/api/run.php rename to Plugin/unraid/api/run.php index faec6c0..f56e573 100644 --- a/Plugin/usr/local/emhttp/plugins/varaverk/api/run.php +++ b/Plugin/unraid/api/run.php @@ -19,16 +19,15 @@ $logFile = vv_job_log_path($id); $logDir = dirname($logFile); if (!is_dir($logDir)) mkdir($logDir, 0755, true); -file_put_contents($logFile, "\n── " . date('Y-m-d H:i:s') . " ──────────────────────\n", FILE_APPEND); - $location = trim($_POST['location'] ?? ''); if ($location && (!str_starts_with($location, '/') || str_contains($location, '..') || preg_match('/[\x00\n\r]/', $location))) { echo json_encode(['ok' => false, 'error' => 'Invalid location']); exit; } +$runner = dirname(__DIR__) . '/run_job.sh'; $flags = vv_job_flags($id); $locArg = $location ? ' ' . escapeshellarg('--location=' . $location) : ''; -exec('nohup bash ' . escapeshellarg($script) . ($flags ? " $flags" : '') . $locArg . ' >> ' . escapeshellarg($logFile) . ' 2>&1 > ' . escapeshellarg($logFile) . ' 2>&1 true]); diff --git a/Plugin/usr/local/emhttp/plugins/varaverk/api/savefolders.php b/Plugin/unraid/api/savefolders.php similarity index 100% rename from Plugin/usr/local/emhttp/plugins/varaverk/api/savefolders.php rename to Plugin/unraid/api/savefolders.php diff --git a/Plugin/usr/local/emhttp/plugins/varaverk/api/scheduler.php b/Plugin/unraid/api/scheduler.php similarity index 91% rename from Plugin/usr/local/emhttp/plugins/varaverk/api/scheduler.php rename to Plugin/unraid/api/scheduler.php index d6bb8d8..18f561a 100644 --- a/Plugin/usr/local/emhttp/plugins/varaverk/api/scheduler.php +++ b/Plugin/unraid/api/scheduler.php @@ -10,7 +10,7 @@ if (!empty($_POST['batch'])) { $id = trim($e['id'] ?? ''); $cron = trim($e['cron'] ?? ''); if (!$id) continue; - if ($cron && !in_array($cron, ['@array_start', '@array_stop'], true) + if ($cron && !in_array($cron, ['array_start', 'array_stop'], true) && !preg_match('/^(\S+\s+){4}\S+$/', $cron)) $cron = ''; $clean[] = [ 'id' => $id, @@ -35,7 +35,7 @@ if (!$id) { } // Basic cron validation — 5 fields, or known @event trigger, or empty -if ($cron && !in_array($cron, ['@array_start', '@array_stop'], true) +if ($cron && !in_array($cron, ['array_start', 'array_stop'], true) && !preg_match('/^(\S+\s+){4}\S+$/', $cron)) { echo json_encode(['ok' => false, 'error' => 'Invalid cron expression']); exit; diff --git a/Plugin/usr/local/emhttp/plugins/varaverk/api/script.php b/Plugin/unraid/api/script.php similarity index 100% rename from Plugin/usr/local/emhttp/plugins/varaverk/api/script.php rename to Plugin/unraid/api/script.php diff --git a/Plugin/unraid/api/scriptinfo.php b/Plugin/unraid/api/scriptinfo.php new file mode 100644 index 0000000..dfb0df2 --- /dev/null +++ b/Plugin/unraid/api/scriptinfo.php @@ -0,0 +1,54 @@ + false, 'error' => 'Invalid id']); + exit; +} + +$isScript = str_ends_with($id, '.sh'); +$name = basename($id, $isScript ? '.sh' : ''); +$dirName = basename(dirname($id)); // e.g. "Media", "Rsync", "Orchestrators" +$slug = strtolower(str_replace(['_', '-'], ' ', $name)); +$parts = explode(' ', $slug); +$first = $parts[0] ?? ''; // e.g. "radarr" from "radarr cleanup" + +// Script header (bash scripts only) +$path = SCRIPTS_DIR . '/' . $id; +$header = ($isScript && file_exists($path)) ? vv_script_header_clean($path) : ''; + +// Section matcher: heading contains the full slug OR first meaningful word (>3 chars) +$matcher = function(string $heading, bool $isIntro) use ($slug, $first): bool { + if ($isIntro) return false; + $h = strtolower(str_replace(['_', '-'], ' ', $heading)); + return str_contains($h, $slug) + || (strlen($first) > 3 && str_contains($h, $first)); +}; + +// Search main README/Manual + module-level files for this script's directory +$scriptsDir = SCRIPTS_DIR; +$searchFiles = []; +foreach (['README', 'Manual'] as $docType) { + $main = "$scriptsDir/$docType.md"; + if (file_exists($main)) $searchFiles[] = [$docType, $main]; + + $mod = "$scriptsDir/$docType-$dirName.md"; + if ($dirName && $dirName !== '.' && file_exists($mod) && $mod !== $main) { + $searchFiles[] = ["$docType — $dirName", $mod]; + } +} + +$sections = []; +foreach ($searchFiles as [$label, $file]) { + $body = vv_readme_section($file, $matcher); + if ($body) $sections[] = ['source' => $label, 'body' => $body]; +} + +echo json_encode([ + 'ok' => true, + 'name' => $name, + 'header' => $header, + 'sections' => $sections, +]); diff --git a/Plugin/usr/local/emhttp/plugins/varaverk/api/settings.php b/Plugin/unraid/api/settings.php similarity index 100% rename from Plugin/usr/local/emhttp/plugins/varaverk/api/settings.php rename to Plugin/unraid/api/settings.php diff --git a/Plugin/usr/local/emhttp/plugins/varaverk/api/snapshot.php b/Plugin/unraid/api/snapshot.php similarity index 100% rename from Plugin/usr/local/emhttp/plugins/varaverk/api/snapshot.php rename to Plugin/unraid/api/snapshot.php diff --git a/Plugin/usr/local/emhttp/plugins/varaverk/api/status.php b/Plugin/unraid/api/status.php similarity index 100% rename from Plugin/usr/local/emhttp/plugins/varaverk/api/status.php rename to Plugin/unraid/api/status.php diff --git a/Plugin/usr/local/emhttp/plugins/varaverk/api/stop.php b/Plugin/unraid/api/stop.php similarity index 100% rename from Plugin/usr/local/emhttp/plugins/varaverk/api/stop.php rename to Plugin/unraid/api/stop.php diff --git a/Plugin/usr/local/emhttp/plugins/varaverk/api/system.php b/Plugin/unraid/api/system.php similarity index 100% rename from Plugin/usr/local/emhttp/plugins/varaverk/api/system.php rename to Plugin/unraid/api/system.php diff --git a/Plugin/unraid/api/watchdog.php b/Plugin/unraid/api/watchdog.php new file mode 100644 index 0000000..9fcc124 --- /dev/null +++ b/Plugin/unraid/api/watchdog.php @@ -0,0 +1,4 @@ + div { flex-wrap: wrap; gap: 6px !important; } + /* Array-event rows: badge already shows the event, cron input is redundant */ + .vv-job-row:has(.vv-event-badge) .vv-cron { display: none; } } /* Plugin settings row (Advanced mode) */ @@ -134,20 +135,25 @@ } /* Phone layout — scheduler row/actions fixes + monitor single-column */ +@media (max-width: 480px) { + /* Phone portrait — hint gone, cron stays inline but compact */ + .vv-cron-hint { display: none !important; } + .vv-job-row .vv-cron { flex: 0 0 68px; width: 68px; font-size: 10px; margin-left: 30px; } +} + @media (max-width: 600px) { /* Allow action buttons to wrap rather than overflow the card */ .vv-job-actions { flex-wrap: wrap; padding-left: 0; } /* Kill the inline margin-left:auto that pushes Advanced off-screen */ .vv-advanced-toggle { margin-left: 0 !important; width: auto !important; } - /* Let the cron input flex rather than hold a fixed 110px */ - .vv-cron { flex: 1 1 80px; width: auto; min-width: 80px; } + /* Keep cron from growing — label owns remaining row space */ + .vv-cron { flex: 0 0 75px; width: 75px; } /* Slightly tighter label on narrow screens */ .vv-job-label { font-size: 14px; } /* Footer buttons wrap instead of overflowing */ .vv-sched-footer { flex-wrap: wrap; } - /* Log toolbar wraps so Stop/Clear stay inside the card */ - .vv-log-toolbar { flex-wrap: wrap; align-items: flex-start; } - .vv-log-toolbar > div { flex-wrap: wrap; } + /* Log toolbar search — narrow on small screens */ + #vv-log-search { width: 80px; } /* Snapshot footer — smaller on mobile */ .vv-snap-footer { gap: 10px !important; } .vv-snap-item { gap: 4px; } @@ -229,8 +235,16 @@ border: 1px solid #333; border-radius: 4px; padding: 10px 12px; resize: none; line-height: 1.5; scroll-behavior: auto; tab-size: 2; width: 100%; box-sizing: border-box; } -/* Syntax-highlighted conf editor overlay */ -#vv-editor-wrap { position: relative; display: block; } +/* Editor layout: gutter + inner area */ +#vv-editor-wrap { display: flex; } +#vv-ln-gutter { width: 42px; min-width: 42px; flex-shrink: 0; + background: #0a0a0a; border-right: 1px solid #1c1c1c; + font-family: monospace; font-size: 12px; line-height: 1.5; tab-size: 2; + color: #3a3a3a; text-align: right; padding: 10px 8px 10px 0; + overflow: hidden; user-select: none; white-space: pre; } +#vv-editor-inner { position: relative; flex: 1; overflow: hidden; } + +/* Syntax-highlight overlay sits over the transparent textarea */ #vv-hl-overlay { display: none; position: absolute; top: 1px; left: 1px; right: 1px; bottom: 1px; margin: 0; padding: 10px 12px; box-sizing: border-box; font-family: monospace; font-size: 12px; line-height: 1.5; tab-size: 2; @@ -271,6 +285,10 @@ .vv-info-body { padding: 2px 10px 10px; } .vv-info-cols { margin: 0; padding-left: 16px; columns: 2; column-gap: 20px; column-fill: balance; } .vv-info-cols li { font-size: 12px; color: #aaa; margin-bottom: 5px; break-inside: avoid; line-height: 1.5; } +.vv-info-cols .vv-info-sep { column-span: all; list-style: none; margin: 10px -4px 6px; + padding: 4px 8px; background: #161a1d; border-left: 2px solid #1e6fa5; + font-size: 10px; font-weight: bold; text-transform: uppercase; + letter-spacing: .08em; color: #4a8ab5; break-inside: avoid; } .vv-info-cols li strong { color: #ccc; } .vv-info-cols code { background: #1a1a1a; padding: 0 4px; border-radius: 2px; font-size: 11px; color: #9ab; border: 1px solid #333; } @@ -286,6 +304,58 @@ border-bottom: 1px solid #333; } +/* Run-status dot colours (tree + activity list) */ +.vv-stat-ok { color: #4caf50; } +.vv-stat-warn { color: #ff9800; } +.vv-stat-error{ color: #f44336; } +.vv-stat-skip { color: #607d8b; } +.vv-stat-none { color: #444; } + +/* Cron humanizer hint — left of cron input; first to collapse under space pressure */ +.vv-cron-hint { font-size: 10px; color: #505050; white-space: nowrap; + flex: 0 10 auto; min-width: 0; max-width: 130px; + overflow: hidden; text-overflow: ellipsis; + pointer-events: none; user-select: none; } + +/* Recent Activity list */ +.vv-activity-list { display: flex; flex-direction: column; } +.vv-activity-row { display: flex; align-items: center; gap: 6px; padding: 4px 0; + border-bottom: 1px solid #1a1a1a; font-size: 11px; cursor: pointer; } +.vv-activity-row:last-child { border-bottom: none; } +.vv-activity-row:hover .vv-activity-label { color: #fff; } +.vv-activity-dot { flex-shrink: 0; font-size: 9px; } +.vv-activity-label { flex: 1; min-width: 0; overflow: hidden; text-overflow: ellipsis; + white-space: nowrap; color: #ccc; } +.vv-activity-ago { color: #555; font-size: 10px; white-space: nowrap; flex-shrink: 0; } +.vv-activity-dur { color: #444; font-size: 10px; white-space: nowrap; flex-shrink: 0; + min-width: 32px; text-align: right; } + +/* Log search */ +#vv-log-search { width: 120px; font-size: 11px; padding: 2px 6px; + background: #111; border: 1px solid #333; color: #ccc; + border-radius: 3px; font-family: monospace; } +#vv-log-search:focus { border-color: #555; outline: none; } +mark { background: #5d4037; color: #ffcc80; border-radius: 2px; } +.vv-log-dim { opacity: 0.2; } + +/* Cron Calculator */ +.vv-calc-wrap { display: flex; flex-direction: column; gap: 8px; } +.vv-calc-in { width: 100%; background: #111; border: 1px solid #333; color: #ccc; + padding: 5px 8px; border-radius: 3px; font-family: monospace; font-size: 12px; + box-sizing: border-box; } +.vv-calc-in:focus { border-color: #555; outline: none; } +.vv-calc-expr { font-family: monospace; font-size: 13px; color: #7cb8e8; + background: #0d1117; padding: 4px 8px; border-radius: 3px; } +.vv-calc-desc { font-size: 12px; color: #aaa; padding: 2px 2px 0; } +.vv-calc-hint { font-size: 11px; color: #555; font-style: italic; } +.vv-calc-runs-lbl { font-size: 10px; color: #555; text-transform: uppercase; + letter-spacing: .05em; margin-top: 4px; } +.vv-calc-runs { display: flex; flex-direction: column; gap: 2px; } +.vv-calc-run-row { display: flex; gap: 8px; font-size: 11px; } +.vv-calc-run-in { color: #4caf50; white-space: nowrap; min-width: 56px; } +.vv-calc-run-at { color: #666; } +.vv-calc-apply-btn { align-self: flex-start; margin-top: 2px; } + /* Board blocks — Next Runs, Errors, Locks, Partner, Disabled */ .vv-board-placeholder { color: #555; font-size: 11px; padding: 3px 0; } @@ -311,7 +381,10 @@ .vv-err-row:last-child { border-bottom: none; } .vv-err-top { display: flex; align-items: baseline; gap: 8px; margin-bottom: 2px; } .vv-err-script { color: #e07070; font-size: 11px; font-weight: bold; } -.vv-err-age { color: #555; font-size: 10px; } +.vv-err-age { color: #555; font-size: 10px; flex: 1; padding-left: 6px; white-space: nowrap; } +.vv-ack-btn { margin-left: auto; font-size: 10px; padding: 1px 6px; + color: #555; border-color: #333; flex-shrink: 0; } +.vv-ack-btn:hover { color: #aaa; border-color: #555; } .vv-err-line { color: #888; font-size: 11px; word-break: break-all; line-height: 1.4; } /* Locks */ @@ -404,6 +477,17 @@ word-break: break-word; background: none; border: none; margin: 0; padding: 8px 12px; line-height: 1.5; } +/* Cog settings button next to script name */ +.vv-cog-btn { cursor:pointer; color:#444; font-size:.85em; margin-right:4px; + line-height:1; vertical-align:middle; user-select:none; } +.vv-cog-btn:hover { color:#aaa; } + +/* Enriched script info blocks (advanced mode: header + docs + config) */ +.vv-sinfo-block { border-top: 1px solid #222; padding: 0; } +.vv-sinfo-block:first-child { border-top: none; } +.vv-sinfo-lbl { font-size: 10px; font-weight: bold; text-transform: uppercase; + letter-spacing: .06em; color: #555; padding: 6px 12px 2px; } + /* Syntax highlight tokens — dark theme (VSCode-inspired) */ .vv-hl-sep { color: #2d2d2d; } .vv-hl-shebang { color: #4a4a4a; } @@ -424,7 +508,7 @@ /* Log panel */ .vv-log-panel { margin-top: 10px; border-top: 1px solid #333; padding-top: 8px; } .vv-log-toolbar { display: flex; justify-content: space-between; align-items: center; - margin-bottom: 4px; } + flex-wrap: wrap; gap: 4px; margin-bottom: 4px; } .vv-log-ts { font-size: 11px; color: #666; } .vv-log-pre { background: #0d0d0d; border: 1px solid #333; border-radius: 4px; padding: 10px 12px; margin: 0; font-family: monospace; font-size: 12px; diff --git a/Plugin/unraid/event/disks_mounted/array_start_jobs b/Plugin/unraid/event/disks_mounted/array_start_jobs new file mode 100755 index 0000000..2b0107b --- /dev/null +++ b/Plugin/unraid/event/disks_mounted/array_start_jobs @@ -0,0 +1,16 @@ +#!/bin/bash +# Varaverk: fire all jobs with cron=array_start (background — non-blocking). +php -r " +require_once '/usr/local/emhttp/plugins/varaverk/include/scheduler.php'; +\$s = vv_schedule_load(); +\$runner = '/usr/local/emhttp/plugins/varaverk/run_job.sh'; +foreach (\$s as \$id => \$e) { + if (strncmp(\$id, '__', 2) === 0) continue; + if ((\$e['cron'] ?? '') !== 'array_start') continue; + if (empty(\$e['enabled'])) continue; + \$script = SCRIPTS_DIR . '/' . \$id; + if (!file_exists(\$script)) continue; + \$flags = !empty(\$e['log_enabled']) ? ' --log' : ''; + exec('nohup bash ' . escapeshellarg(\$runner) . ' ' . escapeshellarg(\$id) . ' ' . escapeshellarg(\$script) . \$flags . ' > /dev/null 2>&1 &'); +} +" 2>/dev/null diff --git a/Plugin/usr/local/emhttp/plugins/varaverk/event/disks_mounted/rebuild_cron b/Plugin/unraid/event/disks_mounted/rebuild_cron similarity index 100% rename from Plugin/usr/local/emhttp/plugins/varaverk/event/disks_mounted/rebuild_cron rename to Plugin/unraid/event/disks_mounted/rebuild_cron diff --git a/Plugin/unraid/event/disks_unmounting/array_stop_jobs b/Plugin/unraid/event/disks_unmounting/array_stop_jobs new file mode 100755 index 0000000..b92ac04 --- /dev/null +++ b/Plugin/unraid/event/disks_unmounting/array_stop_jobs @@ -0,0 +1,16 @@ +#!/bin/bash +# Varaverk: fire all jobs with cron=array_stop (foreground — blocks until all done). +php -r " +require_once '/usr/local/emhttp/plugins/varaverk/include/scheduler.php'; +\$s = vv_schedule_load(); +\$runner = '/usr/local/emhttp/plugins/varaverk/run_job.sh'; +foreach (\$s as \$id => \$e) { + if (strncmp(\$id, '__', 2) === 0) continue; + if ((\$e['cron'] ?? '') !== 'array_stop') continue; + if (empty(\$e['enabled'])) continue; + \$script = SCRIPTS_DIR . '/' . \$id; + if (!file_exists(\$script)) continue; + \$flags = !empty(\$e['log_enabled']) ? ' --log' : ''; + passthru('bash ' . escapeshellarg(\$runner) . ' ' . escapeshellarg(\$id) . ' ' . escapeshellarg(\$script) . \$flags); +} +" 2>/dev/null diff --git a/Plugin/usr/local/emhttp/plugins/varaverk/icons/varaverk.png b/Plugin/unraid/icons/varaverk.png similarity index 100% rename from Plugin/usr/local/emhttp/plugins/varaverk/icons/varaverk.png rename to Plugin/unraid/icons/varaverk.png diff --git a/Plugin/unraid/include/arrs.php b/Plugin/unraid/include/arrs.php new file mode 100644 index 0000000..9cff157 --- /dev/null +++ b/Plugin/unraid/include/arrs.php @@ -0,0 +1,278 @@ +'hostname', ...] +// Returns ['host1' => 'hostname', 'host2' => 'hostname', ...] from master.conf. +// Matches both HOST1="name" and HOST1_NAME="name" (either convention). +function vv_arr_known_hosts(): array { + $vars = vv_conf_vars(); // already parses HOST1="val" correctly + $hosts = []; + foreach ($vars as $k => $v) { + if (preg_match('/^HOST(\d+)$/', $k, $m) && $v !== '') { + $hosts['host' . $m[1]] = $v; + } + } + ksort($hosts); + return $hosts ?: ['host1' => 'HOST1']; +} + +function vv_arr_node_names(): array { + return array_map(fn($name) => $name, vv_arr_known_hosts()); +} + +// ── Discovery ───────────────────────────────────────────────────────────────── + +function vv_discover_arrs(): array { + $nodes = []; + $defs = [ + 'sonarr' => ['SONARR_URL', 'SONARR_API_KEY', 'SONARR_TV_ROOT', 'v3'], + 'radarr' => ['RADARR_URL', 'RADARR_API_KEY', 'RADARR_MOVIES_ROOT', 'v3'], + 'lidarr' => ['LIDARR_URL', 'LIDARR_API_KEY', 'LIDARR_MUSIC_ROOT', 'v1'], + ]; + foreach (array_keys(vv_arr_known_hosts()) as $h) { + $raw = vv_read_conf_raw($h . '.conf'); + if (!$raw) continue; + $pfx = strtoupper($h) . '_'; + $get = fn($k) => vv_arr_scalar($raw, $pfx . $k); + $arrs = []; + foreach ($defs as $type => [$uk, $ak, $rk, $api]) { + $url = $get($uk); + $key = $get($ak); + if ($url && $key && !str_contains($key, 'your-')) { + $arrs[] = ['type' => $type, 'url' => $url, 'key' => $key, + 'root' => $get($rk), 'api' => $api]; + } + } + if ($arrs) $nodes[] = ['host' => $h, 'arrs' => $arrs]; + } + return $nodes; +} + +// ── HTTP ────────────────────────────────────────────────────────────────────── + +function vv_arr_http(string $url, string $apiKey, string $path, int $timeout = 4): ?array { + $ctx = stream_context_create(['http' => [ + 'timeout' => $timeout, + 'header' => "X-Api-Key: $apiKey\r\nAccept: application/json\r\n", + 'ignore_errors' => true, + ]]); + $raw = @file_get_contents(rtrim($url, '/') . $path, false, $ctx); + return $raw ? (json_decode($raw, true) ?: null) : null; +} + +// ── Live arr data ───────────────────────────────────────────────────────────── + +function vv_fetch_arr_live(array $arr): array { + $url = $arr['url']; + $key = $arr['key']; + $base = '/api/' . $arr['api']; + $type = $arr['type']; + + $out = ['online' => false, 'version' => null, 'health' => [], + 'queue' => ['dl' => 0, 'warn' => 0, 'err' => 0], 'disk' => []]; + + $sys = vv_arr_http($url, $key, "$base/system/status"); + if (!$sys) return $out; + $out['online'] = true; + $out['version'] = $sys['version'] ?? null; + + if ($type === 'sonarr') { + $data = vv_arr_http($url, $key, "$base/series"); + if (is_array($data)) { + $out['total'] = count($data); + $out['monitored'] = count(array_filter($data, fn($x) => !empty($x['monitored']))); + $out['episodes'] = array_sum(array_column($data, 'episodeFileCount')); + } + } elseif ($type === 'radarr') { + $data = vv_arr_http($url, $key, "$base/movie"); + if (is_array($data)) { + $out['total'] = count($data); + $out['monitored'] = count(array_filter($data, fn($x) => !empty($x['monitored']))); + $out['files'] = count(array_filter($data, fn($x) => !empty($x['hasFile']))); + } + } elseif ($type === 'lidarr') { + $data = vv_arr_http($url, $key, "$base/artist"); + if (is_array($data)) { + $out['total'] = count($data); + $out['monitored'] = count(array_filter($data, fn($x) => !empty($x['monitored']))); + $out['albums'] = array_sum(array_map( + fn($a) => $a['statistics']['albumCount'] ?? $a['albumCount'] ?? 0, $data)); + } + } + + $q = vv_arr_http($url, $key, "$base/queue?page=1&pageSize=500"); + if (is_array($q)) { + foreach (($q['records'] ?? $q) as $r) { + if (!is_array($r)) continue; + $s = $r['status'] ?? ''; + $tds = strtolower($r['trackedDownloadStatus'] ?? ''); + $tst = strtolower($r['trackedDownloadState'] ?? ''); + if ($s === 'downloading') $out['queue']['dl']++; + if ($tds === 'warning' || $tst === 'downloadingstalled') $out['queue']['warn']++; + if ($tds === 'error') $out['queue']['err']++; + } + } + + $h = vv_arr_http($url, $key, "$base/health"); + if (is_array($h)) $out['health'] = $h; + + $d = vv_arr_http($url, $key, "$base/diskspace"); + if (is_array($d)) $out['disk'] = $d; + + return $out; +} + +// ── Log stats ───────────────────────────────────────────────────────────────── + +function vv_arr_cleanup_stats(string $type): array { + $slugs = ['sonarr' => 'Media/sonarr_cleanup', + 'radarr' => 'Media/radarr_cleanup', + 'lidarr' => 'Media/lidarr_cleanup']; + $base = LOG_DIR . '/' . ($slugs[$type] ?? ''); + $out = ['last_run' => null, 'end' => null, 'status' => null, + 'tracked' => null, 'total' => null, + 'orphans' => 0, 'orphans_sz' => '0B', 'junk' => 0]; + + $jf = $base . '.json'; + if (!file_exists($jf)) return $out; + $meta = json_decode(file_get_contents($jf), true) ?: []; + $out['last_run'] = $meta['start'] ?? null; + $out['end'] = $meta['end'] ?? null; + $out['status'] = $meta['status'] ?? null; + + $lf = $base . '.log'; + if (!file_exists($lf)) return $out; + $log = file_get_contents($lf); + $parts = preg_split('/━{3,}[^\n]*SUMMARY[^\n]*/u', $log); + $blk = count($parts) > 1 ? end($parts) : $log; + + if (preg_match('/Tracked:\s*([\d,]+)\s*files\s*\(([\d,]+)/u', $blk, $m)) { + $out['tracked'] = (int)str_replace(',', '', $m[1]); + $out['total'] = (int)str_replace(',', '', $m[2]); + } + if (preg_match('/Orphans:\s*([\d,]+)\s*files\s*\(([^)]+)\)/u', $blk, $m)) { + $out['orphans'] = (int)str_replace(',', '', $m[1]); + $out['orphans_sz'] = trim($m[2]); + } + if (preg_match('/Junk:\s*([\d,]+)\s*files/u', $blk, $m)) { + $out['junk'] = (int)str_replace(',', '', $m[1]); + } + return $out; +} + +function vv_arr_discovery_stats(string $type): array { + $slugs = ['sonarr' => 'Media/playback_aware_sonarr_discovery', + 'radarr' => 'Media/playback_aware_radarr_discovery', + 'lidarr' => 'Media/playback_aware_lidarr_discovery']; + $base = LOG_DIR . '/' . ($slugs[$type] ?? ''); + $out = ['last_run' => null, 'status' => null, 'added' => null]; + + $jf = $base . '.json'; + if (!file_exists($jf)) return $out; + $meta = json_decode(file_get_contents($jf), true) ?: []; + $out['last_run'] = $meta['start'] ?? null; + $out['status'] = $meta['status'] ?? null; + + $lf = $base . '.log'; + if (file_exists($lf)) { + $log = file_get_contents($lf); + if (preg_match('/Added[:\s]+(\d+)/i', $log, $m)) $out['added'] = (int)$m[1]; + elseif (preg_match('/(\d+)\s+added/i', $log, $m)) $out['added'] = (int)$m[1]; + } + return $out; +} + +function vv_arr_sync_stats(): array { + $base = LOG_DIR . '/Media/arr_sync'; + $out = ['last_run' => null, 'status' => null, 'added' => null, + 'nodes' => null, 'blocklist_count' => null]; + + $jf = $base . '.json'; + if (file_exists($jf)) { + $meta = json_decode(file_get_contents($jf), true) ?: []; + $out['last_run'] = $meta['start'] ?? null; + $out['status'] = $meta['status'] ?? null; + } + + $master = vv_read_conf_raw('master.conf'); + if (preg_match('/ARR_SYNC_BLOCKLIST\s*=\s*"?([^"\n#]+)"?/m', $master, $m)) { + $blPath = trim($m[1]); + if (file_exists($blPath)) { + $out['blocklist_count'] = count(array_filter( + file($blPath, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES))); + } + } + + $lf = $base . '.log'; + if (file_exists($lf)) { + $log = file_get_contents($lf); + if (preg_match('/Total added[:\s]+(\d+)/i', $log, $m)) $out['added'] = (int)$m[1]; + if (preg_match('/Nodes?[:\s]+(\d+)/i', $log, $m)) $out['nodes'] = (int)$m[1]; + } + return $out; +} + +function vv_arr_recovery_stats(): array { + $base = LOG_DIR . '/Media/arrs_failed_stalled_recovery'; + $out = ['last_run' => null, 'status' => null, 'fixed' => 0, 'searched' => 0]; + + $jf = $base . '.json'; + if (!file_exists($jf)) return $out; + $meta = json_decode(file_get_contents($jf), true) ?: []; + $out['last_run'] = $meta['start'] ?? null; + $out['status'] = $meta['status'] ?? null; + + $lf = $base . '.log'; + if (file_exists($lf)) { + $log = file_get_contents($lf); + if (preg_match('/Removed[:\s]+(\d+)/i', $log, $m)) $out['fixed'] = (int)$m[1]; + if (preg_match('/Re-searched[:\s]+(\d+)/i',$log, $m)) $out['searched'] = (int)$m[1]; + } + return $out; +} + +// ── Entry point ─────────────────────────────────────────────────────────────── + +function vv_arrs_all(): array { + $currentHost = vv_detect_host(); + $names = vv_arr_node_names(); + $allNodes = vv_discover_arrs(); + $result = []; + + foreach ($allNodes as $node) { + $h = $node['host']; + $isLocal = ($h === $currentHost || $currentHost === 'unknown'); + $nodeOut = ['host' => $h, 'name' => $names[$h] ?? strtoupper($h), 'local' => $isLocal, 'arrs' => []]; + + foreach ($node['arrs'] as $arr) { + $entry = ['type' => $arr['type'], 'root' => $arr['root']]; + if ($isLocal) { + $entry = array_merge($entry, vv_fetch_arr_live($arr)); + $entry['cleanup'] = vv_arr_cleanup_stats($arr['type']); + $entry['discovery'] = vv_arr_discovery_stats($arr['type']); + } else { + $entry['online'] = false; + $entry['remote'] = true; + } + $nodeOut['arrs'][] = $entry; + } + $result[] = $nodeOut; + } + + return [ + 'nodes' => $result, + 'sync' => vv_arr_sync_stats(), + 'recovery' => vv_arr_recovery_stats(), + 'host' => $currentHost, + 'ts' => time(), + ]; +} diff --git a/Plugin/usr/local/emhttp/plugins/varaverk/include/confform.php b/Plugin/unraid/include/confform.php similarity index 100% rename from Plugin/usr/local/emhttp/plugins/varaverk/include/confform.php rename to Plugin/unraid/include/confform.php diff --git a/Plugin/usr/local/emhttp/plugins/varaverk/include/config.php b/Plugin/unraid/include/config.php similarity index 60% rename from Plugin/usr/local/emhttp/plugins/varaverk/include/config.php rename to Plugin/unraid/include/config.php index 35f4ac8..6a3b2d7 100644 --- a/Plugin/usr/local/emhttp/plugins/varaverk/include/config.php +++ b/Plugin/unraid/include/config.php @@ -14,14 +14,29 @@ function vv_get_hostname(): string { return trim(shell_exec('hostname -s') ?: ''); } +// Mirror of common.sh resolve_tailscale_ip(): tries `tailscale ip -4` first (Tailscale manages +// the mapping so this survives IP changes), falls back to parsing `tailscale status` text. +function vv_resolve_tailscale_ip(string $hostname): string { + $h = strtolower($hostname); + $ip = trim(shell_exec('tailscale ip -4 ' . escapeshellarg($h) . ' 2>/dev/null') ?: ''); + if ($ip) return $ip; + $out = shell_exec('tailscale status 2>/dev/null') ?: ''; + foreach (explode("\n", $out) as $line) { + $cols = preg_split('/\s+/', trim($line)); + if (isset($cols[1]) && stripos($cols[1], $h . '.') === 0) return $cols[0]; + } + return ''; +} + function vv_detect_host(): string { - // Reads master.conf to find HOST1/HOST2 hostnames, returns 'host1', 'host2', or 'unknown' + // Reads master.conf for HOST1="name" (or HOST1_NAME="name") and matches running hostname. + // Returns 'host1', 'host2', 'host3', ... or 'unknown'. Works for any number of hosts. $master = vv_read_conf_raw('master.conf'); - preg_match('/^\s*HOST1_NAME\s*=\s*["\']?(\S+?)["\']?\s*$/m', $master, $m1); - preg_match('/^\s*HOST2_NAME\s*=\s*["\']?(\S+?)["\']?\s*$/m', $master, $m2); + preg_match_all('/^\s*(HOST\d+)(?:_NAME)?\s*=\s*["\']?(\S+?)["\']?\s*$/m', $master, $m); $hostname = vv_get_hostname(); - if (!empty($m1[1]) && strcasecmp($hostname, $m1[1]) === 0) return 'host1'; - if (!empty($m2[1]) && strcasecmp($hostname, $m2[1]) === 0) return 'host2'; + foreach ($m[1] as $i => $key) { + if (strcasecmp($hostname, trim($m[2][$i])) === 0) return strtolower($key); + } return 'unknown'; } @@ -44,10 +59,12 @@ function vv_get_conf_files(): array { $host = vv_detect_host(); $files = []; if ($host === 'host1') { + // Owner sees master.conf + their own host conf $files[] = 'master.conf'; $files[] = 'host1.conf'; - } elseif ($host === 'host2') { - $files[] = 'host2.conf'; + } elseif (preg_match('/^host(\d+)$/', $host)) { + // Any other numbered host sees only their own conf + $files[] = $host . '.conf'; } else { // Unknown host — show all for dev/debug foreach (glob(CONF_DIR . '/*.conf') as $f) { @@ -62,8 +79,7 @@ function vv_conf_vars(): array { $vars = []; $files = ['master.conf']; $host = vv_detect_host(); - if ($host === 'host1') $files[] = 'host1.conf'; - if ($host === 'host2') $files[] = 'host2.conf'; + if (preg_match('/^host\d+$/', $host)) $files[] = $host . '.conf'; foreach ($files as $f) { $raw = vv_read_conf_raw($f); diff --git a/Plugin/usr/local/emhttp/plugins/varaverk/include/docker_folders.php b/Plugin/unraid/include/docker_folders.php similarity index 100% rename from Plugin/usr/local/emhttp/plugins/varaverk/include/docker_folders.php rename to Plugin/unraid/include/docker_folders.php diff --git a/Plugin/usr/local/emhttp/plugins/varaverk/include/docs.php b/Plugin/unraid/include/docs.php similarity index 100% rename from Plugin/usr/local/emhttp/plugins/varaverk/include/docs.php rename to Plugin/unraid/include/docs.php diff --git a/Plugin/unraid/include/fallback.php b/Plugin/unraid/include/fallback.php new file mode 100644 index 0000000..9cb407a --- /dev/null +++ b/Plugin/unraid/include/fallback.php @@ -0,0 +1,184 @@ + 'UNKNOWN', + 'fallback_start' => 0, + 'handback_strikes' => 0, + 'tier2_started' => false, + 'tier3_started' => false, + 'tier4_started' => false, + ]; + foreach (explode("\n", $text) as $line) { + $line = trim($line); + if (!$line || !str_contains($line, '=')) continue; + [$k, $v] = array_pad(explode('=', $line, 2), 2, ''); + $k = trim($k); $v = trim($v, '"\''); + switch ($k) { + case 'state': $out['state'] = $v; break; + case 'fallback_start': $out['fallback_start'] = (int)$v; break; + case 'handback_strikes': $out['handback_strikes'] = (int)$v; break; + case 'tier2_started': $out['tier2_started'] = $v === 'true'; break; + case 'tier3_started': $out['tier3_started'] = $v === 'true'; break; + case 'tier4_started': $out['tier4_started'] = $v === 'true'; break; + } + } + return $out; +} + +function vv_fb_local_state(): array { + $path = '/boot/config/fallback_state.db'; + return vv_fb_parse_state(file_exists($path) ? file_get_contents($path) : ''); +} + +function vv_fb_remote_state(string $ip, string $sshKey): array { + $out = vv_pt_ssh($ip, $sshKey, 'cat /boot/config/fallback_state.db 2>/dev/null'); + return vv_fb_parse_state($out); +} + +// ── Running containers ──────────────────────────────────────────────────────── + +function vv_fb_local_running(): array { + $out = shell_exec("docker ps --format '{{.Names}}' 2>/dev/null") ?: ''; + return array_values(array_filter(explode("\n", trim($out)))); +} + +function vv_fb_remote_running(string $ip, string $sshKey): array { + $out = vv_pt_ssh($ip, $sshKey, "docker ps --format '{{.Names}}' 2>/dev/null"); + return array_values(array_filter(explode("\n", trim($out)))); +} + +// ── Covers — what a node runs for the other when it's down ─────────────────── + +function vv_fb_covers(string $covering, string $remote, string $coveringRaw, string $remoteRaw): array { + $cu = strtoupper($covering); // HOST1 + $ru = strtoupper($remote); // HOST2 + $tiers = []; + for ($t = 1; $t <= 4; $t++) { + $tiers["tier$t"] = vv_fb_bash_array($coveringRaw, "FALLBACK_{$cu}_COVERS_{$ru}_TIER{$t}"); + } + // Delays: how long the remote (covered) host must be down before each tier fires. + // Stored in the *remote* host's conf as REMOTE_TIER*_DELAY. + $tiers['delays'] = [ + 'tier2' => (int)(vv_fb_scalar($remoteRaw, "{$ru}_TIER2_DELAY") ?: 240), + 'tier3' => (int)(vv_fb_scalar($remoteRaw, "{$ru}_TIER3_DELAY") ?: 720), + 'tier4' => (int)(vv_fb_scalar($remoteRaw, "{$ru}_TIER4_DELAY") ?: 1440), + ]; + return $tiers; +} + +// ── Known hosts ─────────────────────────────────────────────────────────────── + +function vv_fb_known_hosts(): array { + $master = vv_read_conf_raw('master.conf'); + preg_match_all('/^\s*(HOST(\d+))(?:_NAME)?\s*=\s*["\']?(\S+?)["\']?\s*$/m', $master, $m); + $hosts = []; + foreach ($m[1] as $i => $key) { + $num = $m[2][$i]; + $name = trim($m[3][$i]); + $hosts['host' . $num] = $name; + } + ksort($hosts); + return $hosts ?: ['host1' => 'HOST1']; +} + +// ── Main data builder ───────────────────────────────────────────────────────── + +function vv_fb_all(): array { + $currentHost = vv_detect_host(); + $hosts = vv_fb_known_hosts(); + $tsPeers = vv_pt_ts_peers(); + $handbackReq = (int)(vv_fb_scalar(vv_read_conf_raw('master.conf'), 'FALLBACK_HANDBACK_STRIKES') ?: 3); + $fbEnabled = vv_fb_scalar(vv_read_conf_raw('master.conf'), 'FALLBACK_ENABLED') === 'true'; + + // Read all host conf raws upfront + $raws = []; + foreach (array_keys($hosts) as $slot) { + $raws[$slot] = vv_read_conf_raw($slot . '.conf'); + } + + // SSH key — from local host conf + $myId = strtoupper($currentHost); + $myRaw = $raws[$currentHost] ?? ''; + $mySshKey = vv_fb_scalar($myRaw, $myId . '_SSH_KEY'); + + $nodes = []; + foreach ($hosts as $slot => $hostname) { + $isMe = ($slot === $currentHost || $currentHost === 'unknown'); + $tsLabel = strtolower($hostname); + $ts = $tsPeers[$tsLabel] ?? ['online' => null, 'active' => false, 'ip' => null]; + $ip = $ts['ip'] ?? null; + + // State + if ($isMe) { + $state = vv_fb_local_state(); + } elseif ($ip && $mySshKey) { + $state = vv_fb_remote_state($ip, $mySshKey); + } else { + $state = vv_fb_parse_state(''); + $state['state'] = $ts['online'] === false ? 'OFFLINE' : 'UNREACHABLE'; + } + + // Running containers + if ($isMe) { + $running = vv_fb_local_running(); + } elseif ($ip && $mySshKey && $ts['online']) { + $running = vv_fb_remote_running($ip, $mySshKey); + } else { + $running = []; + } + + // Covers: for a 2-node setup, each covers the other + // For N nodes this would need a different approach — for now, assume 2-node + $covers = null; + foreach ($hosts as $otherSlot => $otherHostname) { + if ($otherSlot === $slot) continue; + $coveringRaw = $raws[$slot] ?? ''; + $remoteRaw = $raws[$otherSlot] ?? ''; + $covers = [ + 'slot' => $otherSlot, + 'id' => strtoupper($otherSlot), + 'hostname' => $otherHostname, + ] + vv_fb_covers($slot, $otherSlot, $coveringRaw, $remoteRaw); + break; // 2-node only + } + + $nodes[] = [ + 'slot' => $slot, + 'id' => strtoupper($slot), + 'hostname' => $hostname, + 'is_me' => $isMe, + 'ts_online' => $ts['online'], + 'state' => $state, + 'running' => $running, + 'covers' => $covers, + ]; + } + + return [ + 'ts' => time(), + 'fb_enabled' => $fbEnabled, + 'handback_req' => $handbackReq, + 'nodes' => $nodes, + ]; +} diff --git a/Plugin/usr/local/emhttp/plugins/varaverk/include/media.php b/Plugin/unraid/include/media.php similarity index 100% rename from Plugin/usr/local/emhttp/plugins/varaverk/include/media.php rename to Plugin/unraid/include/media.php diff --git a/Plugin/usr/local/emhttp/plugins/varaverk/include/monitor.php b/Plugin/unraid/include/monitor.php similarity index 98% rename from Plugin/usr/local/emhttp/plugins/varaverk/include/monitor.php rename to Plugin/unraid/include/monitor.php index 0127c64..a33263b 100644 --- a/Plugin/usr/local/emhttp/plugins/varaverk/include/monitor.php +++ b/Plugin/unraid/include/monitor.php @@ -306,8 +306,11 @@ function vv_partner_state(): array { if ($h) $tsPeers[$h] = (bool)($peer['Online'] ?? false); } + $hostIds = array_filter(array_keys($vars), fn($k) => preg_match('/^HOST\d+$/', $k) && ($vars[$k] ?? '') !== ''); + sort($hostIds); + $hosts = []; - foreach (['HOST1', 'HOST2', 'HOST3', 'HOST4'] as $id) { + foreach ($hostIds as $id) { $hostname = $vars[$id] ?? ''; if (!$hostname) continue; $isMe = strcasecmp($hostname, $myName) === 0; @@ -364,8 +367,10 @@ function vv_fallback_active(): array { $myName = trim(shell_exec('hostname -s') ?: ''); // Identify which HOST id we are + $allHostIds = array_filter(array_keys($vars), fn($k) => preg_match('/^HOST\d+$/', $k) && ($vars[$k] ?? '') !== ''); + sort($allHostIds); $myId = null; - foreach (['HOST1', 'HOST2', 'HOST3', 'HOST4'] as $id) { + foreach ($allHostIds as $id) { if (strcasecmp($vars[$id] ?? '', $myName) === 0) { $myId = $id; break; } } if (!$myId) return []; @@ -383,7 +388,7 @@ function vv_fallback_active(): array { $rawConf = vv_read_conf_raw($confFile); $result = []; - foreach (['HOST1', 'HOST2', 'HOST3', 'HOST4'] as $covered) { + foreach ($allHostIds as $covered) { if ($covered === $myId) continue; $coveredHostname = $vars[$covered] ?? ''; if (!$coveredHostname) continue; diff --git a/Plugin/unraid/include/partnership.php b/Plugin/unraid/include/partnership.php new file mode 100644 index 0000000..859ee26 --- /dev/null +++ b/Plugin/unraid/include/partnership.php @@ -0,0 +1,178 @@ + ($v['PARTNERSHIP_ENABLED'] ?? 'false') === 'true', + 'owner_host' => $v['PARTNERSHIP_OWNER_HOST'] ?? '', + 'sync_min' => (int)($v['PARTNERSHIP_SYNC_INTERVAL'] ?? 15), + 'grace_hours' => (int)($v['PARTNERSHIP_GRACE_HOURS'] ?? 6), + 'offline_threshold' => (int)($v['PARTNERSHIP_OFFLINE_THRESHOLD'] ?? 30), + 'remove_tailscale' => ($v['PARTNERSHIP_REMOVE_TAILSCALE'] ?? 'true') === 'true', + 'folderview3' => ($v['PARTNERSHIP_FOLDERVIEW3'] ?? 'false') === 'true', + 'tailscale_configured' => !empty($v['TAILSCALE_API_KEY']) && !empty($v['TAILSCALE_TAILNET']), + ]; +} + +// ── State file parser ───────────────────────────────────────────────────────── + +function vv_pt_read_db(string $path): array { + if (!file_exists($path)) return []; + $out = []; + foreach (file($path, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES) as $line) { + [$k, $v] = array_pad(explode('=', trim($line), 2), 2, ''); + if ($k) $out[trim($k)] = trim($v, '"\''); + } + return $out; +} + +// ── Tailscale peers ─────────────────────────────────────────────────────────── + +function vv_pt_ts_peers(): array { + $raw = shell_exec('tailscale status --json 2>/dev/null') ?: '{}'; + $data = json_decode($raw, true) ?: []; + $peers = []; + + // Self + $self = $data['Self'] ?? []; + $selfLabel = strtolower(explode('.', $self['DNSName'] ?? '')[0]); + if ($selfLabel) { + $peers[$selfLabel] = [ + 'online' => true, + 'active' => true, + 'ip' => $self['TailscaleIPs'][0] ?? null, + ]; + } + + // Peers + foreach ($data['Peer'] ?? [] as $peer) { + $label = strtolower(explode('.', $peer['DNSName'] ?? '')[0]); + if (!$label) continue; + $peers[$label] = [ + 'online' => (bool)($peer['Online'] ?? false), + 'active' => (bool)($peer['Active'] ?? false), + 'ip' => $peer['TailscaleIPs'][0] ?? null, + ]; + } + return $peers; +} + +// ── SSH helper — run a single command on a remote host ──────────────────────── + +function vv_pt_ssh(string $ip, string $sshKey, string $cmd, int $timeout = 4): string { + if (!$ip || !$sshKey || !file_exists($sshKey)) return ''; + $full = sprintf( + 'ssh -i %s -o ConnectTimeout=%d -o StrictHostKeyChecking=no -o BatchMode=yes root@%s %s 2>/dev/null', + escapeshellarg($sshKey), $timeout, escapeshellarg($ip), escapeshellarg($cmd) + ); + return shell_exec($full) ?: ''; +} + +// ── System info ─────────────────────────────────────────────────────────────── + +function vv_pt_local_system(): array { + $ver = ''; + if (file_exists('/etc/unraid-version')) { + preg_match('/VERSION="([^"]+)"/', file_get_contents('/etc/unraid-version'), $m); + $ver = $m[1] ?? ''; + } + $uptime = 0; + if (file_exists('/proc/uptime')) { + $uptime = (int)explode(' ', file_get_contents('/proc/uptime'))[0]; + } + return ['unraid_version' => $ver, 'uptime_sec' => $uptime]; +} + +function vv_pt_remote_system(string $ip, string $sshKey): array { + $out = vv_pt_ssh($ip, $sshKey, + 'printf "%s\nUPTIME:%s\n" "$(cat /etc/unraid-version 2>/dev/null)" "$(cat /proc/uptime 2>/dev/null)"'); + $ver = ''; + preg_match('/VERSION="([^"]+)"/', $out, $m); + if ($m) $ver = $m[1]; + $uptime = 0; + if (preg_match('/UPTIME:([\d.]+)/', $out, $m)) $uptime = (int)$m[1]; + return ['unraid_version' => $ver, 'uptime_sec' => $uptime]; +} + +// ── Per-node data ───────────────────────────────────────────────────────────── + +function vv_pt_nodes(): array { + $currentHost = vv_detect_host(); + $hosts = vv_arr_known_hosts(); // ['host1' => 'hostname', ...] + $vars = vv_conf_vars(); + $tsPeers = vv_pt_ts_peers(); + $ownerSlot = strtolower($vars['PARTNERSHIP_OWNER_HOST'] ?? ''); + + // SSH key for this host + $myId = strtoupper($currentHost); + $myRaw = vv_read_conf_raw($currentHost . '.conf'); + $mySshKey = vv_arr_scalar($myRaw, $myId . '_SSH_KEY'); + + $nodes = []; + foreach ($hosts as $slot => $hostname) { + $isMe = ($slot === $currentHost || $currentHost === 'unknown'); + $isOwner = (strtolower($ownerSlot) === $slot); + + // Tailscale + $tsLabel = strtolower($hostname); + $ts = $tsPeers[$tsLabel] ?? ['online' => null, 'active' => false, 'ip' => null]; + + // Fallback state + $fbState = 'UNKNOWN'; + $fbPath = '/boot/config/fallback_state.db'; + if ($isMe) { + $fb = vv_pt_read_db($fbPath); + $fbState = $fb['state'] ?? 'UNKNOWN'; + } elseif ($ts['online'] && $ts['ip'] && $mySshKey) { + $out = vv_pt_ssh($ts['ip'], $mySshKey, 'cat /boot/config/fallback_state.db 2>/dev/null'); + if ($out) { + $fb = []; + foreach (explode("\n", $out) as $line) { + [$k, $v] = array_pad(explode('=', trim($line), 2), 2, ''); + if ($k) $fb[trim($k)] = trim($v, '"\''); + } + $fbState = $fb['state'] ?? 'UNKNOWN'; + } + } + + // Partnership DB — local only (each server writes its own) + $dbPath = "/boot/config/partnership_{$hostname}.db"; + $ptDb = vv_pt_read_db($dbPath); + + // System info + $system = $isMe + ? vv_pt_local_system() + : ($ts['online'] && $ts['ip'] && $mySshKey ? vv_pt_remote_system($ts['ip'], $mySshKey) : []); + + $nodes[] = [ + 'slot' => $slot, + 'id' => strtoupper($slot), + 'hostname' => $hostname, + 'is_me' => $isMe, + 'is_owner' => $isOwner, + 'ts_online' => $ts['online'], + 'ts_active' => $ts['active'], + 'ts_ip' => $ts['ip'], + 'fallback' => $fbState, + 'partnership' => $ptDb, + 'system' => $system, + ]; + } + return $nodes; +} + +// ── Entry point ─────────────────────────────────────────────────────────────── + +function vv_partnership_all(): array { + return [ + 'config' => vv_pt_config(), + 'nodes' => vv_pt_nodes(), + 'ts' => time(), + ]; +} diff --git a/Plugin/usr/local/emhttp/plugins/varaverk/include/scheduler.php b/Plugin/unraid/include/scheduler.php similarity index 91% rename from Plugin/usr/local/emhttp/plugins/varaverk/include/scheduler.php rename to Plugin/unraid/include/scheduler.php index 92af3e8..c46682e 100644 --- a/Plugin/usr/local/emhttp/plugins/varaverk/include/scheduler.php +++ b/Plugin/unraid/include/scheduler.php @@ -8,6 +8,10 @@ require_once __DIR__ . '/confform.php'; define('SCHEDULE_FILE', '/boot/config/plugins/varaverk/schedule.json'); define('CRON_FILE', '/boot/config/plugins/varaverk/varaverk.cron'); +function vv_pretty_label(string $slug): string { + return ucwords(str_replace('_', ' ', $slug)); +} + function vv_schedule_load(): array { if (!file_exists(SCHEDULE_FILE)) return []; $data = json_decode(file_get_contents(SCHEDULE_FILE), true); @@ -89,7 +93,7 @@ function vv_cron_rebuild(array $schedule): bool { foreach ($schedule as $entry) { if (empty($entry['enabled']) || empty($entry['cron']) || empty($entry['id'])) continue; // Event-triggered jobs are handled by static event scripts, not cron. - if (str_starts_with($entry['cron'], '@array_')) continue; + if (str_starts_with($entry['cron'], 'array_')) continue; $id = $entry['id']; // When a child's orch is enabled it is the sole trigger — suppress independent cron. if (isset($childToOrch[$id]) && !empty($schedule[$childToOrch[$id]]['enabled'])) continue; @@ -235,7 +239,7 @@ function vv_custom_scripts(): array { $entry = $schedule[$rel] ?? []; $scripts[] = [ 'id' => $rel, - 'label' => basename($path, '.sh'), + 'label' => vv_pretty_label(basename($path, '.sh')), 'desc' => vv_script_description($path), 'enabled' => (bool)($entry['enabled'] ?? false), 'cron' => $entry['cron'] ?? '', @@ -284,7 +288,7 @@ function vv_script_library(): array { $parts = explode('/', $rel); if (count($parts) < 2 || in_array($parts[0], $exclude)) continue; if (in_array($rel, $orchIds) || isset($confMap[$rel])) continue; - $library[] = ['id' => $rel, 'label' => basename($rel, '.sh')]; + $library[] = ['id' => $rel, 'label' => vv_pretty_label(basename($rel, '.sh'))]; } } catch (Exception $e) {} usort($library, fn($a, $b) => strcmp($a['id'], $b['id'])); @@ -298,52 +302,35 @@ function vv_folders_load(): array { return is_array($f) ? $f : []; } -// Walk the scripts repo and return the job tree: -// hardcoded array-event entries first, then cron-scheduled orchestrators +// Walk the scripts repo and return the job tree. +// Type is derived from the saved cron value: array_start / array_stop → 'event', else 'orchestrator'. +// Well-known event orchs get their cron seeded from $eventDefaults when not yet in schedule.json. +// Any orch or custom script can carry array_start / array_stop as its cron — the event scripts fire all of them. function vv_job_tree(): array { $scriptsDir = SCRIPTS_DIR; $schedule = vv_schedule_load(); - // Hardcoded array-event entries — always present, trigger via Unraid event scripts - $eventDefs = [ - ['id' => 'Orchestrators/array_started.sh', 'cron' => '@array_start', 'label' => 'Array Starting'], - ['id' => 'Orchestrators/array_stopping.sh', 'cron' => '@array_stop', 'label' => 'Array Stopping'], + // First-run seeds only — applied when schedule.json has no entry for these IDs yet. + $eventDefaults = [ + 'Orchestrators/array_started.sh' => 'array_start', + 'Orchestrators/array_stopping.sh' => 'array_stop', ]; - $orchs = []; - $eventIds = []; - foreach ($eventDefs as $ev) { - $id = $ev['id']; - $path = "$scriptsDir/$id"; - $entry = $schedule[$id] ?? []; - $eventIds[] = $id; - $orchs[] = [ - 'id' => $id, - 'label' => $ev['label'], - 'desc' => vv_script_description($path), - 'type' => 'event', - 'enabled' => (bool)($entry['enabled'] ?? false), - 'cron' => $ev['cron'], - 'log_enabled' => (bool)($entry['log_enabled'] ?? false), - 'suggested_cron' => '', - 'suggested_label' => '', - 'children' => vv_script_children($path, $schedule), - ]; - } - // Cron-scheduled orchestrators — discovered by glob, event entries excluded - $orchPattern = "$scriptsDir/Orchestrators/*.sh"; - foreach (glob($orchPattern) ?: [] as $path) { - $id = 'Orchestrators/' . basename($path); - if (in_array($id, $eventIds, true)) continue; - $entry = $schedule[$id] ?? ['enabled' => false, 'cron' => '']; - $suggested = vv_script_suggested_cron($path); - $orchs[] = [ + $orchs = []; + foreach (glob("$scriptsDir/Orchestrators/*.sh") ?: [] as $path) { + $id = 'Orchestrators/' . basename($path); + $default = $eventDefaults[$id] ?? ''; + $entry = $schedule[$id] ?? ['enabled' => false, 'cron' => $default]; + $cron = $entry['cron'] ?? $default; + $isEvent = str_starts_with($cron, 'array_'); + $suggested = $isEvent ? ['cron' => '', 'label' => ''] : vv_script_suggested_cron($path); + $orchs[] = [ 'id' => $id, - 'label' => basename($path, '.sh'), + 'label' => vv_pretty_label(basename($path, '.sh')), 'desc' => vv_script_description($path), - 'type' => 'orchestrator', + 'type' => $isEvent ? 'event' : 'orchestrator', 'enabled' => (bool)($entry['enabled'] ?? false), - 'cron' => $entry['cron'] ?? '', + 'cron' => $cron, 'log_enabled' => (bool)($entry['log_enabled'] ?? false), 'suggested_cron' => $suggested['cron'], 'suggested_label' => $suggested['label'], @@ -351,6 +338,20 @@ function vv_job_tree(): array { 'conf_arrays' => vv_orch_conf_arrays($path), ]; } + + // Events first (array_start before array_stop), then alphabetical by label. + usort($orchs, function($a, $b) { + $ae = $a['type'] === 'event' ? 0 : 1; + $be = $b['type'] === 'event' ? 0 : 1; + if ($ae !== $be) return $ae - $be; + if ($ae === 0) { + $as = str_contains($a['cron'], 'start') ? 0 : 1; + $bs = str_contains($b['cron'], 'start') ? 0 : 1; + if ($as !== $bs) return $as - $bs; + } + return strcmp($a['label'], $b['label']); + }); + return $orchs; } @@ -492,7 +493,7 @@ function vv_script_children(string $orchPath, array $schedule): array { $suggested = vv_script_suggested_cron("$scriptsDir/$rel"); $children[] = [ 'id' => $rel, - 'label' => basename($rel, '.sh'), + 'label' => vv_pretty_label(basename($rel, '.sh')), 'desc' => vv_script_description("$scriptsDir/$rel"), 'type' => 'script', 'enabled' => (bool)($entry['enabled'] ?? false), diff --git a/Plugin/usr/local/emhttp/plugins/varaverk/include/vms.php b/Plugin/unraid/include/vms.php similarity index 100% rename from Plugin/usr/local/emhttp/plugins/varaverk/include/vms.php rename to Plugin/unraid/include/vms.php diff --git a/Plugin/unraid/include/watchdog.php b/Plugin/unraid/include/watchdog.php new file mode 100644 index 0000000..f6ddad6 --- /dev/null +++ b/Plugin/unraid/include/watchdog.php @@ -0,0 +1,341 @@ + $k) $out[$k] = $pairs[2][$i]; + return $out; +} + +function vv_wd_scalar(string $raw, string $varname): string { + return preg_match('/^\s*' . preg_quote($varname, '/') . '\s*=\s*"?([^"\n]*)"?/m', $raw, $m) + ? trim($m[1]) : ''; +} + +// ── State file parsers ──────────────────────────────────────────────────────── + +// Parses key:value format (with optional key=value mixed in) +function vv_wd_parse_kv(string $text): array { + $out = []; + foreach (explode("\n", $text) as $line) { + $line = trim($line); + if (!$line) continue; + if (str_contains($line, ':')) { + [$k, $v] = explode(':', $line, 2); + $out[trim($k)] = trim($v); + } elseif (str_contains($line, '=')) { + [$k, $v] = explode('=', $line, 2); + $out[trim($k)] = trim($v, '"\''); + } + } + return $out; +} + +// Restart log: "container|timestamp" one per line +function vv_wd_parse_restart_log(string $text, int $windowSeconds = 86400): array { + $now = time(); + $cutoff = $now - $windowSeconds; + $entries = []; + foreach (explode("\n", trim($text)) as $line) { + $line = trim($line); + if (!$line || !str_contains($line, '|')) continue; + [$name, $ts] = explode('|', $line, 2); + $ts = (int)$ts; + if ($ts >= $cutoff) $entries[] = ['name' => trim($name), 'ts' => $ts]; + } + usort($entries, fn($a, $b) => $b['ts'] - $a['ts']); + return $entries; +} + +// Skip list: one container name per line +function vv_wd_parse_skiplist(string $text): array { + return array_values(array_filter(array_map('trim', explode("\n", $text)))); +} + +// Reboot log: one timestamp per line +function vv_wd_parse_reboot_log(string $text, int $windowHrs = 12): array { + $cutoff = time() - ($windowHrs * 3600); + $entries = []; + foreach (explode("\n", trim($text)) as $line) { + $ts = (int)trim($line); + if ($ts > 0 && $ts >= $cutoff) $entries[] = $ts; + } + rsort($entries); + return $entries; +} + +// ── Local system snapshot ───────────────────────────────────────────────────── + +function vv_wd_local_system(): array { + // RAM + $memRaw = file_exists('/proc/meminfo') ? file_get_contents('/proc/meminfo') : ''; + $memTotal = 0; $memAvail = 0; + if (preg_match('/^MemTotal:\s+(\d+)/m', $memRaw, $m)) $memTotal = (int)$m[1] * 1024; + if (preg_match('/^MemAvailable:\s+(\d+)/m', $memRaw, $m)) $memAvail = (int)$m[1] * 1024; + + // Load + cores + $loadRaw = file_exists('/proc/loadavg') ? file_get_contents('/proc/loadavg') : '0 0 0'; + $loadParts = explode(' ', trim($loadRaw)); + $load1 = (float)($loadParts[0] ?? 0); + $cores = (int)(trim(shell_exec('nproc 2>/dev/null') ?: '1')); + + // Uptime + $uptimeRaw = file_exists('/proc/uptime') ? file_get_contents('/proc/uptime') : '0'; + $uptime = (int)explode(' ', $uptimeRaw)[0]; + + // Docker daemon alive + $daemonOk = (trim(shell_exec('docker info >/dev/null 2>&1; echo $?') ?: '1') === '0'); + + // OOM count (from stability watchdog OOM file — just the prev cycle count) + $oomFile = '/tmp/system_watchdog_oom.db'; + $oomCount = file_exists($oomFile) ? (int)trim(file_get_contents($oomFile)) : 0; + + return [ + 'mem_total' => $memTotal, + 'mem_avail' => $memAvail, + 'load1' => $load1, + 'cores' => $cores, + 'uptime' => $uptime, + 'daemon_ok' => $daemonOk, + 'oom_count' => $oomCount, + ]; +} + +// ── Local state files ───────────────────────────────────────────────────────── + +function vv_wd_local_states(string $restartLogPath): array { + $rwRaw = @file_get_contents('/tmp/resource_watchdog_state.db') ?: ''; + $dockRaw = @file_get_contents('/tmp/container_watchdog_state.db') ?: ''; + $skipRaw = @file_get_contents('/boot/config/system_watchdog_failed.db') ?: ''; + $sysRaw = @file_get_contents('/tmp/system_watchdog_state.db') ?: ''; + $rebootRaw = @file_get_contents('/boot/config/system_watchdog_reboots.db')?: ''; + $restartRaw= @file_get_contents($restartLogPath) ?: ''; + + $rw = vv_wd_parse_kv($rwRaw); + $dock = vv_wd_parse_kv($dockRaw); + $sys = vv_wd_parse_kv($sysRaw); + + // Container strikes: everything in docker state that isn't a flag + $strikes = []; + foreach ($dock as $k => $v) { + if ($k !== 'daemon_strikes' && $k !== 'daemon_restarted_flag' && (int)$v > 0) + $strikes[$k] = (int)$v; + } + + // Stability strikes: everything in sys state that isn't a flag key + $sysStrikes = []; + foreach ($sys as $k => $v) { + if (!str_contains($k, '=') && (int)$v > 0) + $sysStrikes[$k] = (int)$v; + } + + return [ + 'rw_level' => (int)($rw['rm_action_level'] ?? 0), + 'rw_recover' => (int)($rw['rm_recover_cycles'] ?? 0), + 'rw_paused' => array_filter(explode(',', $rw['rm_paused_containers'] ?? '')), + 'rw_stopped' => array_filter(explode(',', $rw['rm_stopped_containers'] ?? '')), + 'mem_shutdown' => ($rw['mem_shutdown_active'] ?? 'false') === 'true', + 'daemon_strikes' => (int)($dock['daemon_strikes'] ?? 0), + 'daemon_restart' => ($dock['daemon_restarted_flag'] ?? 'false') === 'true', + 'ctr_strikes' => $strikes, + 'skiplist' => vv_wd_parse_skiplist($skipRaw), + 'sys_strikes' => $sysStrikes, + 'reboots' => vv_wd_parse_reboot_log($rebootRaw), + 'restarts' => vv_wd_parse_restart_log($restartRaw), + ]; +} + +// ── Remote data via SSH ─────────────────────────────────────────────────────── + +function vv_wd_remote_data(string $ip, string $sshKey, string $restartLogPath): array { + // Bundle into one SSH call + $cmd = "printf 'UPTIME:%s\nLOAD:%s\nCORES:%s\nMEMTOTAL:%s\nMEMAVAIL:%s\nDAEMON:%s\nOOM:%s\n---RW---\n%s\n---DOCK---\n%s\n---SKIP---\n%s\n---SYS---\n%s\n---REBOOT---\n%s\n---RESTART---\n%s\n' " + . '"$(cat /proc/uptime|cut -d\" \" -f1)" ' + . '"$(cat /proc/loadavg|cut -d\" \" -f1)" ' + . '"$(nproc)" ' + . '"$(grep -m1 MemTotal /proc/meminfo|awk \"{print \\\$2}\")" ' + . '"$(grep -m1 MemAvailable /proc/meminfo|awk \"{print \\\$2}\")" ' + . '"$(docker info >/dev/null 2>&1 && echo ok || echo err)" ' + . '"$(cat /tmp/system_watchdog_oom.db 2>/dev/null||echo 0)" ' + . '"$(cat /tmp/resource_watchdog_state.db 2>/dev/null)" ' + . '"$(cat /tmp/container_watchdog_state.db 2>/dev/null)" ' + . '"$(cat /boot/config/system_watchdog_failed.db 2>/dev/null)" ' + . '"$(cat /tmp/system_watchdog_state.db 2>/dev/null)" ' + . '"$(cat /boot/config/system_watchdog_reboots.db 2>/dev/null)" ' + . '"$(cat ' . escapeshellarg($restartLogPath) . ' 2>/dev/null)"'; + + $out = vv_pt_ssh($ip, $sshKey, $cmd, 8); + if (!$out) return null; + + // Parse sections + $sections = preg_split('/^---\w+---$/m', $out); + $header = $sections[0] ?? ''; + $rwRaw = $sections[1] ?? ''; + $dockRaw = $sections[2] ?? ''; + $skipRaw = $sections[3] ?? ''; + $sysRaw = $sections[4] ?? ''; + $rebootRaw= $sections[5] ?? ''; + $restartRaw=$sections[6] ?? ''; + + // Parse header lines + $hdr = []; + foreach (explode("\n", $header) as $line) { + if (preg_match('/^(\w+):(.*)$/', trim($line), $m)) $hdr[$m[1]] = trim($m[2]); + } + + $rw = vv_wd_parse_kv($rwRaw); + $dock = vv_wd_parse_kv($dockRaw); + $sys = vv_wd_parse_kv($sysRaw); + + $strikes = []; + foreach ($dock as $k => $v) { + if ($k !== 'daemon_strikes' && $k !== 'daemon_restarted_flag' && (int)$v > 0) + $strikes[$k] = (int)$v; + } + $sysStrikes = []; + foreach ($sys as $k => $v) { + if (!str_contains($k, '=') && (int)$v > 0) $sysStrikes[$k] = (int)$v; + } + + $memTotal = (int)($hdr['MEMTOTAL'] ?? 0) * 1024; + $memAvail = (int)($hdr['MEMAVAIL'] ?? 0) * 1024; + + return [ + 'system' => [ + 'mem_total' => $memTotal, + 'mem_avail' => $memAvail, + 'load1' => (float)($hdr['LOAD'] ?? 0), + 'cores' => (int)($hdr['CORES'] ?? 1), + 'uptime' => (int)($hdr['UPTIME'] ?? 0), + 'daemon_ok' => ($hdr['DAEMON'] ?? '') === 'ok', + 'oom_count' => (int)($hdr['OOM'] ?? 0), + ], + 'states' => [ + 'rw_level' => (int)($rw['rm_action_level'] ?? 0), + 'rw_recover' => (int)($rw['rm_recover_cycles'] ?? 0), + 'rw_paused' => array_filter(explode(',', $rw['rm_paused_containers'] ?? '')), + 'rw_stopped' => array_filter(explode(',', $rw['rm_stopped_containers'] ?? '')), + 'mem_shutdown' => ($rw['mem_shutdown_active'] ?? 'false') === 'true', + 'daemon_strikes'=> (int)($dock['daemon_strikes'] ?? 0), + 'daemon_restart'=> ($dock['daemon_restarted_flag'] ?? 'false') === 'true', + 'ctr_strikes' => $strikes, + 'skiplist' => vv_wd_parse_skiplist($skipRaw), + 'sys_strikes' => $sysStrikes, + 'reboots' => vv_wd_parse_reboot_log($rebootRaw), + 'restarts' => vv_wd_parse_restart_log($restartRaw), + ], + ]; +} + +// ── Config inventory ────────────────────────────────────────────────────────── + +function vv_wd_node_config(string $slot, string $raw, string $masterRaw): array { + $id = strtoupper($slot); + return [ + 'monitored' => vv_wd_bash_assoc($raw, "{$id}_WATCHDOG_CONTAINERS"), + 'urls' => vv_wd_bash_assoc($raw, "{$id}_WATCHDOG_CONTAINER_URLS"), + 'required' => vv_wd_bash_array($raw, "{$id}_WATCHDOG_REQUIRED_CONTAINERS"), + 'ignore' => vv_wd_bash_array($raw, "{$id}_WATCHDOG_SCAN_IGNORE"), + 'pause_list' => vv_wd_bash_array($raw, "{$id}_RW_PAUSE_CONTAINERS"), + 'stop_list' => vv_wd_bash_array($raw, "{$id}_RW_STOP_CONTAINERS"), + 'critical' => vv_wd_bash_array($masterRaw, "RW_CRITICAL_CONTAINERS"), + ]; +} + +// ── Entry point ─────────────────────────────────────────────────────────────── + +function vv_wd_all(): array { + $currentHost = vv_detect_host(); + $tsPeers = vv_pt_ts_peers(); + $masterRaw = vv_read_conf_raw('master.conf'); + $restartLog = '/mnt/user/appdata/unraid_scripts/data/container_restart_history.db'; + + // Config thresholds from master.conf + $cfg = [ + 'rw_soft_gb' => (float)(vv_wd_scalar($masterRaw, 'RW_RAM_SOFT_GB') ?: 12), + 'rw_medium_gb' => (float)(vv_wd_scalar($masterRaw, 'RW_RAM_MEDIUM_GB') ?: 8), + 'rw_hard_gb' => (float)(vv_wd_scalar($masterRaw, 'RW_RAM_HARD_GB') ?: 6), + 'rw_recover_gb' => (float)(vv_wd_scalar($masterRaw, 'RW_RAM_RECOVER_GB') ?: 20), + 'rw_load_soft' => (float)(vv_wd_scalar($masterRaw, 'RW_LOAD_SOFT_MULTIPLIER') ?: 2.0), + 'rw_load_med' => (float)(vv_wd_scalar($masterRaw, 'RW_LOAD_MEDIUM_MULTIPLIER') ?: 3.0), + 'sys_mem_gb' => (float)(vv_wd_scalar($masterRaw, 'SYS_WATCHDOG_MEM_GB') ?: 4), + 'sys_strikes' => (int)(vv_wd_scalar($masterRaw, 'SYS_WATCHDOG_STRIKE_LIMIT') ?: 2), + 'reboot_limit' => (int)(vv_wd_scalar($masterRaw, 'SYS_WATCHDOG_REBOOT_LIMIT') ?: 3), + 'reboot_window' => (int)(vv_wd_scalar($masterRaw, 'SYS_WATCHDOG_REBOOT_WINDOW_HRS') ?: 12), + 'restart_limit' => (int)(vv_wd_scalar($masterRaw, 'WATCHDOG_CONTAINER_RESTART_LIMIT') ?: 3), + 'startup_grace' => (int)(vv_wd_scalar($masterRaw, 'WATCHDOG_STARTUP_GRACE') ?: 600), + 'soft_mem_pct' => (int)(vv_wd_scalar($masterRaw, 'SOFT_MEM_THRESHOLD') ?: 80), + 'soft_cpu_pct' => (int)(vv_wd_scalar($masterRaw, 'SOFT_CPU_THRESHOLD') ?: 80), + 'hard_cpu_pct' => (int)(vv_wd_scalar($masterRaw, 'HARD_CPU_THRESHOLD') ?: 85), + 'cpu_fail_lim' => (int)(vv_wd_scalar($masterRaw, 'CPU_FAIL_LIMIT') ?: 2), + 'resp_fail_lim' => (int)(vv_wd_scalar($masterRaw, 'RESP_FAIL_LIMIT') ?: 2), + ]; + + // SSH key from current host conf + $myId = strtoupper($currentHost); + $myRaw = vv_read_conf_raw($currentHost . '.conf'); + $mySshKey = vv_wd_scalar($myRaw, $myId . '_SSH_KEY'); + + // Known hosts + preg_match_all('/^\s*(HOST(\d+))(?:_NAME)?\s*=\s*["\']?(\S+?)["\']?\s*$/m', $masterRaw, $m); + $hosts = []; + foreach ($m[1] as $i => $key) { + $hosts['host' . $m[2][$i]] = trim($m[3][$i]); + } + ksort($hosts); + if (!$hosts) $hosts = ['host1' => 'HOST1']; + + $nodes = []; + foreach ($hosts as $slot => $hostname) { + $isMe = ($slot === $currentHost || $currentHost === 'unknown'); + $tsLabel = strtolower($hostname); + $ts = $tsPeers[$tsLabel] ?? ['online' => null, 'active' => false, 'ip' => null]; + $ip = $ts['ip'] ?? null; + $raw = vv_read_conf_raw($slot . '.conf'); + + if ($isMe) { + $system = vv_wd_local_system(); + $states = vv_wd_local_states($restartLog); + } elseif ($ip && $mySshKey && $ts['online']) { + $remote = vv_wd_remote_data($ip, $mySshKey, $restartLog); + $system = $remote['system'] ?? null; + $states = $remote['states'] ?? null; + } else { + $system = null; + $states = null; + } + + $nodes[] = [ + 'slot' => $slot, + 'id' => strtoupper($slot), + 'hostname' => $hostname, + 'is_me' => $isMe, + 'ts_online' => $ts['online'], + 'system' => $system, + 'states' => $states, + 'config' => vv_wd_node_config($slot, $raw, $masterRaw), + ]; + } + + return [ + 'ts' => time(), + 'cfg' => $cfg, + 'nodes' => $nodes, + ]; +} diff --git a/Plugin/usr/local/emhttp/plugins/varaverk/js/varaverk.js b/Plugin/unraid/js/varaverk.js similarity index 100% rename from Plugin/usr/local/emhttp/plugins/varaverk/js/varaverk.js rename to Plugin/unraid/js/varaverk.js diff --git a/Plugin/unraid/pages/arrs.php b/Plugin/unraid/pages/arrs.php new file mode 100644 index 0000000..e689ac5 --- /dev/null +++ b/Plugin/unraid/pages/arrs.php @@ -0,0 +1,231 @@ + + +
+ Arrs + +
+ +
+
Loading…
+
+ + diff --git a/Plugin/usr/local/emhttp/plugins/varaverk/pages/config.php b/Plugin/unraid/pages/config.php similarity index 100% rename from Plugin/usr/local/emhttp/plugins/varaverk/pages/config.php rename to Plugin/unraid/pages/config.php diff --git a/Plugin/usr/local/emhttp/plugins/varaverk/pages/docs.php b/Plugin/unraid/pages/docs.php similarity index 100% rename from Plugin/usr/local/emhttp/plugins/varaverk/pages/docs.php rename to Plugin/unraid/pages/docs.php diff --git a/Plugin/unraid/pages/fallback.php b/Plugin/unraid/pages/fallback.php new file mode 100644 index 0000000..8b24807 --- /dev/null +++ b/Plugin/unraid/pages/fallback.php @@ -0,0 +1,243 @@ + + +
+ FallBack + +
+ +
+
Loading…
+
+ + diff --git a/Plugin/usr/local/emhttp/plugins/varaverk/pages/monitor.php b/Plugin/unraid/pages/monitor.php similarity index 96% rename from Plugin/usr/local/emhttp/plugins/varaverk/pages/monitor.php rename to Plugin/unraid/pages/monitor.php index fdacfef..a9ccd3a 100644 --- a/Plugin/usr/local/emhttp/plugins/varaverk/pages/monitor.php +++ b/Plugin/unraid/pages/monitor.php @@ -27,36 +27,19 @@
Loading...
- +

Scripts

Loading...
-
-

- Partner & Fallback - - - - - - - - - - - - - - - - - - - - -

+
+

Fallback

+
Loading...
+
+ +
+

Partner

Loading...
@@ -462,20 +445,9 @@ function vvPollMonitor() { `; - // ── Partner & Fallback ─────────────────────────────────────────────────── + // ── Partner ────────────────────────────────────────────────────────────── const pt = d.partner ?? {}; const ptHosts = pt.hosts ?? []; - const fb = d.fallback ?? {}; - const fbActive = d.fallback_active ?? []; - const state = (fb.state ?? 'UNKNOWN').toUpperCase(); - const stateMap = { - NORMAL: ['#4caf50', '✓ Nominal'], - FAILOVER: ['#f44336', '⚠ Failover active'], - NO_INTERNET: ['#ff9800', '⚡ Internet lost'], - DARK: ['#9e9e9e', '◌ Dark mode'], - UNKNOWN: ['#444', '— No state file'], - }; - const [sColor, sLabel] = stateMap[state] ?? ['#444', state]; const ptStatus = pt.enabled ? `enabled · sync every ${pt.sync_min}min` : `disabled`; @@ -497,32 +469,47 @@ function vvPollMonitor() { ● ${label}
`; }); + document.getElementById('vv-partner-body').innerHTML = ptHtml; - ptHtml += `
${sLabel}`; + // ── Fallback ────────────────────────────────────────────────────────────── + const fb = d.fallback ?? {}; + const fbActive = d.fallback_active ?? []; + const state = (fb.state ?? 'UNKNOWN').toUpperCase(); + const stateMap = { + NORMAL: ['#4caf50', '✓ Nominal'], + FAILOVER: ['#f44336', '⚠ Failover active'], + NO_INTERNET: ['#ff9800', '⚡ Internet lost'], + DARK: ['#9e9e9e', '◌ Dark mode'], + UNKNOWN: ['#444', '— No state file'], + }; + const [sColor, sLabel] = stateMap[state] ?? ['#444', state]; + + let fbHtml = `
${sLabel}`; if (state === 'FAILOVER') { const start = parseInt(fb.failover_start ?? 0); if (start > 0) { const sec = Math.floor(Date.now() / 1000) - start; - ptHtml += ` · ${Math.floor(sec/3600)}h ${Math.floor((sec%3600)/60)}m`; + fbHtml += ` · ${Math.floor(sec/3600)}h ${Math.floor((sec%3600)/60)}m`; } } - ptHtml += `
`; + fbHtml += `
`; if (fbActive.length) { fbActive.forEach(group => { - ptHtml += `
Running for ${group.hostname}
`; + fbHtml += `
Running for ${group.hostname}
`; group.containers.forEach(c => { const img = c.image.includes('/') ? c.image.split('/').pop() : c.image; - ptHtml += `
${c.name} ${img}
`; }); }); + } else if (state === 'NORMAL') { + fbHtml += `
No containers active
`; } - - document.getElementById('vv-partner-body').innerHTML = ptHtml; + document.getElementById('vv-fallback-body').innerHTML = fbHtml; // ── CPU ───────────────────────────────────────────────────────────────── const cpu = d.cpu ?? {}; @@ -895,7 +882,7 @@ function vvPollMonitor() { } vvPollMonitor(); -setInterval(vvPollMonitor, 5000); +setInterval(vvPollMonitor, 2000); // Pin pools card width to CPU card width across rows function vvSyncCardWidths() { diff --git a/Plugin/unraid/pages/partnership.php b/Plugin/unraid/pages/partnership.php new file mode 100644 index 0000000..60b8d80 --- /dev/null +++ b/Plugin/unraid/pages/partnership.php @@ -0,0 +1,290 @@ + + +
+ Partnership + +
+ + +
+
Loading…
+
+ + +
+
Loading…
+
+ + +
+

Actions

+
Loading…
+
+ + diff --git a/Plugin/usr/local/emhttp/plugins/varaverk/pages/scheduler.php b/Plugin/unraid/pages/scheduler.php similarity index 76% rename from Plugin/usr/local/emhttp/plugins/varaverk/pages/scheduler.php rename to Plugin/unraid/pages/scheduler.php index 1e18869..8bb807f 100644 --- a/Plugin/usr/local/emhttp/plugins/varaverk/pages/scheduler.php +++ b/Plugin/unraid/pages/scheduler.php @@ -79,14 +79,13 @@ $runningScripts = array_unique($runningScripts); onchange="vvSaveOrch(this)"> - - - - - - + ⚙ + + + @@ -129,7 +128,7 @@ $runningScripts = array_unique($runningScripts); - + @@ -165,9 +164,10 @@ $runningScripts = array_unique($runningScripts); onchange="vvSaveChild(this)"> - + ⚙ + placeholder="cron (orch off only)" + onblur="vvSaveCronBlur(this)"> @@ -230,8 +230,8 @@ $runningScripts = array_unique($runningScripts); onchange="vvSaveJob(this)"> - - + ⚙ + @@ -265,9 +265,10 @@ $runningScripts = array_unique($runningScripts); onchange="vvSaveJob(this)"> - + ⚙ + placeholder="cron expression" + onblur="vvSaveCronBlur(this)"> @@ -295,16 +296,13 @@ $runningScripts = array_unique($runningScripts); -
- -
@@ -319,17 +317,20 @@ $runningScripts = array_unique($runningScripts); Scheduler Information -
+
+ - + + @@ -355,6 +356,10 @@ $runningScripts = array_unique($runningScripts); +
@@ -370,36 +375,42 @@ $runningScripts = array_unique($runningScripts);
-
+
▾ How do I use this
    -
  • Toggle — saves to schedule.json immediately
    • -
  • Cron — 5-field expression; hit Save Schedule to apply all at once
    • -
  • Run — fires script immediately regardless of schedule
    • -
  • Dry Run — same as Run with DRY_RUN=1
    • -
  • Log — opens script log in this panel; auto-scrolls to latest
    • -
  • Stop — SIGTERM → 3 s → SIGKILL; clears stuck locks
    • -
  • Verbose — appends --log for detailed per-step output
    • -
  • Orch ON — sole trigger; cron fires it; it calls children in order
    • -
  • Orch OFF — never runs; all children suppressed and get individual cron fields
    • -
  • Child (orch ON) — toggle comments/uncomments the script's line in master.conf
    • -
  • Child (orch ON, no array) — orch hardcodes the call; toggle shows active but is display-only
    • -
  • Child (orch OFF) — give it its own cron to run standalone
    • -
  • ⚡ Array events — triggered by Unraid array start/stop; no cron field
    • -
  • Rsync badge — writes TIER_RSYNC_ENABLED flag to master.conf directly
    • -
  • Rsync (orch ON) — Run/Dry Run/Log hidden; orch controls the rsync call
    • -
  • Rsync (orch OFF) — enter a location path and standalone cron, then Save; both must be filled for the cron to fire rsync independently
    • -
  • Arrange — drag scripts between orchs or reorder within an orch; right panel shows unassigned scripts that can be dragged in; drag to the right panel removes from orch
    • -
  • Save Arrangement — commits all moves and reorders to master.conf
    • -
  • + Folder — creates a named collapsible subfolder in Custom Scripts; scripts can be dragged into folders
    • -
  • Script name — click opens settings (non-Advanced) or full source editor (Advanced)
    • -
  • Suggested cron — click the cron code in the Scripts panel to apply & save it to that card instantly
    • -
  • Advanced — top-right button; turns blue when on; script-name click shows full editable source; conf file buttons appear in the status bar
    • -
  • ← Scheduler Info — back button returns here from any log, editor, or script view
    • +
  • Toggle — saves immediately to schedule.json
    • +
  • Cron field — auto-saved on focus-out; hint shows plain English; red border = invalid
    • +
  • ⚙ Cog — opens settings panel: script header + matching README/Manual sections + config fields
    • +
  • Script name — click navigates to the matching entry in the Orch tree (right panel)
    • +
  • Run — fires script immediately; resets the cron timer so next fire is one full interval later
    • +
  • Dry Run — same as Run but passes --dry-run; no changes written to disk
    • +
  • Log — opens script log here; auto-scrolls; use Search box to filter/highlight lines
    • +
  • Stop — SIGTERM → 3 s → SIGKILL; clears stuck lock files
    • +
  • Verbose — appends --log flag for per-item detail output
    • +
  • Cancel — discards changes and returns here; file is never touched until Save
    • +
  • ⚡ Array events — set any cron field to array_start or array_stop; badge appears; fires on Unraid array start/stop; works on any orch or custom script
    • +
  • Arrange — drag scripts between orchs; drop to right panel removes from orch; Save Arrangement commits to master.conf
    • +
  • + Folder — collapsible subfolder in Custom Scripts; drag scripts in
    • +
  • Suggested cron — click the cron code badge in the Orch tree to apply and save instantly
    • +
  • Status dot (tree) — green/orange/red ● = last run ok/warn/error; dim = never run; hover for detail
    • +
  • Cron Calculator — type an expression or plain English; shows description + next 5 fires; Apply pushes to last-focused cron field
    • +
  • Recent Activity — last 24 runs; click a row to open that log; auto-expands on errors
    • +
  • Recent Errors — last error per script (7 days); click to open log; auto-expands on new errors
    • +
  • Advanced — top-right button, turns blue; ⚙ shows enriched view (header + docs + config); raw conf editing unlocked
    • +
  • Log search — filter box in toolbar when log is open; highlights matches, dims others
    • +
  • ← Scheduler Info — returns here from any log, editor, conf, or script view
    • +
  • Orchestrator & Child Behaviour
    • +
  • Orch ON — sole trigger; cron fires the orch; it calls children in sequence
    • +
  • Orch OFF — never runs automatically; children suppressed; each can get its own standalone cron
    • +
  • Child (orch ON) — toggle comments/uncomments the script in master.conf
    • +
  • Child (orch ON, no array) — orch hardcodes the call; toggle is display-only
    • +
  • Child (orch OFF) — enter a cron to run it standalone
    • +
  • Rsync badge — toggle writes TIER_RSYNC_ENABLED directly to master.conf
    • +
  • Rsync (orch OFF) — fill location + standalone cron + Save; both required for independent firing
@@ -415,6 +426,39 @@ $runningScripts = array_unique($runningScripts);
+ +
+
+ ▸ + Cron Calculator +
+ +
+ + +
+
+ ▸ + Recent Activity + +
+ +
+
@@ -516,7 +560,7 @@ common.sh provides the shared runtime every script sources: acquire_lock(), log_ Logs are written per script to the appdata path. The Log button shows the latest run. Verbose mode passes --log for per-step detail. -Array events bypass cron entirely. @array_start fires when Unraid brings the array up; @array_stop fires before shutdown. array_started handles all post-start setup in sequence; array_stopping handles the graceful teardown. +Array events bypass cron entirely. array_start fires when Unraid brings the array up; array_stop fires before shutdown. array_started handles all post-start setup in sequence; array_stopping handles the graceful teardown. The watchdog tier runs every 15 minutes as a single-pass chain: resource_watchdog frees memory pressure first, docker_watchdog restarts down containers into that headroom, system_watchdog checks storage/WebGUI/network, stability_watchdog reboots as a last resort. The order matters — restarting containers before freeing memory just restarts them into the same pressure.
@@ -643,13 +687,18 @@ Still the same two servers, two households, the same media stack running itself. - - + + ⚡ - - + + "> +
@@ -667,8 +716,13 @@ Still the same two servers, two households, the same media stack running itself. - - + + "> + @@ -766,9 +820,13 @@ Still the same two servers, two households, the same media stack running itself.
- - + +
+ + +
@@ -897,7 +955,9 @@ function vvFitRight() { } const ed = document.getElementById('vv-editor'); if (ed.style.display !== 'none') { - document.getElementById('vv-editor-body').style.height = Math.max(60, contentH - 38) + 'px'; + const _edH = Math.max(60, contentH - 38) + 'px'; + document.getElementById('vv-editor-body').style.height = _edH; + document.getElementById('vv-ln-gutter').style.height = _edH; } const cf = document.getElementById('vv-confform'); if (cf.style.display !== 'none') { @@ -920,7 +980,9 @@ function vvShowLogMode(id) { document.getElementById('vv-arrange-workspace').style.display = 'none'; document.getElementById('vv-log-pre').style.display = ''; document.getElementById('vv-back-btn').style.display = ''; + document.getElementById('vv-log-search').style.display = ''; document.getElementById('vv-clear-btn').style.display = ''; + document.getElementById('vv-cancel-edit-btn').style.display = 'none'; document.getElementById('vv-save-script-btn').style.display = 'none'; document.getElementById('vv-save-conf-btn').style.display = 'none'; document.getElementById('vv-save-rawconf-btn').style.display = 'none'; @@ -934,8 +996,6 @@ function vvShowLogMode(id) { const _pre = document.getElementById('vv-log-pre'); _pre.removeEventListener('scroll', vvOnLogScroll); _pre.addEventListener('scroll', vvOnLogScroll); - document.getElementById('vv-save-schedule-btn').disabled = false; - document.getElementById('vv-save-schedule-btn').style.opacity = ''; const name = id.replace(/\.sh$/, '').split('/').pop(); document.getElementById('vv-log-title').textContent = name; document.getElementById('vv-restore-btn').style.display = 'none'; @@ -980,7 +1040,10 @@ function vvBackToSuggestions() { // Clear script info selection if (vvSelectedRow) { vvSelectedRow.classList.remove('vv-sb-selected'); vvSelectedRow = null; } vvCurrentSiId = null; + document.getElementById('vv-log-search').style.display = 'none'; + document.getElementById('vv-log-search').value = ''; document.getElementById('vv-clear-btn').style.display = 'none'; + document.getElementById('vv-cancel-edit-btn').style.display = 'none'; document.getElementById('vv-save-script-btn').style.display = 'none'; document.getElementById('vv-save-conf-btn').style.display = 'none'; document.getElementById('vv-save-rawconf-btn').style.display = 'none'; @@ -990,8 +1053,6 @@ function vvBackToSuggestions() { document.getElementById('vv-delete-script-btn').style.display = 'none'; document.getElementById('vv-confform').style.display = 'none'; vvConfId = null; - document.getElementById('vv-save-schedule-btn').disabled = false; - document.getElementById('vv-save-schedule-btn').style.opacity = ''; const lastJob = localStorage.getItem('vv-last-job'); if (lastJob) { const lname = lastJob.replace(/\.sh$/, '').split('/').pop(); @@ -1042,13 +1103,17 @@ function vvFetchRight() { if (!d.ok) { pre.textContent = '✗ ' + (d.error ?? 'Error'); return; } const content = d.content || ''; vvSetLogBtnState(vvActiveId, content.trim().length > 0); - pre.textContent = content.trim() ? (invert ? content.split('\n').reverse().join('\n') : content) : '(no log yet)'; + const display = content.trim() ? (invert ? content.split('\n').reverse().join('\n') : content) : '(no log yet)'; + pre._raw = display; + const term = document.getElementById('vv-log-search')?.value.trim() ?? ''; + if (term) vvFilterLog(term); else pre.textContent = display; if (autoScroll) { pre.scrollTop = invert ? 0 : pre.scrollHeight; } else { requestAnimationFrame(() => { pre.scrollTop = savedScroll; }); } - ts.textContent = d.ts ? 'Last run: ' + new Date(d.ts * 1000).toLocaleString() : ''; + const _d = d.ts ? new Date(d.ts * 1000) : null; + ts.innerHTML = _d ? 'Last run:' + _d.toLocaleDateString([],{month:'numeric',day:'numeric',year:'numeric'}) + '' + _d.toLocaleTimeString([],{hour:'2-digit',minute:'2-digit',second:'2-digit'}) + '' : ''; }) .catch(() => {}); } @@ -1129,6 +1194,30 @@ function vvSelectLog(btn) { vvOpenRight(job.dataset.id); } +// Run a script by ID directly — no DOM card needed. +function vvRunById(id) { + vvOpenRight(id); + vvSetDot(id); + vvRunningSet.add(id); + vvPost('/plugins/varaverk/api/run.php', {id, manual: '1'}) + .then(d => { + if (!d.ok) { + document.getElementById('vv-log-pre').textContent = '✗ ' + (d.error ?? 'Failed to start'); + vvClearDot(id); + vvRunningSet.delete(id); + } + }); +} + +function vvGitPull(btn) { + const id = 'git_pull_execute.sh'; + btn.disabled = true; + btn.textContent = '⟳ Pulling…'; + vvRunById(id); + // Re-enable once the log poll confirms it's running or done + setTimeout(() => { btn.disabled = false; btn.textContent = '↻ Git Pull'; }, 4000); +} + function vvRunJob(btn) { const job = btn.closest('[data-id]'); const id = job.dataset.id; @@ -1206,6 +1295,14 @@ function vvFlashStatus(el, msg, ok) { el._t = setTimeout(() => { el.textContent = ''; }, 3000); } +// Cron blur-to-save — fires when focus leaves any cron input. +// Skips rsync standalone (conf_flag) cron fields — those are saved together with location via their own Save button. +function vvSaveCronBlur(input) { + const job = input.closest('[data-id]'); + if (!job || job.dataset.type === 'conf_flag') return; + vvSaveJob(input); +} + // Generic save — used for log_enabled toggles and custom scripts. function vvSaveJob(el) { const job = el.closest('[data-id]'); @@ -1280,8 +1377,10 @@ function vvSaveChild(el) { const enabled = el.checked; if (child.dataset.type === 'conf_flag') { - const name = child.dataset.flagName; + const name = child.dataset.flagName; child.dataset.flagValue = enabled ? '1' : '0'; + const badge = child.querySelector('.vv-flag-badge'); + if (badge) badge.style.display = enabled ? '' : 'none'; vvPost('/plugins/varaverk/api/flag_toggle.php', {name, enabled: enabled ? '1' : '0'}) .then(d => { if (d.ok) vvFlashSaved(child); }); return; @@ -1298,33 +1397,6 @@ function vvSaveChild(el) { } } -function vvSaveAll() { - const status = document.getElementById('vv-save-all-status'); - const toSave = []; - document.querySelectorAll('#vv-sched-left [data-id]').forEach(job => { - if (job.classList.contains('vv-script')) { - if (job.dataset.type === 'conf_flag') return; - const orchCard = job.closest('.vv-sched-card'); - const orchOn = orchCard?.querySelector('.vv-orch-row .vv-enabled')?.checked ?? false; - if (orchOn) return; - } - toSave.push(job); - }); - if (!toSave.length) { vvFlashStatus(status, '✓ Saved', true); return; } - status.textContent = 'Saving…'; - const batch = toSave.map(job => ({ - id: job.dataset.id, - enabled: job.querySelector('.vv-enabled').checked ? '1' : '0', - cron: job.querySelector('.vv-cron')?.value.trim() ?? '', - log_enabled: job.querySelector('.vv-log-enabled')?.checked ? '1' : '0', - })); - vvPost('/plugins/varaverk/api/scheduler.php', {batch: JSON.stringify(batch)}) - .then(d => { - if (d.ok) toSave.forEach(job => vvFlashSaved(job)); - vvFlashStatus(status, d.ok ? '✓ All saved' : '✗ ' + (d.error ?? 'Failed'), d.ok); - }) - .catch(() => vvFlashStatus(status, '✗ Request failed', false)); -} function vvToggleAdvanced(btn) { const card = btn.closest('.vv-sched-card'); @@ -1362,6 +1434,7 @@ function vvAddScript() { nameEl.readOnly = false; document.getElementById('vv-editor-body').value = '#!/bin/bash\n\n'; vvShowEditorMode('New Script'); + vvSyncHlOverlay(); nameEl.focus(); } @@ -1382,14 +1455,20 @@ function vvEditScript(id) { fetch('/plugins/varaverk/api/script.php?id=' + encodeURIComponent(id)) .then(r => r.json()) - .then(d => { document.getElementById('vv-editor-body').value = d.ok ? d.content : '# Error loading script'; }) - .catch(() => { document.getElementById('vv-editor-body').value = '# Error loading script'; }); + .then(d => { + document.getElementById('vv-editor-body').value = d.ok ? d.content : '# Error loading script'; + vvSyncHlOverlay(); + }) + .catch(() => { + document.getElementById('vv-editor-body').value = '# Error loading script'; + vvSyncHlOverlay(); + }); vvShowEditorMode(name); } function vvShowEditorMode(title) { - document.getElementById('vv-editor').classList.remove('vv-editor-hl'); + document.getElementById('vv-editor').classList.add('vv-editor-hl'); document.getElementById('vv-suggestions').style.display = 'none'; document.getElementById('vv-log-pre').style.display = 'none'; document.getElementById('vv-si-view').style.display = 'none'; @@ -1400,6 +1479,8 @@ function vvShowEditorMode(title) { document.getElementById('vv-save-script-btn').style.display = ''; document.getElementById('vv-save-rawconf-btn').style.display = 'none'; document.getElementById('vv-advanced-mode-btn').style.display = 'none'; + document.getElementById('vv-log-search').style.display = 'none'; + document.getElementById('vv-log-search').value = ''; document.getElementById('vv-clear-btn').style.display = 'none'; document.getElementById('vv-stop-btn').style.display = 'none'; document.getElementById('vv-auto-scroll-label').style.display = 'none'; @@ -1407,9 +1488,8 @@ function vvShowEditorMode(title) { document.getElementById('vv-log-title').textContent = title; document.getElementById('vv-log-ts').textContent = ''; document.getElementById('vv-confform').style.display = 'none'; + document.getElementById('vv-cancel-edit-btn').style.display = ''; document.getElementById('vv-save-conf-btn').style.display = 'none'; - document.getElementById('vv-save-schedule-btn').disabled = true; - document.getElementById('vv-save-schedule-btn').style.opacity = '0.4'; document.getElementById('vv-delete-script-btn').style.display = vvEditorId ? '' : 'none'; if (vvPollTimer) { clearInterval(vvPollTimer); vvPollTimer = null; } requestAnimationFrame(vvFitRight); @@ -1460,19 +1540,20 @@ function vvShowConfMode(title) { document.getElementById('vv-confform').style.display = ''; document.getElementById('vv-back-btn').style.display = ''; document.getElementById('vv-restore-btn').style.display = 'none'; + document.getElementById('vv-cancel-edit-btn').style.display = ''; document.getElementById('vv-save-script-btn').style.display = 'none'; document.getElementById('vv-save-conf-btn').style.display = ''; document.getElementById('vv-save-rawconf-btn').style.display = 'none'; document.getElementById('vv-advanced-mode-btn').style.display = 'none'; document.getElementById('vv-delete-script-btn').style.display = 'none'; + document.getElementById('vv-log-search').style.display = 'none'; + document.getElementById('vv-log-search').value = ''; document.getElementById('vv-clear-btn').style.display = 'none'; document.getElementById('vv-stop-btn').style.display = 'none'; document.getElementById('vv-auto-scroll-label').style.display = 'none'; document.getElementById('vv-invert-log-label').style.display = 'none'; document.getElementById('vv-log-title').textContent = title; document.getElementById('vv-log-ts').textContent = ''; - document.getElementById('vv-save-schedule-btn').disabled = true; - document.getElementById('vv-save-schedule-btn').style.opacity = '0.4'; if (vvPollTimer) { clearInterval(vvPollTimer); vvPollTimer = null; } requestAnimationFrame(vvFitRight); } @@ -1532,7 +1613,7 @@ function vvSaveConf() { if (!vvConfId) return; const _cfName = vvConfId.replace(/\.sh$/, '').split('/').pop(); if (!confirm('Save configuration changes for "' + _cfName + '"?')) return; - const inputs = document.querySelectorAll('#vv-confform .vv-cf-input'); + const inputs = document.querySelectorAll('#vv-confform .vv-cf-input, #vv-si-view .vv-cf-input'); const changes = []; inputs.forEach(el => changes.push({key: el.dataset.key, file: el.dataset.file, type: el.dataset.type, value: el.value})); const btn = document.getElementById('vv-save-conf-btn'); @@ -1608,28 +1689,41 @@ function vvApplySugCron(el, event) { .then(d => { if (d.ok) vvFlashSaved(jobEl); }); } -// Script name label click: non-advanced → conf settings; advanced → full script/editor +// Script name click: navigate to the matching row in the Orch tree and open its info. +// Falls back to cog behaviour for scripts not in the tree (e.g. custom scripts). function vvClickLabel(el) { const job = el.closest('[data-id]'); const id = job.dataset.id; - if (vvAdvancedMode) { - if (id.startsWith('Custom/')) { - vvEditScript(id); - } else { - if (vvActiveId) { - const old = document.querySelector('[data-id="' + CSS.escape(vvActiveId) + '"]'); - if (old) old.querySelector('.vv-job-row').classList.remove('vv-row-selected'); - } - vvActiveId = null; - if (vvPollTimer) { clearInterval(vvPollTimer); vvPollTimer = null; } - if (vvSelectedRow) { vvSelectedRow.classList.remove('vv-sb-selected'); vvSelectedRow = null; } - vvCurrentSiId = id; - vvCurrentSiHdr = ''; - vvShowScriptInfoMode(id.replace(/\.sh$/, '').split('/').pop(), '', id); - } - } else { - vvEditConf(id); + const treeRow = document.querySelector('#vv-sb-tree [data-id="' + CSS.escape(id) + '"]'); + if (!treeRow) { vvClickCog(el); return; } + // Expand parent sb-children if collapsed so the row is visible on back-nav + const parent = treeRow.parentElement; + if (parent?.classList.contains('vv-sb-children') && parent.style.display === 'none') { + parent.style.display = ''; + const expand = parent.previousElementSibling?.querySelector('.vv-sb-expand'); + if (expand) expand.textContent = '▾'; } + vvSelectScript(treeRow); +} + +// Cog click: open settings / enriched info for this script (moved from label click). +function vvClickCog(el) { + const job = el.closest('[data-id]'); + const id = job.dataset.id; + if (id.startsWith('Custom/')) { + vvAdvancedMode ? vvEditScript(id) : vvEditConf(id); + return; + } + if (vvActiveId) { + const old = document.querySelector('[data-id="' + CSS.escape(vvActiveId) + '"]'); + if (old) old.querySelector('.vv-job-row').classList.remove('vv-row-selected'); + } + vvActiveId = null; + if (vvPollTimer) { clearInterval(vvPollTimer); vvPollTimer = null; } + if (vvSelectedRow) { vvSelectedRow.classList.remove('vv-sb-selected'); vvSelectedRow = null; } + vvCurrentSiId = id; + vvCurrentSiHdr = ''; + vvShowScriptInfoMode(id.replace(/\.sh$/, '').split('/').pop(), '', id); } // ── Advanced mode toggle ────────────────────────────────────────────────────── @@ -1669,7 +1763,7 @@ let vvBoardTimer = null; // Minimal cron parser — returns next Date after now, or null if unparseable. function vvCronNext(expr) { - if (!expr || /^@/.test(expr)) return null; + if (!expr || expr === 'array_start' || expr === 'array_stop') return null; const parts = expr.trim().split(/\s+/); if (parts.length !== 5) return null; const [mF, hF, dF, moF, wF] = parts; @@ -1719,7 +1813,7 @@ function vvBuildNextRuns() { if (!enabled) return; const cronEl = orchRow.querySelector('.vv-cron'); const cron = cronEl?.value.trim() ?? ''; - if (!cron || cron.startsWith('@')) return; + if (!cron || cron === 'array_start' || cron === 'array_stop') return; const label = orchRow.querySelector('.vv-job-label')?.textContent.trim() ?? card.dataset.id; const next = vvCronNext(cron); if (next) rows.push({ label, cron, next, diffMs: next - now }); @@ -1787,26 +1881,66 @@ function vvUpdateLocks(locks) { body.innerHTML = html + ''; } +// Auto-expand a suggestions block body if it's currently collapsed. +function vvAutoExpand(bodyId) { + const body = document.getElementById(bodyId); + if (!body || body.style.display !== 'none') return; + body.style.display = ''; + const chevron = body.previousElementSibling?.querySelector('.vv-sug-chevron'); + if (chevron) chevron.textContent = '▾'; +} + +function vvErrIsAcked(script, ts) { + return parseInt(localStorage.getItem('vv-ack-' + script) || '0') >= ts; +} + +function vvAckError(script, ts, btn) { + localStorage.setItem('vv-ack-' + script, ts); + const row = btn.closest('.vv-err-row'); + const list = row?.closest('.vv-errors-list'); + row?.remove(); + const badge = document.getElementById('vv-errors-badge'); + const body = document.getElementById('vv-errors-body'); + const remaining = list?.querySelectorAll('.vv-err-row').length ?? 0; + if (!remaining && body) + body.innerHTML = '
All errors acknowledged.
'; + const cur = Math.max(0, parseInt(badge?.textContent || '0') - 1); + if (badge) { badge.textContent = cur || ''; badge.style.display = cur ? '' : 'none'; } +} + function vvUpdateErrors(errors) { const body = document.getElementById('vv-errors-body'); const badge = document.getElementById('vv-errors-badge'); if (!body) return; - badge.style.display = errors.length ? '' : 'none'; - badge.textContent = errors.length || ''; if (!errors.length) { + badge.style.display = 'none'; badge.textContent = ''; body.innerHTML = '
No recent errors.
'; return; } + // Filter out acknowledged entries + const unacked = errors.filter(e => !vvErrIsAcked(e.script, e.ts)); + badge.style.display = unacked.length ? '' : 'none'; + badge.textContent = unacked.length || ''; + if (!unacked.length) { + body.innerHTML = '
All errors acknowledged.
'; + return; + } const now = Math.floor(Date.now() / 1000); let html = '
'; - for (const e of errors) { + for (const e of unacked) { + const jobId = JSON.stringify(e.script + '.sh'); + const label = e.script.split('/').pop().replace(/_/g, ' '); html += '
' - + '
' + vvEscHtml(e.script) + '' - + '' + vvFmtAge(now - e.ts) + '
' + + '
' + + '' + vvEscHtml(label) + '' + + '' + vvFmtAge(now - e.ts) + '' + + '' + + '
' + '
' + vvEscHtml(e.line) + '
' + '
'; } body.innerHTML = html + '
'; + vvAutoExpand('vv-errors-body'); } function vvUpdatePartner(partner) { @@ -1845,7 +1979,7 @@ function vvClearLock(file, btn) { function vvValidateCronExpr(expr) { if (!expr) return true; - if (/^@/.test(expr)) return true; + if (expr === 'array_start' || expr === 'array_stop') return true; const parts = expr.trim().split(/\s+/); if (parts.length !== 5) return false; const ranges = [[0,59],[0,23],[1,31],[1,12],[0,7]]; @@ -1859,6 +1993,309 @@ function vvValidateCronExpr(expr) { }); } +// ── Log search ─────────────────────────────────────────────────────────────── + +function vvFilterLog(term) { + const pre = document.getElementById('vv-log-pre'); + const raw = pre._raw || ''; + if (!term.trim()) { pre.textContent = raw; return; } + const esc = s => s.replace(/&/g,'&').replace(//g,'>'); + const escRe = term.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + const re = new RegExp('(' + escRe + ')', 'gi'); + pre.innerHTML = raw.split('\n').map(line => { + const e = esc(line); + if (!line) return ''; + if (re.test(line)) { re.lastIndex = 0; return e.replace(re, '$1'); } + re.lastIndex = 0; + return '' + e + ''; + }).join('\n'); +} + +// ── Cron humanizer ─────────────────────────────────────────────────────────── + +function vvHumanCron(expr) { + expr = (expr || '').trim(); + if (!expr) return ''; + if (expr === 'array_start') return 'on array start'; + if (expr === 'array_stop') return 'on array stop'; + const p = expr.split(/\s+/); + if (p.length !== 5) return ''; + const [min, hour, dom, month, dow] = p; + const DAYS = ['Sun','Mon','Tue','Wed','Thu','Fri','Sat']; + const hhmm = (h, m) => { + const H = parseInt(h), M = parseInt(m); + if (H === 0 && M === 0) return 'midnight'; + if (H === 12 && M === 0) return 'noon'; + const ampm = H < 12 ? 'am' : 'pm'; + const h12 = H === 0 ? 12 : H > 12 ? H - 12 : H; + return M === 0 ? `${h12}${ampm}` : `${h12}:${String(M).padStart(2,'0')}${ampm}`; + }; + // Every minute / every N minutes + if (min === '*' && [hour,dom,month,dow].every(f => f === '*')) return 'every minute'; + if (/^\*\/(\d+)$/.test(min) && [hour,dom,month,dow].every(f => f === '*')) + return parseInt(min.slice(2)) === 1 ? 'every minute' : 'every ' + min.slice(2) + ' min'; + // Every hour / every N hours + if (min === '0' && hour === '*' && [dom,month,dow].every(f => f === '*')) return 'every hour'; + if (min === '0' && /^\*\/(\d+)$/.test(hour) && [dom,month,dow].every(f => f === '*')) + return parseInt(hour.slice(2)) === 1 ? 'every hour' : 'every ' + hour.slice(2) + ' hours'; + if (!/^\d+$/.test(min) || !/^\d+$/.test(hour)) return ''; + const T = hhmm(hour, min); + // Weekdays / weekends + if (dom === '*' && month === '*' && dow === '1-5') return 'weekdays at ' + T; + if (dom === '*' && month === '*' && /^(6,0|0,6)$/.test(dow)) return 'weekends at ' + T; + // Nth weekday of month (1-7 = 1st, 8-14 = 2nd, 15-21 = 3rd, 22-28 = 4th) + const NTH = [[1,7,'1st'],[8,14,'2nd'],[15,21,'3rd'],[22,28,'4th']]; + for (const [lo,hi,lbl] of NTH) { + if (dom === `${lo}-${hi}` && month === '*' && /^\d+$/.test(dow)) + return `${lbl} ${DAYS[parseInt(dow)] ?? 'day'} of month at ${T}`; + } + // Monthly on any specific day-of-month + if (/^\d+$/.test(dom) && month === '*' && dow === '*') { + const d = parseInt(dom); + const sfx = [11,12,13].includes(d) ? 'th' + : d % 10 === 1 ? 'st' : d % 10 === 2 ? 'nd' : d % 10 === 3 ? 'rd' : 'th'; + return (d === 1 ? 'monthly' : `${d}${sfx} of every month`) + ' at ' + T; + } + // Specific day(s) of week + if (dom === '*' && month === '*' && /^\d+$/.test(dow)) + return (DAYS[parseInt(dow)] ?? 'day ' + dow) + 's at ' + T; + if (dom === '*' && month === '*' && /^\d+(,\d+)+$/.test(dow)) + return dow.split(',').map(d => DAYS[parseInt(d)] ?? 'd'+d).join('/') + ' at ' + T; + // Daily + if (dom === '*' && month === '*' && dow === '*') return 'daily at ' + T; + return ''; +} + +// ── Natural language → cron expression ────────────────────────────────────── + +function vvNlToCron(text) { + const t = text.toLowerCase().replace(/\s+/g, ' ').trim(); + let m; + // Pure interval patterns (no time needed) + if (/\bevery\s+minute\b/.test(t)) return '* * * * *'; + if (/\bhourly\b/.test(t)) return '0 * * * *'; + if (/\bevery\s+hour\b/.test(t) && !/\d+\s*h/.test(t)) return '0 * * * *'; + if ((m = t.match(/\bevery\s+(\d+)\s*(?:minutes?|mins?)\b/))) return `*/${m[1]} * * * *`; + if ((m = t.match(/\bevery\s+(\d+)\s*hours?\b/))) return `0 */${m[1]} * * *`; + + // Parse time — optional, defaults to midnight when not given + const parseTime = s => { + let h = -1, min = 0, tm; + if (/\bmidnight\b/.test(s)) { h = 0; min = 0; } + else if (/\bnoon\b/.test(s)) { h = 12; min = 0; } + else if ((tm = s.match(/\b(\d{1,2}):(\d{2})\s*(am|pm)?\b/))) { h = parseInt(tm[1]); min = parseInt(tm[2]); if (tm[3]==='pm'&&h<12) h+=12; if (tm[3]==='am'&&h===12) h=0; } + else if ((tm = s.match(/\b(\d{1,2})\s*(am|pm)\b/))) { h = parseInt(tm[1]); if (tm[2]==='pm'&&h<12) h+=12; if (tm[2]==='am'&&h===12) h=0; } + else if ((tm = s.match(/\bat\s+(\d{1,2}):(\d{2})\b/))) { h = parseInt(tm[1]); min = parseInt(tm[2]); } + return h >= 0 && h < 24 ? {h, min} : null; + }; + const time = parseTime(t); + const H = time?.h ?? 0; // default midnight + const M = time?.min ?? 0; + + // Ordinals: 1st/2nd/3rd/4th and first/second/third/fourth + const ORD = {first:1,'1st':1, second:2,'2nd':2, third:3,'3rd':3, fourth:4,'4th':4}; + const ordM = t.match(/\b(first|1st|second|2nd|third|3rd|fourth|4th)\b/); + const ordN = ordM ? ORD[ordM[1]] : null; + + const DOW = {sun:0,sunday:0,mon:1,monday:1,tue:2,tuesday:2,wed:3,wednesday:3, + thu:4,thursday:4,fri:5,friday:5,sat:6,saturday:6}; + const dayHits = [...t.matchAll(/\b(sun(?:day)?|mon(?:day)?|tue(?:s(?:day)?)?|wed(?:nesday)?|thu(?:rs(?:day)?)?|fri(?:day)?|sat(?:urday)?)\b/g)]; + + // Nth weekday of month: "every 2nd sunday", "first monday of the month" + if (ordN && dayHits.length) { + const key = dayHits[0][1].slice(0, 3); + const dayNum = DOW[key] ?? DOW[dayHits[0][1]]; + if (dayNum !== undefined) { + const lo = (ordN - 1) * 7 + 1, hi = ordN * 7; + return `${M} ${H} ${lo}-${hi} * ${dayNum}`; + } + } + + // Monthly (no time required — defaults to midnight) + if (/\bmonthly\b|\bevery\s+month\b|\b(?:once\s+a\s+month)\b/.test(t)) return `${M} ${H} 1 * *`; + + // Day-of-week patterns + if (/\bweekdays?\b/.test(t) && !/weekend/.test(t)) return `${M} ${H} * * 1-5`; + if (/\bweekends?\b/.test(t)) return `${M} ${H} * * 6,0`; + + if (dayHits.length) { + const nums = [...new Set(dayHits.map(d => { const k=d[1].slice(0,3); return DOW[k]??DOW[d[1]]; }).filter(n=>n!==undefined))]; + if (nums.length) return `${M} ${H} * * ${nums.join(',')}`; + } + + // Require explicit time for a plain "daily" default + if (!time) return null; + return `${M} ${H} * * *`; +} + +// ── Cron calculator state ──────────────────────────────────────────────────── + +let vvCalcTarget = null; +let vvCalcExpr = ''; + +document.addEventListener('focusin', e => { + if (!e.target.classList.contains('vv-cron')) return; + if (e.target.classList.contains('vv-rsync-standalone')) return; + if (!e.target.closest('#vv-sched-left')) return; + vvCalcTarget = e.target; + const label = e.target.closest('[data-id]')?.querySelector('.vv-job-label')?.textContent.trim() ?? ''; + const lbl = document.getElementById('vv-calc-target-label'); + if (lbl) lbl.textContent = label || 'selected'; + const applyBtn = document.getElementById('vv-calc-apply'); + if (applyBtn && vvCalcExpr) applyBtn.style.display = ''; +}); + +function vvCalcNextFires(expr, n) { + const parts = expr.split(/\s+/); + const [mF, hF, dF, moF, wF] = parts; + const match = (f, v) => { + if (f==='*') return true; + if (/^\*\/\d+$/.test(f)) return v % parseInt(f.slice(2)) === 0; + if (/^\d+$/.test(f)) return parseInt(f) === v; + if (/^\d+-\d+$/.test(f)) { const [a,b]=f.split('-').map(Number); return v>=a&&v<=b; } + if (f.includes(',')) return f.split(',').some(x=>parseInt(x)===v); + return false; + }; + const fires = [], t = new Date(); + t.setSeconds(0,0); t.setMinutes(t.getMinutes()+1); + for (let i = 0; fires.length < n && i < 10080; i++, t.setMinutes(t.getMinutes()+1)) { + if (match(moF,t.getMonth()+1) && match(dF,t.getDate()) && match(wF,t.getDay()) && + match(hF,t.getHours()) && match(mF,t.getMinutes())) fires.push(new Date(t)); + } + return fires; +} + +function vvCalcUpdate(val) { + const result = document.getElementById('vv-calc-result'); + const applyBtn = document.getElementById('vv-calc-apply'); + val = val.trim(); + if (!val) { result.innerHTML = ''; applyBtn.style.display = 'none'; vvCalcExpr = ''; return; } + + let expr = '', desc = '', hint = ''; + + if (val === 'array_start' || val === 'array_stop') { + expr = val; desc = vvHumanCron(val) || val; + } else if (vvValidateCronExpr(val) && val.split(/\s+/).length === 5) { + expr = val; desc = vvHumanCron(val); + } else { + const nl = vvNlToCron(val); + if (nl) { expr = nl; desc = vvHumanCron(nl) || nl; } + else hint = 'Try: "every 15 min", "daily at 3am", "sundays at 2:30am", "weekdays at 9am"'; + } + + vvCalcExpr = expr; + + if (hint) { + result.innerHTML = '
' + vvEscHtml(hint) + '
'; + applyBtn.style.display = 'none'; + return; + } + + const esc = vvEscHtml; + let html = '
' + esc(expr) + '
'; + if (desc) html += '
' + esc(desc) + '
'; + + if (expr !== 'array_start' && expr !== 'array_stop') { + const fires = vvCalcNextFires(expr, 5); + if (fires.length) { + const now = new Date(); + html += '
Next ' + fires.length + ' fires
'; + for (const f of fires) { + const dur = vvFmtDuration(f - now); + const at = f.toLocaleTimeString([], {hour:'2-digit', minute:'2-digit'}); + const day = f.toLocaleDateString([], {weekday:'short', month:'short', day:'numeric'}); + html += '
in ' + dur + '' + + '' + day + ' · ' + at + '
'; + } + html += '
'; + } + } + + result.innerHTML = html; + applyBtn.style.display = vvCalcTarget ? '' : 'none'; +} + +function vvCalcApply() { + if (!vvCalcExpr || !vvCalcTarget) return; + vvCalcTarget.value = vvCalcExpr; + vvUpdateCronHint(vvCalcTarget); + vvSaveCronBlur(vvCalcTarget); + const btn = document.getElementById('vv-calc-apply'); + const lbl = document.getElementById('vv-calc-target-label'); + const saved = btn.innerHTML; + btn.innerHTML = '✓ Applied to ' + (lbl?.textContent ?? 'job'); + btn.style.color = '#4caf50'; + setTimeout(() => { btn.innerHTML = saved; btn.style.color = ''; }, 1800); +} + +function vvUpdateCronHint(input) { + if (!input || input.classList.contains('vv-rsync-standalone')) return; + // No hint when the row already shows an ⚡ array-event badge + const row = input.closest('.vv-job-row'); + if (row?.querySelector('.vv-event-badge')) { + const stale = input.previousElementSibling; + if (stale?.classList.contains('vv-cron-hint')) stale.style.display = 'none'; + return; + } + const hint = vvHumanCron(input.value); + let el = input.previousElementSibling; + if (!el || !el.classList.contains('vv-cron-hint')) { + el = document.createElement('span'); + el.className = 'vv-cron-hint'; + input.before(el); + } + el.textContent = hint; + el.style.display = hint ? '' : 'none'; +} + +// ── Recent Activity ────────────────────────────────────────────────────────── + +function vvTimeAgo(s) { + if (s < 60) return s + 's ago'; + if (s < 3600) return Math.floor(s/60) + 'm ago'; + if (s < 86400) return Math.floor(s/3600) + 'h ago'; + return Math.floor(s/86400) + 'd ago'; +} +function vvFmtDur(s) { + if (s <= 0) return '—'; + if (s < 60) return s + 's'; + if (s < 3600) return Math.floor(s/60) + 'm ' + (s%60) + 's'; + return Math.floor(s/3600) + 'h ' + Math.floor((s%3600)/60) + 'm'; +} + +function vvLoadRecentActivity() { + const body = document.getElementById('vv-activity-body'); + const badge = document.getElementById('vv-activity-badge'); + if (!body) return; + fetch('/plugins/varaverk/api/recent.php') + .then(r => r.json()) + .then(d => { + if (!d.ok || !d.runs?.length) { + body.innerHTML = '
No completed runs yet.
'; + if (badge) badge.style.display = 'none'; + return; + } + const now = Math.floor(Date.now() / 1000); + let errors = 0; + let html = '
'; + for (const r of d.runs) { + const cls = r.status === 'ok' ? 'vv-stat-ok' : r.status === 'warn' ? 'vv-stat-warn' : 'vv-stat-error'; + if (r.status !== 'ok' && r.status !== 'skipped') errors++; + html += '
' + + '●' + + '' + vvEscHtml(r.label) + '' + + '' + vvTimeAgo(now - r.start) + '' + + '' + vvFmtDur(r.dur) + '' + + '
'; + } + html += '
'; + body.innerHTML = html; + if (badge) { badge.textContent = errors || ''; badge.style.display = errors ? '' : 'none'; } + if (errors) vvAutoExpand('vv-activity-body'); + }) + .catch(() => {}); +} + function vvValidateCrons() { document.querySelectorAll('input.vv-cron').forEach(inp => { if (inp.disabled || inp.type === 'hidden') return; @@ -1872,16 +2309,19 @@ function vvBoardInit() { vvBuildNextRuns(); vvBoardPoll(); vvValidateCrons(); + vvLoadRecentActivity(); if (!vvBoardTimer) { - vvBoardTimer = setInterval(() => { vvBuildNextRuns(); vvBoardPoll(); }, 30000); + vvBoardTimer = setInterval(() => { vvBuildNextRuns(); vvBoardPoll(); vvLoadRecentActivity(); }, 30000); } - // Live cron validation on input - document.querySelectorAll('input.vv-cron').forEach(inp => { - inp.addEventListener('input', () => { - const ok = vvValidateCronExpr(inp.value.trim()); - inp.style.borderColor = inp.value.trim() && !ok ? '#f44336' : ''; - }); + // Live cron validation + humanizer hint on input + document.getElementById('vv-sched-left').addEventListener('input', e => { + if (!e.target.classList.contains('vv-cron')) return; + const ok = vvValidateCronExpr(e.target.value.trim()); + e.target.style.borderColor = e.target.value.trim() && !ok ? '#f44336' : ''; + vvUpdateCronHint(e.target); }); + // Seed hints for all existing cron values on load + document.querySelectorAll('#vv-sched-left input.vv-cron').forEach(vvUpdateCronHint); } // Apply syntax highlighting to all embedded header blocks @@ -2076,19 +2516,20 @@ function vvShowRawConfMode(title) { document.getElementById('vv-confform').style.display = 'none'; document.getElementById('vv-back-btn').style.display = ''; document.getElementById('vv-restore-btn').style.display = 'none'; + document.getElementById('vv-cancel-edit-btn').style.display = ''; document.getElementById('vv-save-script-btn').style.display = 'none'; document.getElementById('vv-save-conf-btn').style.display = 'none'; document.getElementById('vv-save-rawconf-btn').style.display = ''; document.getElementById('vv-advanced-mode-btn').style.display = 'none'; document.getElementById('vv-delete-script-btn').style.display = 'none'; + document.getElementById('vv-log-search').style.display = 'none'; + document.getElementById('vv-log-search').value = ''; document.getElementById('vv-clear-btn').style.display = 'none'; document.getElementById('vv-stop-btn').style.display = 'none'; document.getElementById('vv-auto-scroll-label').style.display = 'none'; document.getElementById('vv-invert-log-label').style.display = 'none'; document.getElementById('vv-log-title').textContent = title; document.getElementById('vv-log-ts').textContent = ''; - document.getElementById('vv-save-schedule-btn').disabled = true; - document.getElementById('vv-save-schedule-btn').style.opacity = '0.4'; if (vvPollTimer) { clearInterval(vvPollTimer); vvPollTimer = null; } requestAnimationFrame(vvFitRight); } @@ -2120,14 +2561,100 @@ function vvSyncHlOverlay() { const ov = document.getElementById('vv-hl-overlay'); if (!ov || !document.getElementById('vv-editor').classList.contains('vv-editor-hl')) return; const ta = document.getElementById('vv-editor-body'); - ov.innerHTML = vvHl(ta.value, false) + '\n'; - ov.scrollTop = ta.scrollTop; + ov.innerHTML = vvHl(ta.value, false) + '\n'; + ov.scrollTop = ta.scrollTop; + ov.scrollLeft = ta.scrollLeft; + vvUpdateLineNums(); } function vvSyncHlScroll() { + const ta = document.getElementById('vv-editor-body'); const ov = document.getElementById('vv-hl-overlay'); if (!ov || !document.getElementById('vv-editor').classList.contains('vv-editor-hl')) return; - ov.scrollTop = document.getElementById('vv-editor-body').scrollTop; + ov.scrollTop = ta.scrollTop; + ov.scrollLeft = ta.scrollLeft; + const gutter = document.getElementById('vv-ln-gutter'); + if (gutter) gutter.scrollTop = ta.scrollTop; +} + +function vvUpdateLineNums() { + const gutter = document.getElementById('vv-ln-gutter'); + const ta = document.getElementById('vv-editor-body'); + if (!gutter || !ta) return; + const count = ta.value.split('\n').length; + let nums = ''; + for (let i = 1; i <= count; i++) nums += i + '\n'; + gutter.textContent = nums; + gutter.scrollTop = ta.scrollTop; +} + +function vvEditorKeydown(e) { + const ta = e.target; + const start = ta.selectionStart; + const end = ta.selectionEnd; + const val = ta.value; + + // Ctrl/Cmd+S — save + if (e.key === 's' && (e.ctrlKey || e.metaKey)) { + e.preventDefault(); + if (vvRawConfFile) vvSaveRawConf(); else vvSaveScript(); + return; + } + + // Ctrl/Cmd+/ — toggle line comment + if (e.key === '/' && (e.ctrlKey || e.metaKey)) { + e.preventDefault(); + const lineStart = val.lastIndexOf('\n', start - 1) + 1; + const lineEnd = val.indexOf('\n', start); + const line = val.slice(lineStart, lineEnd === -1 ? undefined : lineEnd); + const newLine = /^\s*#/.test(line) + ? line.replace(/^(\s*)#\s?/, '$1') + : line.replace(/^(\s*)/, '$1# '); + const delta = newLine.length - line.length; + ta.value = val.slice(0, lineStart) + newLine + val.slice(lineEnd === -1 ? val.length : lineEnd); + ta.selectionStart = ta.selectionEnd = start + delta; + vvSyncHlOverlay(); return; + } + + // Tab / Shift+Tab — indent / dedent (2 spaces) + if (e.key === 'Tab') { + e.preventDefault(); + if (end > start && val.slice(start, end).includes('\n')) { + // Multi-line selection: indent/dedent every line in the selection + const blockStart = val.lastIndexOf('\n', start - 1) + 1; + const blockEnd = val.indexOf('\n', end - 1); + const block = val.slice(blockStart, blockEnd === -1 ? val.length : blockEnd); + const newBlock = e.shiftKey ? block.replace(/^ /gm, '') : block.replace(/^/gm, ' '); + ta.value = val.slice(0, blockStart) + newBlock + val.slice(blockEnd === -1 ? val.length : blockEnd); + ta.selectionStart = blockStart; + ta.selectionEnd = blockStart + newBlock.length; + } else if (e.shiftKey) { + // Dedent current line + const lineStart = val.lastIndexOf('\n', start - 1) + 1; + if (val.slice(lineStart, lineStart + 2) === ' ') { + ta.value = val.slice(0, lineStart) + val.slice(lineStart + 2); + ta.selectionStart = ta.selectionEnd = Math.max(lineStart, start - 2); + } + } else { + // Insert 2 spaces at cursor + ta.value = val.slice(0, start) + ' ' + val.slice(end); + ta.selectionStart = ta.selectionEnd = start + 2; + } + vvSyncHlOverlay(); return; + } + + // Enter — auto-indent: match current line's leading whitespace; extra indent after { + if (e.key === 'Enter') { + e.preventDefault(); + const lineStart = val.lastIndexOf('\n', start - 1) + 1; + const currentLine = val.slice(lineStart, start); + const indent = currentLine.match(/^(\s*)/)[1]; + const extra = /\{\s*$/.test(currentLine) ? ' ' : ''; + const insert = '\n' + indent + extra; + ta.value = val.slice(0, start) + insert + val.slice(end); + ta.selectionStart = ta.selectionEnd = start + insert.length; + vvSyncHlOverlay(); return; + } } // ── Script browser tree ─────────────────────────────────────────────────────── @@ -2163,6 +2690,7 @@ function vvShowScriptInfoMode(name, hdr, id) { document.getElementById('vv-si-view').style.display = ''; document.getElementById('vv-back-btn').style.display = ''; document.getElementById('vv-restore-btn').style.display = 'none'; + document.getElementById('vv-cancel-edit-btn').style.display = 'none'; document.getElementById('vv-save-script-btn').style.display = 'none'; document.getElementById('vv-save-conf-btn').style.display = 'none'; document.getElementById('vv-save-rawconf-btn').style.display = 'none'; @@ -2170,32 +2698,71 @@ function vvShowScriptInfoMode(name, hdr, id) { _advBtn.style.display = ''; _advBtn.classList.toggle('vv-adv-mode-on', vvAdvancedMode); document.getElementById('vv-delete-script-btn').style.display = 'none'; + document.getElementById('vv-log-search').style.display = 'none'; + document.getElementById('vv-log-search').value = ''; document.getElementById('vv-clear-btn').style.display = 'none'; document.getElementById('vv-stop-btn').style.display = 'none'; document.getElementById('vv-auto-scroll-label').style.display = 'none'; document.getElementById('vv-invert-log-label').style.display = 'none'; document.getElementById('vv-log-title').textContent = name; document.getElementById('vv-log-ts').textContent = ''; - document.getElementById('vv-save-schedule-btn').disabled = false; - document.getElementById('vv-save-schedule-btn').style.opacity = ''; if (vvPollTimer) { clearInterval(vvPollTimer); vvPollTimer = null; } - const si = document.getElementById('vv-si-view'); + const si = document.getElementById('vv-si-view'); const _isMd = /\.md$/i.test(id); - if (vvAdvancedMode || _isMd) { + + if (_isMd) { + // Markdown files always show rendered content si.innerHTML = '
(loading…)
'; fetch('/plugins/varaverk/api/readscript.php?id=' + encodeURIComponent(id)) .then(r => r.json()) .then(d => { - si.innerHTML = '
'
-          + (d.ok
-              ? (_isMd ? vvEscHtml(d.content) : vvHl(d.content, false))
-              : '# Error loading')
-          + '
'; + si.innerHTML = '
' + (d.ok ? vvEscHtml(d.content) : '# Error loading') + '
'; requestAnimationFrame(vvFitRight); }) .catch(() => { si.innerHTML = '
# Load failed
'; }); + + } else if (vvAdvancedMode) { + // Advanced mode: header + README/Manual sections + inline config settings + si.innerHTML = '

Loading…

'; + Promise.all([ + fetch('/plugins/varaverk/api/scriptinfo.php?id=' + encodeURIComponent(id)).then(r => r.json()), + fetch('/plugins/varaverk/api/confform.php?id=' + encodeURIComponent(id)).then(r => r.json()), + ]).then(([info, conf]) => { + let html = ''; + const dispHdr = (info.ok && info.header) ? info.header : hdr; + + if (dispHdr) { + html += '
' + + '
' + vvHl(dispHdr, true) + '
' + + '
'; + } + + if (info.ok && info.sections?.length) { + for (const sec of info.sections) { + html += '
' + + '
' + vvEscHtml(sec.source) + '
' + + '
' + vvEscHtml(sec.body) + '
' + + '
'; + } + } + + if (conf.ok && conf.groups?.length) { + html += '
' + + '
Config
' + + vvRenderConfForm(conf.groups) + + '
'; + vvConfId = id; + document.getElementById('vv-cancel-edit-btn').style.display = ''; + document.getElementById('vv-save-conf-btn').style.display = ''; + } + + si.innerHTML = html || '

No additional information found.

'; + requestAnimationFrame(vvFitRight); + }).catch(() => { si.innerHTML = '

Failed to load info.

'; }); + } else { + // Basic mode: header only si.innerHTML = '
' + vvHl(hdr, true) + '
'; } requestAnimationFrame(vvFitRight); diff --git a/Plugin/unraid/pages/watchdog.php b/Plugin/unraid/pages/watchdog.php new file mode 100644 index 0000000..b5970bb --- /dev/null +++ b/Plugin/unraid/pages/watchdog.php @@ -0,0 +1,370 @@ + + +
+ Watchdog + +
+ +
+
Loading…
+
+ + diff --git a/Plugin/unraid/run_job.sh b/Plugin/unraid/run_job.sh new file mode 100644 index 0000000..351a400 --- /dev/null +++ b/Plugin/unraid/run_job.sh @@ -0,0 +1,99 @@ +#!/bin/bash +# Varaverk job runner — wraps script execution with JSON status tracking. +# Called by /etc/cron.d/varaverk for every scheduled job. +# +# Usage: bash run_job.sh [flags...] +# +# Flags consumed by run_job.sh (stripped before passing to script): +# --manual — marks a UI-triggered run; writes a sentinel on completion so +# the next cron fire is suppressed if it falls within the job's +# own cron interval (prevents double-firing after manual run). +# +# Writes: /var/log/varaverk/.json — status, timestamps, exit code, pid +# /var/log/varaverk/.log — appended per run, trimmed to LOG_MAX_LINES +# /var/log/varaverk/.manual_ts — sentinel: epoch of last manual completion +# +# Status values: running → ok (exit 0) | warn (exit 1) | error (exit 2+) + +JOB_ID="$1" +SCRIPT="$2" +shift 2 + +LOG_DIR="/var/log/varaverk" +BASE="${JOB_ID%.sh}" +LOG_FILE="$LOG_DIR/$BASE.log" +STAT_FILE="$LOG_DIR/$BASE.json" +MANUAL_TS_FILE="$LOG_DIR/$BASE.manual_ts" +LOG_MAX_LINES=1000 + +# Strip --manual from script args — it's for run_job.sh only +MANUAL=false +SCRIPT_ARGS=() +for _arg in "$@"; do + [[ "$_arg" == "--manual" ]] && MANUAL=true || SCRIPT_ARGS+=("$_arg") +done +unset _arg + +mkdir -p "$(dirname "$LOG_FILE")" "$(dirname "$STAT_FILE")" + +# Cron invocation: skip if a manual run completed recently within this job's own interval. +if [[ "$MANUAL" == false && -f "$MANUAL_TS_FILE" ]]; then + LAST_MANUAL=$(cat "$MANUAL_TS_FILE" 2>/dev/null || echo 0) + ELAPSED=$(( $(date +%s) - LAST_MANUAL )) + + # Only suppress for interval-based crons (*/N * * * *). + # Static schedules like "30 2 * * 0" run at their appointed time and are never suppressed. + INTERVAL=0 + SCHEDULE_FILE="/boot/config/plugins/varaverk/schedule.json" + if [[ -f "$SCHEDULE_FILE" ]]; then + CRON_EXPR=$(php -r " + \$s = json_decode(file_get_contents('$SCHEDULE_FILE'), true) ?: []; + echo \$s['$JOB_ID']['cron'] ?? ''; + " 2>/dev/null) + if [[ "$CRON_EXPR" =~ ^\*/([0-9]+) ]]; then + INTERVAL=$(( ${BASH_REMATCH[1]} * 60 )) + fi + fi + + if (( INTERVAL > 0 && ELAPSED < INTERVAL )); then + printf '\n── %s [SKIPPED — ran manually %ds ago, interval %ds] ────────\n' \ + "$(date '+%Y-%m-%d %H:%M:%S')" "$ELAPSED" "$INTERVAL" >> "$LOG_FILE" + printf '{"id":"%s","status":"skipped","manual_elapsed":%s,"interval":%s}\n' \ + "$JOB_ID" "$ELAPSED" "$INTERVAL" > "$STAT_FILE" + rm -f "$MANUAL_TS_FILE" + exit 0 + fi + rm -f "$MANUAL_TS_FILE" +fi + +START=$(date +%s) +printf '{"id":"%s","status":"running","start":%s,"pid":%s}\n' \ + "$JOB_ID" "$START" "$$" > "$STAT_FILE" + +printf '\n── %s ────────────────────────────────────────────────\n' \ + "$(date '+%Y-%m-%d %H:%M:%S')" >> "$LOG_FILE" + +bash "$SCRIPT" "${SCRIPT_ARGS[@]}" >> "$LOG_FILE" 2>&1 +EC=$? + +END=$(date +%s) +if [ "$EC" -eq 0 ]; then STATUS="ok" +elif [ "$EC" -eq 1 ]; then STATUS="warn" +else STATUS="error" +fi + +printf '{"id":"%s","status":"%s","start":%s,"end":%s,"exit":%s}\n' \ + "$JOB_ID" "$STATUS" "$START" "$END" "$EC" > "$STAT_FILE" + +# Write manual sentinel so cron can skip the next fire within the interval +if [[ "$MANUAL" == true ]]; then + echo "$END" > "$MANUAL_TS_FILE" +fi + +# Trim to last LOG_MAX_LINES to prevent unbounded growth +line_count=$(wc -l < "$LOG_FILE" 2>/dev/null || echo 0) +if [ "$line_count" -gt "$LOG_MAX_LINES" ]; then + tail -n "$LOG_MAX_LINES" "$LOG_FILE" > "${LOG_FILE}.tmp" && mv "${LOG_FILE}.tmp" "$LOG_FILE" +fi + +exit $EC diff --git a/Plugin/usr/local/emhttp/plugins/varaverk/api/board.php b/Plugin/usr/local/emhttp/plugins/varaverk/api/board.php deleted file mode 100644 index a263738..0000000 --- a/Plugin/usr/local/emhttp/plugins/varaverk/api/board.php +++ /dev/null @@ -1,88 +0,0 @@ - true]; - -// ── Active locks ────────────────────────────────────────────────────────── -$lockDir = '/tmp/unraid_locks'; -$locks = []; -if (is_dir($lockDir)) { - foreach (glob($lockDir . '/*.lock') ?: [] as $lf) { - $name = basename($lf, '.lock'); - $age = time() - (int)filemtime($lf); - $content = trim(file_get_contents($lf) ?: ''); - // content is "PID:scriptname" — extract PID - $pid = preg_match('/^(\d+)/', $content, $pm) ? $pm[1] : ''; - // Skip if PID is still alive (it's legitimately running) - if ($pid && file_exists("/proc/$pid")) continue; - $locks[] = ['name' => $name, 'file' => basename($lf), 'age' => $age]; - } -} -$out['locks'] = $locks; - -// ── Recent errors ────────────────────────────────────────────────────────── -$errors = []; -if (is_dir(LOG_DIR)) { - $cutoff = time() - 7 * 86400; // only logs touched in last 7 days - foreach (glob(LOG_DIR . '/*.log') ?: [] as $lf) { - if (filemtime($lf) < $cutoff) continue; - $script = basename($lf, '.log'); - $lines = array_slice(@file($lf) ?: [], -200); - $lastErr = null; - foreach (array_reverse($lines) as $raw) { - // Strip ANSI escape codes - $clean = preg_replace('/\033\[[0-9;]*[mK]/', '', rtrim($raw)); - if (!$clean) continue; - if (preg_match('/\[(?:ERROR|WARN|CRITICAL|FAILED)\]/i', $clean) || - preg_match('/\b(?:ERROR|CRITICAL|FAILED):\s/i', $clean) || - str_contains($clean, '✗') || - (str_contains($clean, '⚠') && !str_contains($clean, '♥'))) { - $lastErr = mb_substr($clean, 0, 220); - break; - } - } - if ($lastErr !== null) { - $errors[] = ['script' => $script, 'line' => $lastErr, 'ts' => (int)filemtime($lf)]; - } - } - usort($errors, fn($a, $b) => $b['ts'] - $a['ts']); -} -$out['errors'] = array_slice($errors, 0, 20); - -// ── Partner reachability ─────────────────────────────────────────────────── -$cacheFile = '/tmp/vv_partner_cache.json'; -$cacheTtl = 30; -$partnerData = null; - -if (file_exists($cacheFile) && (time() - (int)filemtime($cacheFile)) < $cacheTtl) { - $partnerData = json_decode(file_get_contents($cacheFile), true); -} else { - $confRaw = vv_read_conf_raw('master.conf'); - preg_match('/^\s*HOST1(?:_NAME)?\s*=\s*["\']?(\S+?)["\']?\s*(?:#.*)?$/m', $confRaw, $m1); - preg_match('/^\s*HOST2(?:_NAME)?\s*=\s*["\']?(\S+?)["\']?\s*(?:#.*)?$/m', $confRaw, $m2); - $host1 = $m1[1] ?? ''; - $host2 = $m2[1] ?? ''; - $mine = vv_get_hostname(); - $partnerHost = null; - if ($mine && $host1 && strcasecmp($mine, $host1) === 0) $partnerHost = $host2; - if ($mine && $host2 && strcasecmp($mine, $host2) === 0) $partnerHost = $host1; - - if ($partnerHost) { - $start = microtime(true); - $result = shell_exec('ping -c1 -W1 ' . escapeshellarg($partnerHost) . ' 2>&1'); - $elapsed = (int)round((microtime(true) - $start) * 1000); - $reached = str_contains((string)$result, '1 received') - || str_contains((string)$result, '1 packets received'); - $partnerData = [ - 'host' => $partnerHost, - 'reachable' => $reached, - 'latency' => $reached ? $elapsed : null, - ]; - @file_put_contents($cacheFile, json_encode($partnerData)); - } -} -$out['partner'] = $partnerData; - -echo json_encode($out); diff --git a/Plugin/usr/local/emhttp/plugins/varaverk/event/disks_mounted/array_start_jobs b/Plugin/usr/local/emhttp/plugins/varaverk/event/disks_mounted/array_start_jobs deleted file mode 100755 index 7351af4..0000000 --- a/Plugin/usr/local/emhttp/plugins/varaverk/event/disks_mounted/array_start_jobs +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -# Varaverk: run array_started.sh if enabled in the schedule (background — non-blocking). -php -r " -require_once '/usr/local/emhttp/plugins/varaverk/include/scheduler.php'; -\$s = vv_schedule_load(); -\$e = \$s['Orchestrators/array_started.sh'] ?? []; -if (empty(\$e['enabled'])) exit(0); -\$runner = '/usr/local/emhttp/plugins/varaverk/run_job.sh'; -\$script = SCRIPTS_DIR . '/Orchestrators/array_started.sh'; -if (!file_exists(\$script)) exit(0); -\$flags = !empty(\$e['log_enabled']) ? ' --log' : ''; -exec('nohup bash ' . escapeshellarg(\$runner) . ' Orchestrators/array_started.sh ' . escapeshellarg(\$script) . \$flags . ' > /dev/null 2>&1 &'); -" 2>/dev/null diff --git a/Plugin/usr/local/emhttp/plugins/varaverk/event/disks_unmounting/array_stop_jobs b/Plugin/usr/local/emhttp/plugins/varaverk/event/disks_unmounting/array_stop_jobs deleted file mode 100755 index c95a51f..0000000 --- a/Plugin/usr/local/emhttp/plugins/varaverk/event/disks_unmounting/array_stop_jobs +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -# Varaverk: run array_stopping.sh if enabled in the schedule (foreground — blocks until done). -php -r " -require_once '/usr/local/emhttp/plugins/varaverk/include/scheduler.php'; -\$s = vv_schedule_load(); -\$e = \$s['Orchestrators/array_stopping.sh'] ?? []; -if (empty(\$e['enabled'])) exit(0); -\$runner = '/usr/local/emhttp/plugins/varaverk/run_job.sh'; -\$script = SCRIPTS_DIR . '/Orchestrators/array_stopping.sh'; -if (!file_exists(\$script)) exit(0); -\$flags = !empty(\$e['log_enabled']) ? ' --log' : ''; -passthru('bash ' . escapeshellarg(\$runner) . ' Orchestrators/array_stopping.sh ' . escapeshellarg(\$script) . \$flags); -" 2>/dev/null diff --git a/Plugin/usr/local/emhttp/plugins/varaverk/run_job.sh b/Plugin/usr/local/emhttp/plugins/varaverk/run_job.sh deleted file mode 100644 index 77518f4..0000000 --- a/Plugin/usr/local/emhttp/plugins/varaverk/run_job.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash -# Varaverk job runner — wraps script execution with JSON status tracking. -# Called by /etc/cron.d/varaverk for every scheduled job. -# -# Usage: bash run_job.sh [flags...] -# -# Writes: /var/log/varaverk/.json — status, timestamps, exit code, pid -# /var/log/varaverk/.log — appended per run, trimmed to LOG_MAX_LINES -# -# Status values: running → ok (exit 0) | warn (exit 1) | error (exit 2+) - -JOB_ID="$1" -SCRIPT="$2" -shift 2 - -LOG_DIR="/var/log/varaverk" -BASE="${JOB_ID%.sh}" -LOG_FILE="$LOG_DIR/$BASE.log" -STAT_FILE="$LOG_DIR/$BASE.json" -LOG_MAX_LINES=1000 - -mkdir -p "$(dirname "$LOG_FILE")" "$(dirname "$STAT_FILE")" - -START=$(date +%s) -printf '{"id":"%s","status":"running","start":%s,"pid":%s}\n' \ - "$JOB_ID" "$START" "$$" > "$STAT_FILE" - -printf '\n── %s ────────────────────────────────────────────────\n' \ - "$(date '+%Y-%m-%d %H:%M:%S')" >> "$LOG_FILE" - -bash "$SCRIPT" "$@" >> "$LOG_FILE" 2>&1 -EC=$? - -END=$(date +%s) -if [ "$EC" -eq 0 ]; then STATUS="ok" -elif [ "$EC" -eq 1 ]; then STATUS="warn" -else STATUS="error" -fi - -printf '{"id":"%s","status":"%s","start":%s,"end":%s,"exit":%s}\n' \ - "$JOB_ID" "$STATUS" "$START" "$END" "$EC" > "$STAT_FILE" - -# Trim to last LOG_MAX_LINES to prevent unbounded growth -line_count=$(wc -l < "$LOG_FILE" 2>/dev/null || echo 0) -if [ "$line_count" -gt "$LOG_MAX_LINES" ]; then - tail -n "$LOG_MAX_LINES" "$LOG_FILE" > "${LOG_FILE}.tmp" && mv "${LOG_FILE}.tmp" "$LOG_FILE" -fi - -exit $EC diff --git a/Transcodes/transcode_cleanup.sh b/Transcodes/transcode_cleanup.sh index a08bfb1..3a92576 100644 --- a/Transcodes/transcode_cleanup.sh +++ b/Transcodes/transcode_cleanup.sh @@ -175,11 +175,11 @@ fi cleanup_location() { local location="$1" label="$2" max_age="$3" - local files_removed=0 bytes_freed=0 files_skipped=0 files_active=0 + local files_removed=0 bytes_freed=0 files_skipped=0 files_active=0 files_streaming=0 if [[ ! -d "$location" ]]; then log "$label does not exist — skipping" - LOCATION_REMOVED=0 LOCATION_FREED="0B" LOCATION_SKIPPED=0 LOCATION_ACTIVE=0 + LOCATION_REMOVED=0 LOCATION_FREED="0B" LOCATION_SKIPPED=0 LOCATION_ACTIVE=0 LOCATION_STREAMING=0 return fi @@ -195,7 +195,8 @@ cleanup_location() { while IFS= read -r open_file; do [[ -n "$open_file" ]] && OPEN_FILES_MAP["$open_file"]=1 done < <(timeout 15 lsof +D "$location" 2>/dev/null | awk 'NR>1 {print $9}' | sort -u) - log "${#OPEN_FILES_MAP[@]} files currently open in $label" + files_streaming=${#OPEN_FILES_MAP[@]} + log "$files_streaming files currently open in $label" fi # Process aged files @@ -247,19 +248,20 @@ cleanup_location() { freed_human="0B" fi - log "$label — removed $files_removed files ($freed_human freed) | active: $files_active | skipped: $files_skipped" + log "$label — removed $files_removed files ($freed_human freed) | streaming: $files_streaming | aged+open: $files_active | skipped: $files_skipped" LOCATION_REMOVED=$files_removed LOCATION_FREED=$freed_human LOCATION_SKIPPED=$files_skipped LOCATION_ACTIVE=$files_active + LOCATION_STREAMING=$files_streaming } # ============================================================================================== # ━━━ Transcode Cleanup ━━━ # ============================================================================================== START=$(date +%s) -TOTAL_REMOVED=0 TOTAL_SKIPPED=0 TOTAL_ACTIVE=0 +TOTAL_REMOVED=0 TOTAL_SKIPPED=0 TOTAL_ACTIVE=0 TOTAL_STREAMING=0 RAMDISK_FREED="0B" SSD_FREED="0B" # Cleanup ramdisk @@ -268,6 +270,7 @@ if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then TOTAL_REMOVED=$(( TOTAL_REMOVED + LOCATION_REMOVED )) TOTAL_SKIPPED=$(( TOTAL_SKIPPED + LOCATION_SKIPPED )) TOTAL_ACTIVE=$(( TOTAL_ACTIVE + LOCATION_ACTIVE )) + TOTAL_STREAMING=$(( TOTAL_STREAMING + LOCATION_STREAMING )) RAMDISK_FREED=$LOCATION_FREED else log "Ramdisk not mounted — skipping ramdisk cleanup" @@ -279,6 +282,7 @@ if [[ -d "$TRANSCODE_SSD" ]]; then TOTAL_REMOVED=$(( TOTAL_REMOVED + LOCATION_REMOVED )) TOTAL_SKIPPED=$(( TOTAL_SKIPPED + LOCATION_SKIPPED )) TOTAL_ACTIVE=$(( TOTAL_ACTIVE + LOCATION_ACTIVE )) + TOTAL_STREAMING=$(( TOTAL_STREAMING + LOCATION_STREAMING )) SSD_FREED=$LOCATION_FREED else log "SSD fallback not found — skipping SSD cleanup" @@ -307,7 +311,8 @@ echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_RAM Ramdisk freed: $RAMDISK_FREED" echo "$ICON_DISK SSD freed: $SSD_FREED" echo "$ICON_TRASH Removed: $TOTAL_REMOVED files" -echo "$ICON_RUNNING Active: $TOTAL_ACTIVE files (open — protected)" +echo "$ICON_RUNNING Streaming: $TOTAL_STREAMING files (open by Emby — untouched)" +echo "$ICON_SHIELD Protected: $TOTAL_ACTIVE aged files saved by open-file check" echo "$ICON_TRASH Skipped: $TOTAL_SKIPPED files" echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" diff --git a/git_pull_execute.sh b/git_pull_execute.sh index 6cb5a43..aea4bff 100755 --- a/git_pull_execute.sh +++ b/git_pull_execute.sh @@ -259,6 +259,50 @@ fi END=$(date +%s) +# ============================================================================================== +# ━━━ Conf Upgrade ━━━ +# ============================================================================================== +# Merges new conf structure into the live conf files after every pull. +# New keys → added with template defaults (user fills in once). +# Removed keys → dropped. Existing values → always preserved. +# Silent when already up to date — no overhead on unchanged pulls. + +echo "" +echo "━━━ $ICON_GEAR Conf Upgrade ━━━" + +UPGRADE_SCRIPT="$TARGET_DIR/Deployment/conf_upgrade.sh" +TMPL_DIR="$TARGET_DIR/Deployment/conf_templates" +CONF_DIR="$TARGET_DIR/Configurations" + +if [[ ! -f "$UPGRADE_SCRIPT" ]]; then + log "conf_upgrade.sh not found — skipping (pre-deployment-folder repo)" +elif [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would upgrade master.conf and ${MY_ID,,}.conf" +elif [[ "$SYNC_SUCCESS" == true ]]; then + _DRY="" + + # master.conf + if [[ -f "$TMPL_DIR/master.conf" && -f "$CONF_DIR/master.conf" ]]; then + bash "$UPGRADE_SCRIPT" \ + --template "$TMPL_DIR/master.conf" \ + --target "$CONF_DIR/master.conf" \ + --backup $_DRY + else + warn "master.conf template or target not found — skipping" + fi + + # This server's host conf only — sparse checkout ensures we have it + HOST_CONF="$CONF_DIR/${MY_ID,,}.conf" + if [[ -f "$TMPL_DIR/host.conf.template" && -f "$HOST_CONF" ]]; then + bash "$UPGRADE_SCRIPT" \ + --template "$TMPL_DIR/host.conf.template" \ + --target "$HOST_CONF" \ + --backup $_DRY + else + warn "${MY_ID,,}.conf or host.conf.template not found — skipping" + fi +fi + # ============================================================================================== # ━━━ Summary ━━━ # ==============================================================================================