Varaverk: FallBack + Watchdog tabs; plugin path restructure to Plugin/unraid/
- FallBack tab: per-node tier inventory + active fallback card with duration, tier, handback strikes, running container status - Watchdog tab: live system health (RAM bar + thresholds, load, uptime, daemon), docker watchdog strikes + skip list + restart history, stability strikes + reboot log, resource pressure alert card, config inventory (mem limits, required, pause/stop lists) - Swapped partnership/arrs tab order; FallBack between partnership and watchdog - Plugin source tree moved from Plugin/usr/local/emhttp/plugins/varaverk/ to Plugin/unraid/ - Deployment/ conf templates added
This commit is contained in:
Gmer4Lfe
@@ -0,0 +1,769 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ========================== HOST1 CONFIGURATION — unRAID-Gmer4Lfe ============================
|
||||
# ==============================================================================================
|
||||
# HOST1-specific variables — credentials, container names, share paths, failover lists.
|
||||
# Sourced after master.conf — values here extend shared profile arrays and add HOST1-specific
|
||||
# identity, credentials, and container configuration.
|
||||
#
|
||||
# Sparse checkout (git) ensures HOST2 never receives this file.
|
||||
# HOST2 never sees HOST1 credentials — clean separation at the file level.
|
||||
#
|
||||
# DO NOT put shared config here — thresholds, toggles, profiles belong in master.conf.
|
||||
# DO NOT put HOST2 variables here — they belong in host2.conf.
|
||||
#
|
||||
# ── INDEX ─────────────────────────────────────────────────────────────────────────────────────
|
||||
#
|
||||
# ── IDENTITY & CONNECTIVITY ────────────────────────────────────────────────────────────────
|
||||
# IDENTITY hostname, SSH key
|
||||
# EMBY container name, URL, API key
|
||||
# NOTIFICATIONS Discord webhook
|
||||
# PARTNERSHIP auth containers, backup paths
|
||||
#
|
||||
# ── RSYNC ──────────────────────────────────────────────────────────────────────────────────
|
||||
# DAILY SYNC SHARES media shares HOST1 owns and pushes to HOST2
|
||||
# WEEKLY SYNC SHARES appdata shares synced weekly (Sunday 2:30am)
|
||||
# CRITICAL SYNC SHARES appdata shares synced every 30 minutes
|
||||
# BACKUP VERIFY shares for checksum verification against remote
|
||||
# HOST1 RSYNC PROFILE host1-appdata profile for HOST1-specific appdata syncs
|
||||
#
|
||||
# ── DOCKER ─────────────────────────────────────────────────────────────────────────────────
|
||||
# DOCKER DAILY RESTART containers restarted daily
|
||||
# DOCKER WEEKLY RESTART containers restarted weekly
|
||||
# DOCKER WATCHDOG memory limits, health URLs, required containers, ignore list
|
||||
# DOCKER NETWORK CONNECT networks and containers for docker_network_connect.sh
|
||||
#
|
||||
# ── FALLBACK ───────────────────────────────────────────────────────────────────────────────
|
||||
# DDNS DDNS containers managed by HOST1
|
||||
# INTERNET LOSS containers stopped when internet is lost
|
||||
# FALLBACK TIERS what HOST1 runs for HOST2 per tier
|
||||
# TIER DELAYS how long HOST1 must be down before each tier activates on HOST2
|
||||
# RSYNC WRITEBACK HOST1 appdata synced back on handback
|
||||
#
|
||||
# ── MEDIA ──────────────────────────────────────────────────────────────────────────────────
|
||||
# MEDIA PERMISSIONS share list for media_shares_permissions.sh
|
||||
# MEDIA CLEANER folder lists for media_cleaner.sh
|
||||
#
|
||||
# ── MONITORS ───────────────────────────────────────────────────────────────────────────────
|
||||
# CERTIFICATE MONITOR domains checked for SSL expiry
|
||||
# SMART HEALTH drives to skip in SMART monitoring
|
||||
# ZFS REPORT pools to exclude from ZFS health report
|
||||
#
|
||||
# ── TRANSCODES ─────────────────────────────────────────────────────────────────────────────
|
||||
# TRANSCODES ramdisk size, thresholds, SSD path, server array
|
||||
#
|
||||
# ── ARR STACK ──────────────────────────────────────────────────────────────────────────────
|
||||
# DOWNLOADERS slskd, SABnzbd, qBittorrent credentials and URLs
|
||||
# LIDARR URL, API key, path map
|
||||
# SONARR URL, API key, path map
|
||||
# RADARR URL, API key, path map
|
||||
# ARR RECOVERY per-arr recovery toggles
|
||||
#
|
||||
# ── SYSTEM WATCHDOG ────────────────────────────────────────────────────────────────────────
|
||||
# SYSTEM WATCHDOG per-host check toggles and NIC configuration
|
||||
#
|
||||
# ── RESOURCE MANAGER ───────────────────────────────────────────────────────────────────────
|
||||
# RESOURCE MANAGER containers paused/stopped under memory pressure
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
# ==============================================================================================
|
||||
# ── IDENTITY & CONNECTIVITY ───────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Identity ━━━
|
||||
# HOST1 hostname lives in master.conf (not a credential — safe for all servers).
|
||||
# SSH key used for all server-to-server operations — rsync, failover container commands.
|
||||
# Must be in /root/.ssh/ and authorised in HOST2's /root/.ssh/authorized_keys.
|
||||
HOST1_SSH_KEY="" # /root/.ssh/your_key_name
|
||||
HOST1_OWNER="" # your Gitea/GitHub username
|
||||
HOST1_OWNER_EMAIL="" # your email
|
||||
|
||||
# ━━━ Emby ━━━
|
||||
# Referenced by transcode_manager.sh, emby_session_report.sh, emby_database_repair.sh,
|
||||
# weekly_sync_maintenance.sh, and HOST1_TRANSCODE_SERVERS below.
|
||||
# API key: Emby Dashboard → API Keys → + New Key
|
||||
HOST1_EMBY_CONTAINER="Emby"
|
||||
HOST1_EMBY_URL="http://localhost:8096"
|
||||
HOST1_EMBY_API_KEY="" # Emby Dashboard → API Keys → + New Key
|
||||
|
||||
# ━━━ Jellyfin ━━━
|
||||
# API key: Jellyfin Dashboard → Administration → API Keys → + New Key
|
||||
HOST1_JELLYFIN_CONTAINER="Jellyfin"
|
||||
HOST1_JELLYFIN_URL="http://localhost:8096"
|
||||
HOST1_JELLYFIN_API_KEY="" # Jellyfin Dashboard → API Keys → + New Key
|
||||
|
||||
# ━━━ Gitea ━━━
|
||||
# Personal access token for gitea_ssh_setup.sh — registers this server's SSH public key
|
||||
# with Gitea so git operations use key auth instead of passwords.
|
||||
# Create in Gitea: Settings → Applications → Generate Token → scope: write:user
|
||||
HOST1_GITEA_API_TOKEN="" # Gitea Settings → Applications → Generate Token
|
||||
|
||||
# ━━━ Notifications ━━━
|
||||
# Discord webhook — leave blank to disable.
|
||||
# Per-host so HOST1 and HOST2 can post to different channels or only one server notifies.
|
||||
HOST1_DISCORD_WEBHOOK="" # Discord channel → Integrations → Webhooks
|
||||
|
||||
# ━━━ Partnership ━━━
|
||||
# HOST1 is always the owner (source of truth) unless --transfer has been run.
|
||||
# See README-Partnership.md and master.conf PARTNERSHIP section for full lifecycle docs.
|
||||
|
||||
# Auth containers reconfigured on onboard/offboard.
|
||||
# Format: "ContainerName|WebUIPort"
|
||||
# On onboard → WebUI pointed at owner's Tailscale IP (mirror clicks NPM, gets owner's auth)
|
||||
# On offboard → WebUI pointed back at localhost
|
||||
HOST1_PARTNERSHIP_AUTH_WEBUIS=(
|
||||
"NginxProxyManager|81"
|
||||
"Lldap-Gmer4Lfe|17170"
|
||||
"Authelia|9091"
|
||||
"Authelia-Secondary|9092"
|
||||
)
|
||||
|
||||
# XML templates (from this server's templates-user/) pushed to mirror during onboard.
|
||||
# These become the mirror's active auth stack, backed by the rsync-synced appdata.
|
||||
# Update filename if Lldap is renamed to drop the host suffix.
|
||||
HOST1_PARTNERSHIP_AUTH_STACK=(
|
||||
# Dependencies first — Mariadb/Redis must be healthy before Authelia starts
|
||||
"my-Mariadb-Authelia.xml"
|
||||
"my-Mariadb-Authelia-Secondary.xml"
|
||||
"my-Redis-Authelia.xml"
|
||||
"my-Redis-Authelia-Secondary.xml"
|
||||
# Auth apps — deployed after their deps are confirmed healthy
|
||||
"my-Authelia.xml"
|
||||
"my-Authelia-Secondary.xml"
|
||||
"my-NginxProxyManager.xml"
|
||||
"my-Lldap-Gmer4Lfe.xml"
|
||||
# Source of truth — must be available on HOST2 independently of the auth stack
|
||||
"my-Gitea.xml"
|
||||
)
|
||||
|
||||
# XML templates pushed to mirror for the arr stack during onboard.
|
||||
# Deps (e.g. databases) first if any — same ordering rule as auth stack.
|
||||
HOST1_PARTNERSHIP_ARR_STACK=(
|
||||
# "my-Sonarr.xml"
|
||||
# "my-Radarr.xml"
|
||||
# "my-Lidarr.xml"
|
||||
# "my-Prowlarr.xml"
|
||||
# "my-Bazarr.xml"
|
||||
)
|
||||
|
||||
# Paths HOST2 should collect during the grace window after offboard.
|
||||
# Notified on offboard — no auto-deletion, HOST2 must collect manually within PARTNERSHIP_GRACE_HOURS.
|
||||
HOST1_PARTNERSHIP_MIRROR_BACKUPS=(
|
||||
# "/mnt/user/appdata-Fallback/Jayred365-Emby"
|
||||
)
|
||||
|
||||
# Containers parked on this server when partnership is active.
|
||||
# Stopped on onboard (owner deploys its stack instead), restarted on offboard.
|
||||
HOST1_PARTNERSHIP_OWN_CONTAINERS=(
|
||||
# "Emby"
|
||||
# "NginxProxyManager"
|
||||
)
|
||||
|
||||
# Emby admin provisioning — toggle is owner-only, credentials are per-host.
|
||||
# Owner enables/disables the feature. Each host sets the account they want on the shared Emby.
|
||||
# On onboard: owner reads mirror's HOST*_PARTNERSHIP_EMBY_ADMIN_* and creates that account.
|
||||
# On offboard: account is deleted. Username collision → onboard exits with error.
|
||||
HOST1_PARTNERSHIP_PROVISION_EMBY_ADMIN=false # owner controls whether Emby is shared
|
||||
HOST1_PARTNERSHIP_EMBY_PORT=8096
|
||||
HOST1_PARTNERSHIP_EMBY_ADMIN_USER="" # this server's desired Emby username
|
||||
HOST1_PARTNERSHIP_EMBY_ADMIN_PASS="" # this server's desired Emby password
|
||||
|
||||
# ==============================================================================================
|
||||
# ── RSYNC ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Daily Sync Shares ━━━
|
||||
# Shares HOST1 pushes to all other nodes every night (1am via daily_sync_maintenance.sh).
|
||||
# Mesh model: every node pushes every media share — no ownership, no mirrors.
|
||||
# arr_sync ensures all arr libraries converge (union). rsync spreads files (additive, no --delete).
|
||||
# arr_cleanup removes true orphans based on local arr state.
|
||||
# Any node can download content to any share — it propagates to all nodes on the next cycle.
|
||||
# Nextcloud is intentionally one-directional (HOST1→HOST2 offsite backup — not arr-managed).
|
||||
# Uses DEFAULT_RSYNC_OPTS from master.conf — no profile needed.
|
||||
# For shares needing container stops or custom options — add a profile in master.conf.
|
||||
HOST1_DAILY_SYNC_SHARES=(
|
||||
/mnt/user/Books
|
||||
/mnt/user/Intros
|
||||
/mnt/user/Kids_Movies
|
||||
/mnt/user/Kids_Tv_Shows
|
||||
/mnt/user/Movies
|
||||
/mnt/user/Music
|
||||
/mnt/user/Music_Videos
|
||||
/mnt/user/Nextcloud
|
||||
/mnt/user/stand-up_comedy
|
||||
/mnt/user/Sports
|
||||
/mnt/user/Tv_Shows
|
||||
/mnt/user/Anime_Shows-Old
|
||||
/mnt/user/Anime_Movies-Old
|
||||
/mnt/user/Anime_Movies
|
||||
/mnt/user/Anime_Shows
|
||||
)
|
||||
|
||||
# Personal encrypted shares — synced for offsite backup, independent of media shares.
|
||||
# ZFS encrypted at dataset level — remote receives encrypted blocks, cannot read content.
|
||||
# See README-Rsync_Setup.md for ZFS encryption setup before uncommenting.
|
||||
HOST1_PERSONAL_SHARES=(
|
||||
# /mnt/user/HOST1-Personal # uncomment after creating encrypted dataset
|
||||
)
|
||||
|
||||
# ━━━ Weekly Sync Shares ━━━
|
||||
# Appdata shares synced during the weekly maintenance window (Sunday 2:30am).
|
||||
# Containers stopped both sides before sync — full clean state guaranteed.
|
||||
# Profiles drive container stops, excludes, and options — configured in master.conf RSYNC section.
|
||||
# Order matters — Emby first (larger transfer), then Critical-Data (auth stack).
|
||||
HOST1_WEEKLY_SYNC_SHARES=(
|
||||
"/mnt/user/Media_Server/Emby" # emby profile — full clean mirror
|
||||
"/mnt/user/appdata-Fallback/Critical-Data" # critical-data profile — auth stack
|
||||
)
|
||||
|
||||
# ━━━ Intermediate Sync Shares ━━━
|
||||
# Shares synced every 4 hours by intermediate_sync_maintenance.sh.
|
||||
# Uses DEFAULT_RSYNC_OPTS (no --delete) — for sub-daily propagation of metadata or watch state.
|
||||
# Full media share sync stays in the daily window. Leave empty to skip mid-day rsync entirely.
|
||||
HOST1_INTERMEDIATE_SYNC_SHARES=(
|
||||
# Add shares here to enable mid-day rsync
|
||||
# Example: "/mnt/user/Emby_Metadata"
|
||||
)
|
||||
|
||||
# ━━━ Critical Sync Shares ━━━
|
||||
# Appdata shares synced every 30 minutes by critical_sync_maintenance.sh.
|
||||
# Format: "/path/to/share" or "/path/to/share|profile-name"
|
||||
# Order matters — Critical-Data first (auth stack), then Emby dirty sync.
|
||||
HOST1_CRITICAL_SYNC_SHARES=(
|
||||
"/mnt/user/appdata-Fallback/Critical-Data|critical-fallback" # auth dirty sync — stays running
|
||||
"/mnt/user/Media_Server/Emby|emby-fallback" # Emby dirty sync — stays running
|
||||
)
|
||||
|
||||
# ━━━ Backup Verify ━━━
|
||||
# Shares verified by backup_verify.sh — random file checksum comparison against remote.
|
||||
# Leave empty to use HOST1_DAILY_SYNC_SHARES automatically.
|
||||
# Sample size and minimum file size defined in master.conf.
|
||||
HOST1_BACKUP_VERIFY_SHARES=(
|
||||
# leave empty to use HOST1_DAILY_SYNC_SHARES automatically
|
||||
)
|
||||
|
||||
# ━━━ HOST1 Rsync Profile — host1-appdata ━━━
|
||||
# HOST1-specific appdata sync profile — extends the shared PROFILE_* arrays in master.conf.
|
||||
# Use for appdata unique to HOST1 (Organizrv2, VaultWarden, UptimeKuma etc.)
|
||||
# Shared appdata (auth stack, Emby) use dedicated profiles defined in master.conf.
|
||||
# Run manually: bash Rsync/rsync.sh /mnt/user/appdata-Fallback/HOST1-Appdata --profile=host1-appdata
|
||||
PROFILE_RSYNC_OPTS[host1-appdata]="-av --info=progress2 --bwlimit=${PROFILE_BW_LIMIT[host1-appdata]:-8000}"
|
||||
PROFILE_BW_LIMIT[host1-appdata]=8000
|
||||
PROFILE_RETRY_COUNT[host1-appdata]=3
|
||||
PROFILE_SLEEP[host1-appdata]=300
|
||||
PROFILE_CRITICAL_CONTAINER_NAMES[host1-appdata]="Organizrv2-Gmer4Lfe UptimeKuma-Gmer4Lfe VaultWarden-Gmer4Lfe"
|
||||
PROFILE_DELAYED_CONTAINERS[host1-appdata]=""
|
||||
PROFILE_CONTAINER_DELAY[host1-appdata]=5
|
||||
PROFILE_EXCLUDE_DIRS[host1-appdata]="logs *.tmp"
|
||||
|
||||
# ==============================================================================================
|
||||
# ── DOCKER ────────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Docker Daily Restart ━━━
|
||||
# Containers restarted every day via DAILY_MAINTENANCE_SCRIPTS.
|
||||
# Dispatcharr degrades over time without restart — daily is intentional, not just housekeeping.
|
||||
# Order matters — auth stack first, then media services.
|
||||
HOST1_DAILY_RESTART_CONTAINERS=(
|
||||
"NginxProxyManager"
|
||||
"Lldap-Gmer4Lfe"
|
||||
"Authelia"
|
||||
"Authelia-Secondary"
|
||||
"Dispatcharr-Iptv-Users"
|
||||
"Dispatcharr" # Live TV scheduler — degrades without daily restart
|
||||
"Dispatcharr-Basic"
|
||||
"ErsatzTV-Emby"
|
||||
"Slskd" # Soulseek connection drops after extended uptime; restart refreshes share index
|
||||
)
|
||||
|
||||
# ━━━ Docker Weekly Restart ━━━
|
||||
# Less critical services restarted weekly via WEEKLY_MAINTENANCE_SCRIPTS (Sunday 2:30am).
|
||||
# Containers already stopped for weekly sync — restart adds zero extra downtime.
|
||||
HOST1_WEEKLY_RESTART_CONTAINERS=(
|
||||
"NextCloud"
|
||||
"Organizrv2-Gmer4Lfe"
|
||||
"AdGuard-Home"
|
||||
"Immich-Gmer4Lfe"
|
||||
)
|
||||
|
||||
# ━━━ Docker Watchdog ━━━
|
||||
# Per-HOST1 container configuration for docker_watchdog.sh.
|
||||
# Shared thresholds and toggles live in master.conf.
|
||||
|
||||
# Memory hard limits in MB — immediate restart if exceeded.
|
||||
# Set at "container is clearly broken" not "container is busy".
|
||||
# 20GB=20480 18GB=18432 16GB=16384 12GB=12288 8GB=8192 4GB=4096 2GB=2048 1GB=1024
|
||||
declare -A HOST1_WATCHDOG_CONTAINERS=(
|
||||
["Emby"]=18432 # 18GB — large library + active transcodes
|
||||
["LidaTube"]=6144 # 6GB — memory leak over time
|
||||
["Tdarr"]=6144 # 6GB — encoding is memory intensive
|
||||
["Code-Server"]=1024 # 1GB — should never need more
|
||||
)
|
||||
|
||||
# HTTP health check URLs — checked every cycle, strike system before restart.
|
||||
# Only add containers with a meaningful web interface to check.
|
||||
declare -A HOST1_WATCHDOG_CONTAINER_URLS=(
|
||||
["Emby"]="http://localhost:8096"
|
||||
)
|
||||
|
||||
# Required containers — must always be running on HOST1.
|
||||
# Strike system before restart — repeated failures go on skip list, auto-clears on recovery.
|
||||
# Listed in dependency order — dependencies before dependents.
|
||||
HOST1_WATCHDOG_REQUIRED_CONTAINERS=(
|
||||
"NginxProxyManager"
|
||||
"Lldap-Gmer4Lfe"
|
||||
"Mariadb-Authelia"
|
||||
"Mariadb-Authelia-Secondary"
|
||||
"Redis-Authelia"
|
||||
"Redis-Authelia-Secondary"
|
||||
"Authelia"
|
||||
"Authelia-Secondary"
|
||||
)
|
||||
|
||||
# Containers to skip in Tier 2 global scan — legitimately stopped or frequently restarting.
|
||||
# Watchdog leaves these alone entirely — no restart attempts, no crash loop tracking.
|
||||
HOST1_WATCHDOG_SCAN_IGNORE=(
|
||||
"DashGate"
|
||||
"PIA-WG-Config-Generator"
|
||||
"Aperture"
|
||||
"Aperture-Kids"
|
||||
"pgvector-18-Apeture-Kids"
|
||||
"Pgvector18-Aperture"
|
||||
)
|
||||
|
||||
# Dependency ordering — skip restarting a container if its dependency is also down.
|
||||
# Prevents watchdog from restarting Authelia before Mariadb is back up.
|
||||
# SPACE-SEPARATED STRINGS — converted to array at runtime.
|
||||
declare -A HOST1_WATCHDOG_DEPENDENCIES=(
|
||||
["Authelia"]="Mariadb-Authelia Redis-Authelia"
|
||||
["Authelia-Secondary"]="Mariadb-Authelia Redis-Authelia-Secondary"
|
||||
["NextCloud"]="Postgres-NextCloud"
|
||||
)
|
||||
|
||||
# Per-container appdata growth suppress ceilings in MB.
|
||||
# ONLY needed in specific cases — growth rate detection covers all containers automatically.
|
||||
# Use this when a container legitimately has large stable data and you want to guarantee
|
||||
# it never triggers a false-positive growth alert. Growth warnings are suppressed while the
|
||||
# container's dir stays below this ceiling; above it, warnings resume as normal.
|
||||
# 50GB=51200 25GB=25600 20GB=20480 15GB=15360 10GB=10240 5GB=5120
|
||||
declare -A HOST1_WATCHDOG_APPDATA_SIZES=(
|
||||
["Tdarr"]="25600" # 25GB — transcode cache grows legitimately during active jobs
|
||||
["7dtd"]="20480" # 20GB — game server world data, expected to be large
|
||||
)
|
||||
|
||||
# ━━━ Network Watchdog ━━━
|
||||
# Host-specific connectivity config for Watchdogs/System/network_watchdog.sh.
|
||||
HOST1_NETWORK_WATCHDOG_DDNS_DOMAIN="gmer4lfe.com"
|
||||
HOST1_NETWORK_WATCHDOG_DDNS_CONTAINER="Gmer4Lfe.com"
|
||||
HOST1_NETWORK_WATCHDOG_NPM_URL="https://gmer4lfe.com"
|
||||
|
||||
# ━━━ Docker Network Connect ━━━
|
||||
# Containers connected to custom networks at array start by docker_network_connect.sh.
|
||||
# Networks created if they don't exist — idempotent, safe to re-run.
|
||||
HOST1_NETWORK_CONNECT_CONTAINERS=(
|
||||
"memcached"
|
||||
"Npm-CrowdSec"
|
||||
)
|
||||
|
||||
HOST1_NETWORK_CONNECT_NETWORKS=(
|
||||
"high-availability"
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── FALLBACK ──────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ DDNS ━━━
|
||||
# DDNS containers HOST1 manages — started/stopped by fallback.sh per DDNS absolute rules:
|
||||
# Internet loss → stop immediately
|
||||
# Failover → HOST2 starts HOST1's DDNS as Tier 1 (before any other containers)
|
||||
# Handback → stop HOST1's DDNS on HOST2 → rsync → start containers → start local DDNS last
|
||||
HOST1_DDNS_CONTAINERS=(
|
||||
"Gmer4Lfe.com"
|
||||
)
|
||||
|
||||
# ━━━ Internet Loss ━━━
|
||||
# Containers stopped immediately on HOST1 when internet connection is lost.
|
||||
# Prevents external-facing services from operating without connectivity.
|
||||
FALLBACK_HOST1_STOP_ON_NO_NET=(
|
||||
"Gmer4Lfe.com"
|
||||
)
|
||||
|
||||
# ━━━ Fallback Tiers — HOST1 Runs for HOST2 ━━━
|
||||
# Containers HOST1 starts when HOST2 goes down.
|
||||
# Tier 1 is always immediate — vital services cannot wait.
|
||||
# Higher tiers activate after HOST2_TIER*_DELAY minutes (set in host2.conf).
|
||||
FALLBACK_HOST1_COVERS_HOST2_TIER1=(
|
||||
"Gmer4Lfe.us"
|
||||
"VaultWarden-Jayred365"
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_COVERS_HOST2_TIER2=(
|
||||
# "container-placeholder"
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_COVERS_HOST2_TIER3=(
|
||||
# "container-placeholder"
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_COVERS_HOST2_TIER4=(
|
||||
# "container-placeholder"
|
||||
)
|
||||
|
||||
# ━━━ Tier Delays — HOST1's Containers on HOST2 ━━━
|
||||
# How long HOST1 must be down before each tier activates on HOST2 — in minutes.
|
||||
# Tier 1 is always immediate — no delay var needed.
|
||||
HOST1_TIER2_DELAY=240 # 4 hours — NextCloud, Immich
|
||||
HOST1_TIER3_DELAY=720 # 12 hours — secondary services
|
||||
HOST1_TIER4_DELAY=1440 # 24 hours — arrs + downloaders
|
||||
|
||||
# ━━━ Rsync Writeback — HOST1 Appdata Back on Handback ━━━
|
||||
# Syncs HOST1 appdata BACK to HOST1 when it comes back online after a failover.
|
||||
# Containers stopped before writeback — clean source, no competing writes.
|
||||
#
|
||||
# HOST1_TIER1_WRITEBACK_DELAY: short outages skip Tier 1 writeback — primary state
|
||||
# is more reliable than dirty sync data for brief outages.
|
||||
HOST1_TIER1_WRITEBACK_DELAY=60 # skip Emby writeback if outage under 1hr
|
||||
|
||||
# Tier 4 automatically syncs HOST1_DAILY_SYNC_SHARES — only list paths NOT in that array.
|
||||
FALLBACK_HOST1_WRITEBACK_TIER1=(
|
||||
"/mnt/user/Media_Server/Emby" # watch states built up during outage
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_WRITEBACK_TIER2=(
|
||||
"/mnt/user/appdata-Fallback/Important-Data" # NextCloud + Postgres — files added during outage
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_WRITEBACK_TIER3=(
|
||||
# "location-placeholder"
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_WRITEBACK_TIER4=(
|
||||
"/mnt/user/appdata-Fallback/Arrs_Stack" # arr databases — downloads queued during outage
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── MEDIA ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Media Permissions ━━━
|
||||
# Shares that media_shares_permissions.sh applies PERMISSIONS_MODE and PERMISSIONS_OWNER to.
|
||||
# Runs first in DAILY_MAINTENANCE_SCRIPTS — arr cleanup depends on correct ownership.
|
||||
HOST1_MEDIA_PERMISSION_SHARES=(
|
||||
/mnt/user/Anime_Movies
|
||||
/mnt/user/Anime_Movies-Old
|
||||
/mnt/user/Anime_Shows
|
||||
/mnt/user/Anime_Shows-Old
|
||||
/mnt/user/appcache
|
||||
/mnt/user/Books
|
||||
/mnt/user/Downloads
|
||||
/mnt/user/Games
|
||||
/mnt/user/Intros
|
||||
/mnt/user/Kids_Movies
|
||||
/mnt/user/Kids_Tv_Shows
|
||||
/mnt/user/Movie_Recordings
|
||||
/mnt/user/Movies
|
||||
/mnt/user/Music
|
||||
/mnt/user/Music_Videos
|
||||
/mnt/user/Photo
|
||||
/mnt/user/Sports
|
||||
/mnt/user/stand-up_comedy
|
||||
/mnt/user/Temp_Storage
|
||||
/mnt/user/Tv_Recordings
|
||||
/mnt/user/Tv_Shows
|
||||
/mnt/user/YouTube
|
||||
)
|
||||
|
||||
# ━━━ Media Cleaner ━━━
|
||||
# Folder lists for media_cleaner.sh — two profiles: anime and media.
|
||||
# File patterns shared across all servers — defined in master.conf.
|
||||
# Called via DAILY_MAINTENANCE_SCRIPTS. Run manually: Media/media_cleaner.sh anime|media
|
||||
HOST1_ANIME_CLEAN_FOLDERS=(
|
||||
/mnt/user/Anime_Movies
|
||||
/mnt/user/Anime_Movies-Old
|
||||
/mnt/user/Anime_Shows
|
||||
/mnt/user/Anime_Shows-Old
|
||||
)
|
||||
|
||||
HOST1_MEDIA_CLEAN_FOLDERS=(
|
||||
/mnt/user/Kids_Movies
|
||||
/mnt/user/Kids_Tv_Shows
|
||||
/mnt/user/Movies
|
||||
/mnt/user/Music
|
||||
/mnt/user/Sports
|
||||
/mnt/user/stand-up_comedy
|
||||
/mnt/user/Tv_Shows
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── MONITORS ──────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Certificate Monitor ━━━
|
||||
# Domains checked via direct openssl connection — not relying on NPM's certificate state.
|
||||
# Checks the actual certificate served, not what NPM thinks it has.
|
||||
# Thresholds (CERT_WARN_DAYS, CERT_CRIT_DAYS) defined in master.conf.
|
||||
HOST1_CERT_MONITOR_DOMAINS=(
|
||||
"Gmer4Lfe.com"
|
||||
"Gmer4Lfe.us"
|
||||
)
|
||||
|
||||
# ━━━ SMART Health ━━━
|
||||
# Drives skipped in SMART attribute monitoring — hardware is server-specific.
|
||||
# Thresholds read from dynamix.cfg at runtime — fallbacks in master.conf.
|
||||
HOST1_SMART_IGNORE_DRIVES=(
|
||||
"sda" # boot USB — SMART not meaningful on flash drives
|
||||
)
|
||||
|
||||
# ━━━ ZFS Report ━━━
|
||||
# Pools excluded from the weekly ZFS health report — reduces noise from single-disk array pools.
|
||||
# These are individual array disks formatted as ZFS — converting to XFS over time via unBalance.
|
||||
# Pool health thresholds defined in master.conf.
|
||||
HOST1_ZFS_REPORT_IGNORE_POOLS=(
|
||||
"disk5"
|
||||
"disk6"
|
||||
"disk8"
|
||||
"disk9"
|
||||
"disk10"
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── TRANSCODES ────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# Ramdisk size ceiling — tmpfs only uses RAM actually needed, not the full size upfront.
|
||||
# Real-world: 9 streams peaked at ~5.5GB — 10G gives generous headroom on 128GB RAM.
|
||||
HOST1_RAMDISK_SIZE="10G"
|
||||
|
||||
# Usage thresholds — coupled to HOST1_RAMDISK_SIZE, adjust all three together if size changes.
|
||||
# Hysteresis gap (8.5 - 7 = 1.5GB) prevents flip-flop between ramdisk and SSD.
|
||||
HOST1_RAMDISK_WARN_GB=8.5 # flip to SSD when ramdisk usage reaches this
|
||||
HOST1_RAMDISK_LOW_GB=7 # flip back to ramdisk when usage drops to this
|
||||
|
||||
# SSD fallback path — where transcodes land when ramdisk exceeds HOST1_RAMDISK_WARN_GB.
|
||||
# Must be on cache pool — array disks too slow for active transcode writes.
|
||||
HOST1_TRANSCODE_SSD="/mnt/cache/Temp_Storage/Emby/Transcodes/"
|
||||
|
||||
# Media servers sharing the ramdisk transcode space on HOST1.
|
||||
# Format: "ContainerName|URL|APIKey|Type" — Type: emby | jellyfin | plex
|
||||
# Entries with placeholder API keys are skipped automatically.
|
||||
# ⚠️ Tdarr does NOT belong here — keep Tdarr on SSD, not ramdisk.
|
||||
HOST1_TRANSCODE_SERVERS=(
|
||||
"${HOST1_EMBY_CONTAINER}|${HOST1_EMBY_URL}|${HOST1_EMBY_API_KEY}|emby"
|
||||
"${HOST1_JELLYFIN_CONTAINER}|${HOST1_JELLYFIN_URL}|${HOST1_JELLYFIN_API_KEY}|jellyfin"
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── ARR STACK ─────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Used by arr cleanup scripts and arrs_failed_stalled_recovery.sh.
|
||||
# detect_hosts() selects HOST1 vars when running on HOST1.
|
||||
#
|
||||
# PATH MAPS — container path → host path translation.
|
||||
# Arr stores file paths using container-internal paths — scripts need host paths to scan.
|
||||
# Add one entry per root folder in arr Settings → Media Management → Root Folders.
|
||||
|
||||
# ━━━ Downloaders ━━━
|
||||
# Used by downloaders_reset.sh — runs every 30min via CRITICAL_MAINTENANCE_SCRIPTS.
|
||||
# Clears stuck states, purges old history, prepares each client for a clean cycle.
|
||||
|
||||
# slskd — clears stuck searches, dead transfers, purges expired failed imports.
|
||||
# SLSKD_FAILED_IMPORTS_DIR: where Soularr moves albums Lidarr rejected.
|
||||
HOST1_SLSKD_URL="http://localhost:8980"
|
||||
HOST1_SLSKD_API_KEY="" # slskd Settings → API key
|
||||
HOST1_SLSKD_FAILED_IMPORTS_DIR="/mnt/user/Temp_Storage/Slskd/completed/failed_imports"
|
||||
|
||||
# SABnzbd
|
||||
HOST1_SABNZBD_URL="http://localhost:8180"
|
||||
HOST1_SABNZBD_API_KEY="" # SABnzbd Config → General → API Key
|
||||
|
||||
# qBittorrent — deleteFiles=false removes torrent from qBit but leaves files on disk.
|
||||
# Radarr/Sonarr manage actual files independently.
|
||||
HOST1_QBIT_URL="http://localhost:8080"
|
||||
HOST1_QBIT_USERNAME="root"
|
||||
HOST1_QBIT_PASSWORD="" # qBittorrent WebUI password
|
||||
|
||||
# ━━━ Lidarr — HOST1 only ━━━
|
||||
# HOST2 does not run Lidarr — HOST1_LIDARR_RECOVERY flag handles the exit cleanly.
|
||||
HOST1_LIDARR_URL="http://localhost:8686"
|
||||
HOST1_LIDARR_API_KEY="" # Lidarr Settings → General → API Key
|
||||
HOST1_LIDARR_MUSIC_ROOT="/mnt/user/Music-New"
|
||||
HOST1_FANART_API_KEY="" # fanart.tv account → API Key
|
||||
HOST1_LASTFM_API_KEY="" # last.fm API → Create API Account
|
||||
|
||||
declare -A HOST1_LIDARR_PATH_MAP=(
|
||||
["/ext-music"]="/mnt/user/Music-New"
|
||||
)
|
||||
|
||||
# ━━━ Sonarr ━━━
|
||||
HOST1_SONARR_URL="http://localhost:8989"
|
||||
HOST1_SONARR_API_KEY="" # Sonarr Settings → General → API Key
|
||||
HOST1_SONARR_TV_ROOT="/mnt/user/Tv_Shows"
|
||||
|
||||
# Note: stand-up_comedy in both Sonarr + Radarr — TV specials and movie specials, one folder
|
||||
declare -A HOST1_SONARR_PATH_MAP=(
|
||||
["/tv"]="/mnt/user/Tv_Shows"
|
||||
["/ext-standup-comedy"]="/mnt/user/stand-up_comedy"
|
||||
["/kids tv"]="/mnt/user/Kids_Tv_Shows"
|
||||
["/ext-anime-shows"]="/mnt/user/Anime_Shows-Old"
|
||||
)
|
||||
|
||||
# ━━━ Radarr ━━━
|
||||
HOST1_RADARR_URL="http://localhost:7878"
|
||||
HOST1_RADARR_API_KEY="" # Radarr Settings → General → API Key
|
||||
HOST1_TMDB_API_KEY="" # themoviedb.org → Settings → API
|
||||
HOST1_RADARR_MOVIES_ROOT="/mnt/user/Movies"
|
||||
|
||||
# Note: stand-up_comedy in both Radarr + Sonarr — movie specials and TV specials, one folder
|
||||
declare -A HOST1_RADARR_PATH_MAP=(
|
||||
["/movies"]="/mnt/user/Movies"
|
||||
["/kids movies"]="/mnt/user/Kids_Movies"
|
||||
["/ext-stand-up-comedy"]="/mnt/user/stand-up_comedy"
|
||||
["/anime-movies"]="/mnt/user/Anime_Movies-Old"
|
||||
)
|
||||
|
||||
# ━━━ Arr Recovery Toggles ━━━
|
||||
# false = skip that arr on this host — exits cleanly without error
|
||||
HOST1_SONARR_RECOVERY=true
|
||||
HOST1_RADARR_RECOVERY=true
|
||||
HOST1_LIDARR_RECOVERY=true # HOST1 only — exits cleanly on HOST2
|
||||
|
||||
# ==============================================================================================
|
||||
# ── SYSTEM WATCHDOG ───────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Per-host check toggles and NIC config for system_watchdog.sh.
|
||||
# Aliased by detect_hosts() — script uses unprefixed SYS_WATCHDOG_* names.
|
||||
# HOST1: TR1950X 128GB — full media server, active transcoding, ZFS cache pools.
|
||||
#
|
||||
# Three-tier response — all critical checks enabled by default on HOST1:
|
||||
# Tier 1 (bypass strikes, reboot now): docker daemon, rootfs full, kernel oops, FD, /boot
|
||||
# Tier 2 (bypass strikes with OOM): RAM critical + OOM kills in cycle
|
||||
# Tier 3 (standard strike system): everything else
|
||||
#
|
||||
# RAM tiers, OOM limits, and reboot loop settings in master.conf System Watchdog section.
|
||||
|
||||
# ━━━ Primary NIC ━━━
|
||||
# Network interface for NIC state check — verify with: ip link show | grep "^[0-9]"
|
||||
# Common values: eth0, bond0, br0, eno1
|
||||
HOST1_SYS_WATCHDOG_NIC="eth0"
|
||||
|
||||
# ━━━ Tier 1 — Critical Checks ━━━
|
||||
# These bypass the strike system — a single hit triggers immediate reboot.
|
||||
# Disabling any of these is not recommended — they protect against acute system failure.
|
||||
|
||||
# Docker daemon unresponsive → try restart, reboot if restart fails.
|
||||
# Without a working daemon docker_watchdog.sh is blind and containers cannot be managed.
|
||||
HOST1_SYS_WATCHDOG_CHECK_DOCKER_DAEMON=true
|
||||
|
||||
# rootfs at critical threshold (ROOTFS_CRITICAL_PCT=99) → reboot immediately.
|
||||
# At 99% rootfs writes fail silently — logs stop, Docker errors out, SSH may stop working.
|
||||
# Standard 95% threshold still uses strike system — only 99%+ is critical tier.
|
||||
HOST1_SYS_WATCHDOG_CHECK_ROOTFS=true
|
||||
|
||||
# Kernel BUG/Oops in dmesg delta since last cycle → reboot immediately.
|
||||
# A kernel oops means the kernel ran with a corrupted state — stability is not guaranteed.
|
||||
HOST1_SYS_WATCHDOG_CHECK_KERNEL_OOPS=true
|
||||
|
||||
# File descriptor exhaustion at FD_CRITICAL_PCT (95%) → reboot immediately.
|
||||
# At 95% FD: new connections fail, Docker can't spawn processes, SSH drops.
|
||||
HOST1_SYS_WATCHDOG_CHECK_FD=true
|
||||
|
||||
# /boot read-only detected → reboot immediately.
|
||||
# Unexpected read-only /boot means state files and config writes are silently failing.
|
||||
# Fallback state, watchdog reboot log, and lock files all go stale silently.
|
||||
HOST1_SYS_WATCHDOG_CHECK_BOOT=true
|
||||
|
||||
# ━━━ Tier 2 — Urgent OOM Check ━━━
|
||||
# Bypass strikes when RAM is critically low AND OOM kill rate confirms active crisis.
|
||||
# Both must be enabled for Tier 2 bypass to function — disable either to always use strikes.
|
||||
|
||||
# Track kernel OOM kills each cycle via /proc/vmstat oom_kill delta.
|
||||
# Also provides diagnostic context in reboot messages (which processes were killed).
|
||||
HOST1_SYS_WATCHDOG_CHECK_OOM=true
|
||||
|
||||
# Free RAM check — required for both Tier 2 bypass and RAM tier logic.
|
||||
# Tiers: MEM_WARN_GB(10) → notify | MEM_SHUTDOWN_GB(6) → stop containers | MEM_GB(4) → strikes
|
||||
HOST1_SYS_WATCHDOG_CHECK_RAM=true
|
||||
|
||||
# ━━━ Tier 3 — Standard Checks (strike system) ━━━
|
||||
# Each check must fail SYS_WATCHDOG_STRIKE_LIMIT consecutive cycles before action is taken.
|
||||
# Single spikes are ignored — sustained problems trigger reboot.
|
||||
|
||||
# /var/log filesystem usage above SYS_WATCHDOG_LOG_PCT.
|
||||
# Log spam (Docker log storms, syslog loops) fills rootfs — indicates something broken.
|
||||
HOST1_SYS_WATCHDOG_CHECK_LOG=true
|
||||
|
||||
# ZFS ARC memory pinned above SYS_WATCHDOG_ARC_PINNED_PCT after cache drop.
|
||||
# Enabled on HOST1 — ZFS cache pools actively used. Disable on hosts without ZFS.
|
||||
HOST1_SYS_WATCHDOG_CHECK_ARC=true
|
||||
|
||||
# CPU temperature above SYS_WATCHDOG_CPU_TEMP_MAX (95°C).
|
||||
# Sustained high temp causes kernel throttling or panic. Requires lm-sensors.
|
||||
HOST1_SYS_WATCHDOG_CHECK_CPU_TEMP=true
|
||||
|
||||
# Load average above SYS_WATCHDOG_LOAD_MULTIPLIER × core count.
|
||||
# DISABLED on HOST1 — Tdarr and Emby cause legitimate sustained load spikes during encoding.
|
||||
# Enable on idle servers or adjust SYS_WATCHDOG_LOAD_MULTIPLIER if load is always high.
|
||||
HOST1_SYS_WATCHDOG_CHECK_LOAD=false
|
||||
|
||||
# Zombie process count above SYS_WATCHDOG_ZOMBIE_LIMIT (50).
|
||||
# Large zombie counts indicate serious process management failure — something is stuck.
|
||||
HOST1_SYS_WATCHDOG_CHECK_ZOMBIES=true
|
||||
|
||||
# Check docker_watchdog.sh persistent skip list — required containers on skip list.
|
||||
# Cross-watchdog coordination: if docker_watchdog gave up, system_watchdog escalates.
|
||||
# ENABLED — HOST1 fully built and operational, skip list is meaningful.
|
||||
HOST1_SYS_WATCHDOG_CHECK_CONTAINERS=true
|
||||
|
||||
# /tmp filesystem usage above SYS_WATCHDOG_TMP_PCT with auto-clear attempt.
|
||||
# Script tries to clear aged /tmp files first — only strikes if clear fails.
|
||||
# Lock files, rsync temp files, and Docker ops use /tmp — 100% means lock failures.
|
||||
HOST1_SYS_WATCHDOG_CHECK_TMP=true
|
||||
|
||||
# Array disk error count delta in /proc/mdstat — accumulating errors = disk failing now.
|
||||
# Triggers on SYS_WATCHDOG_MDSTAT_ERROR_LIMIT new errors in one cycle.
|
||||
HOST1_SYS_WATCHDOG_CHECK_MDSTAT=true
|
||||
|
||||
# Primary NIC operstate — detects NIC going down (physical or driver failure).
|
||||
# Uses HOST1_SYS_WATCHDOG_NIC above. Strike system — brief flaps don't trigger reboot.
|
||||
HOST1_SYS_WATCHDOG_CHECK_NETWORK=true
|
||||
|
||||
# sshd running check — attempts restart before escalating.
|
||||
# sshd down = no remote access. Script tries rc.sshd start, notifies, strikes on failure.
|
||||
HOST1_SYS_WATCHDOG_CHECK_SSHD=true
|
||||
|
||||
# Runaway process detection — single process above SYS_WATCHDOG_RUNAWAY_CPU_PCT sustained.
|
||||
# DISABLED — Tdarr encoding and Emby transcoding legitimately peg CPU for extended periods.
|
||||
# Enable only if HOST1 has no CPU-intensive workloads.
|
||||
HOST1_SYS_WATCHDOG_CHECK_RUNAWAY=false
|
||||
|
||||
# ==============================================================================================
|
||||
# ── RESOURCE MANAGER ──────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Containers to manage under pressure — see master.conf RW_CRITICAL_CONTAINERS for exclusions.
|
||||
|
||||
# docker pause at medium pressure (RAM < RW_RAM_MEDIUM_GB or load > medium threshold)
|
||||
# Suspended in-place — instant to pause/unpause, no state lost, no restart delay.
|
||||
HOST1_RW_PAUSE_CONTAINERS=(
|
||||
"Huntarr" # arr search automation — safe to suspend
|
||||
"Cleanuparr" # download cleanup — safe to suspend
|
||||
"Healarr" # arr health checks — safe to suspend
|
||||
"Soularr" # Slskd automation — background only
|
||||
"ChannelTube" # YouTube archiver — background only
|
||||
"Pinchflat" # YouTube archiver — background only
|
||||
)
|
||||
|
||||
# docker stop at hard pressure (RAM < RW_RAM_HARD_GB)
|
||||
# Full stop — these are optional/heavy services that free significant RAM when stopped.
|
||||
# resource_watchdog.sh restarts them when pressure fully clears (RAM >= RW_RAM_RECOVER_GB).
|
||||
HOST1_RW_STOP_CONTAINERS=(
|
||||
"LocalAI" # GPU/CPU heavy — largest RAM consumer when idle
|
||||
"7DaysToDie" # game server — optional
|
||||
"V-Rising" # game server — optional
|
||||
"Code-Server" # IDE — not needed during pressure events
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ──────────────────────── End Of HOST1 Variables ──────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
File diff suppressed because it is too large
Load Diff
Executable
+200
@@ -0,0 +1,200 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# conf_upgrade.sh — Merge new conf structure into an existing conf while preserving user values.
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Usage:
|
||||
# conf_upgrade.sh --template <template.conf> --target <target.conf> [--dry-run] [--backup]
|
||||
#
|
||||
# What it does:
|
||||
# • Keys in template only → ADDED with template's placeholder/default — user fills in once
|
||||
# • Keys in target only → REMOVED — deprecated in new version
|
||||
# • Keys in both → KEPT — target's value always preserved, template's ignored
|
||||
# • Comments / blank lines → always from template (structure follows new version)
|
||||
#
|
||||
# Supports all conf variable patterns:
|
||||
# Simple scalar: KEY="value"
|
||||
# Indexed array: KEY=(\n "item"\n)
|
||||
# Associative array: declare -A KEY=(\n ["k"]="v"\n)
|
||||
#
|
||||
# Options:
|
||||
# --template New version conf file (source of structure and defaults)
|
||||
# --target Existing user conf (source of real values — preserved)
|
||||
# --dry-run Show what would change without writing
|
||||
# --backup Write a .bak copy of target before modifying
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
# ── Arguments ────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
TEMPLATE=""
|
||||
TARGET=""
|
||||
DRY_RUN=false
|
||||
BACKUP=false
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--template) TEMPLATE="$2"; shift 2 ;;
|
||||
--target) TARGET="$2"; shift 2 ;;
|
||||
--dry-run) DRY_RUN=true; shift ;;
|
||||
--backup) BACKUP=true; shift ;;
|
||||
*) echo "Unknown option: $1" >&2; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
[[ -z "$TEMPLATE" ]] && { echo "Error: --template required" >&2; exit 1; }
|
||||
[[ -z "$TARGET" ]] && { echo "Error: --target required" >&2; exit 1; }
|
||||
[[ -f "$TEMPLATE" ]] || { echo "Error: template not found: $TEMPLATE" >&2; exit 1; }
|
||||
[[ -f "$TARGET" ]] || { echo "Error: target not found: $TARGET" >&2; exit 1; }
|
||||
|
||||
# ── Parse target → KEY → full definition block ───────────────────────────────────────────────
|
||||
|
||||
declare -A HOST_MAP # KEY → complete definition line(s) from user's conf
|
||||
|
||||
_parse_target() {
|
||||
local in_block=false cur_key="" cur_block="" line
|
||||
|
||||
while IFS= read -r line || [[ -n "$line" ]]; do
|
||||
if [[ "$in_block" == true ]]; then
|
||||
cur_block+="$line"$'\n'
|
||||
# Closing ) — optional trailing whitespace and comment
|
||||
if [[ "$line" =~ ^[[:space:]]*\)[[:space:]]*(#.*)?$ ]]; then
|
||||
HOST_MAP["$cur_key"]="$cur_block"
|
||||
in_block=false; cur_key=""; cur_block=""
|
||||
fi
|
||||
else
|
||||
# declare -A KEY=(
|
||||
if [[ "$line" =~ ^[[:space:]]*declare[[:space:]]+-[a-zA-Z]+[[:space:]]+([A-Z0-9_]+)[[:space:]]*=\( ]]; then
|
||||
cur_key="${BASH_REMATCH[1]}"; in_block=true; cur_block="$line"$'\n'
|
||||
# KEY=(
|
||||
elif [[ "$line" =~ ^[[:space:]]*([A-Z0-9_]+)[[:space:]]*=\( ]]; then
|
||||
cur_key="${BASH_REMATCH[1]}"; in_block=true; cur_block="$line"$'\n'
|
||||
# KEY=value (simple scalar)
|
||||
elif [[ "$line" =~ ^[[:space:]]*([A-Z0-9_]+)[[:space:]]*= ]]; then
|
||||
HOST_MAP["${BASH_REMATCH[1]}"]="$line"$'\n'
|
||||
fi
|
||||
fi
|
||||
done < "$TARGET"
|
||||
}
|
||||
|
||||
# ── Walk template — collect stats (must run in current shell so arrays persist) ──────────────
|
||||
|
||||
declare -a ADDED=() KEPT=() REMOVED=()
|
||||
declare -A TMPL_SEEN=()
|
||||
|
||||
_collect_stats() {
|
||||
local in_block=false cur_key="" line
|
||||
|
||||
while IFS= read -r line || [[ -n "$line" ]]; do
|
||||
if [[ "$in_block" == true ]]; then
|
||||
if [[ "$line" =~ ^[[:space:]]*\)[[:space:]]*(#.*)?$ ]]; then
|
||||
in_block=false
|
||||
TMPL_SEEN["$cur_key"]=1
|
||||
if [[ -n "${HOST_MAP[$cur_key]+_}" ]]; then KEPT+=("$cur_key")
|
||||
else ADDED+=("$cur_key"); fi
|
||||
cur_key=""
|
||||
fi
|
||||
else
|
||||
if [[ "$line" =~ ^[[:space:]]*declare[[:space:]]+-[a-zA-Z]+[[:space:]]+([A-Z0-9_]+)[[:space:]]*=\( ]]; then
|
||||
cur_key="${BASH_REMATCH[1]}"; in_block=true
|
||||
elif [[ "$line" =~ ^[[:space:]]*([A-Z0-9_]+)[[:space:]]*=\( ]]; then
|
||||
cur_key="${BASH_REMATCH[1]}"; in_block=true
|
||||
elif [[ "$line" =~ ^[[:space:]]*([A-Z0-9_]+)[[:space:]]*= ]]; then
|
||||
local k="${BASH_REMATCH[1]}"
|
||||
TMPL_SEEN["$k"]=1
|
||||
if [[ -n "${HOST_MAP[$k]+_}" ]]; then KEPT+=("$k")
|
||||
else ADDED+=("$k"); fi
|
||||
fi
|
||||
fi
|
||||
done < "$TEMPLATE"
|
||||
|
||||
for key in "${!HOST_MAP[@]}"; do
|
||||
[[ -z "${TMPL_SEEN[$key]+_}" ]] && REMOVED+=("$key")
|
||||
done
|
||||
}
|
||||
|
||||
# ── Walk template — write merged output ──────────────────────────────────────────────────────
|
||||
# Runs in a subshell (stdout redirected) — array mutations are intentionally discarded here.
|
||||
|
||||
_write_merged() {
|
||||
local in_block=false cur_key="" cur_block="" line
|
||||
|
||||
while IFS= read -r line || [[ -n "$line" ]]; do
|
||||
if [[ "$in_block" == true ]]; then
|
||||
cur_block+="$line"$'\n'
|
||||
if [[ "$line" =~ ^[[:space:]]*\)[[:space:]]*(#.*)?$ ]]; then
|
||||
in_block=false
|
||||
if [[ -n "${HOST_MAP[$cur_key]+_}" ]]; then printf '%s' "${HOST_MAP[$cur_key]}"
|
||||
else printf '%s' "$cur_block"; fi
|
||||
cur_key=""; cur_block=""
|
||||
fi
|
||||
else
|
||||
if [[ "$line" =~ ^[[:space:]]*declare[[:space:]]+-[a-zA-Z]+[[:space:]]+([A-Z0-9_]+)[[:space:]]*=\( ]]; then
|
||||
cur_key="${BASH_REMATCH[1]}"; in_block=true; cur_block="$line"$'\n'
|
||||
elif [[ "$line" =~ ^[[:space:]]*([A-Z0-9_]+)[[:space:]]*=\( ]]; then
|
||||
cur_key="${BASH_REMATCH[1]}"; in_block=true; cur_block="$line"$'\n'
|
||||
elif [[ "$line" =~ ^[[:space:]]*([A-Z0-9_]+)[[:space:]]*= ]]; then
|
||||
local k="${BASH_REMATCH[1]}"
|
||||
if [[ -n "${HOST_MAP[$k]+_}" ]]; then printf '%s' "${HOST_MAP[$k]}"
|
||||
else printf '%s\n' "$line"; fi
|
||||
else
|
||||
printf '%s\n' "$line"
|
||||
fi
|
||||
fi
|
||||
done < "$TEMPLATE"
|
||||
}
|
||||
|
||||
# ── Run ───────────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
_parse_target
|
||||
_collect_stats
|
||||
|
||||
# ── Report ────────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
TARGET_NAME="$(basename "$TARGET")"
|
||||
echo ""
|
||||
echo "── conf_upgrade: $TARGET_NAME ──────────────────────────────────────────"
|
||||
|
||||
if [[ ${#ADDED[@]} -gt 0 ]]; then
|
||||
echo " ADDED (new — fill in your values where needed):"
|
||||
for k in "${ADDED[@]}"; do echo " + $k"; done
|
||||
fi
|
||||
|
||||
if [[ ${#REMOVED[@]} -gt 0 ]]; then
|
||||
echo " REMOVED (deprecated — no longer in this version):"
|
||||
for k in "${REMOVED[@]}"; do echo " - $k"; done
|
||||
fi
|
||||
|
||||
echo " KEPT ${#KEPT[@]} existing vars — your values preserved"
|
||||
|
||||
if [[ ${#ADDED[@]} -eq 0 && ${#REMOVED[@]} -eq 0 ]]; then
|
||||
echo " Already up to date — no changes needed."
|
||||
echo "────────────────────────────────────────────────────────────────────────"
|
||||
echo ""
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "────────────────────────────────────────────────────────────────────────"
|
||||
echo ""
|
||||
|
||||
# ── Apply ─────────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "(dry-run — no changes written)"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
TMPOUT="$(mktemp)"
|
||||
trap 'rm -f "$TMPOUT"' EXIT
|
||||
|
||||
_write_merged > "$TMPOUT"
|
||||
|
||||
if [[ "$BACKUP" == true ]]; then
|
||||
cp "$TARGET" "${TARGET}.bak"
|
||||
echo "Backup: ${TARGET}.bak"
|
||||
fi
|
||||
|
||||
cp "$TMPOUT" "$TARGET"
|
||||
echo "Updated: $TARGET"
|
||||
@@ -161,6 +161,9 @@ validate_unraid_cmd \
|
||||
# Lock before detect_hosts — large library scans take time, wait mode appropriate
|
||||
[[ -n "${LIDARR_LOCK_WARN_AGE:-}" ]] && LOCK_WARN_AGE="$LIDARR_LOCK_WARN_AGE"
|
||||
acquire_lock "wait"
|
||||
TMP_DIR="/tmp/lidarr_cleanup_$$"
|
||||
mkdir -p "$TMP_DIR"
|
||||
trap "_release_all_locks; rm -rf $TMP_DIR" EXIT
|
||||
|
||||
if ! command -v docker &>/dev/null; then
|
||||
error "Docker command not found"
|
||||
@@ -402,10 +405,6 @@ fi
|
||||
|
||||
info "Found $ARTIST_COUNT artists — fetching track files..."
|
||||
|
||||
TMP_DIR="/tmp/lidarr_cleanup_$$"
|
||||
mkdir -p "$TMP_DIR"
|
||||
trap "rm -rf $TMP_DIR" EXIT
|
||||
|
||||
TRACKED_FILE="$TMP_DIR/tracked_paths.txt"
|
||||
> "$TRACKED_FILE"
|
||||
|
||||
|
||||
@@ -133,7 +133,7 @@ fi
|
||||
|
||||
# ── Temp dir for subshell fetch/fail counters ─────────────────────────────────────────────────
|
||||
LIDARR_TMP=$(mktemp -d)
|
||||
trap 'rm -rf "$LIDARR_TMP"' EXIT
|
||||
trap '_release_all_locks; rm -rf "$LIDARR_TMP"' EXIT
|
||||
touch "$LIDARR_TMP/album_fetches" "$LIDARR_TMP/album_fails" \
|
||||
"$LIDARR_TMP/artist_fetches" "$LIDARR_TMP/artist_fails"
|
||||
|
||||
|
||||
@@ -147,6 +147,9 @@ validate_unraid_cmd \
|
||||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||
|
||||
acquire_lock "wait"
|
||||
TMP_DIR="/tmp/radarr_cleanup_$$"
|
||||
mkdir -p "$TMP_DIR"
|
||||
trap "_release_all_locks; rm -rf $TMP_DIR" EXIT
|
||||
|
||||
if ! command -v docker &>/dev/null; then
|
||||
error "Docker command not found"
|
||||
@@ -388,10 +391,6 @@ fi
|
||||
|
||||
info "Found $MOVIE_COUNT movies — fetching movie files..."
|
||||
|
||||
TMP_DIR="/tmp/radarr_cleanup_$$"
|
||||
mkdir -p "$TMP_DIR"
|
||||
trap "rm -rf $TMP_DIR" EXIT
|
||||
|
||||
TRACKED_FILE="$TMP_DIR/tracked_paths.txt"
|
||||
> "$TRACKED_FILE"
|
||||
|
||||
|
||||
@@ -147,6 +147,9 @@ validate_unraid_cmd \
|
||||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||
|
||||
acquire_lock "wait"
|
||||
TMP_DIR="/tmp/sonarr_cleanup_$$"
|
||||
mkdir -p "$TMP_DIR"
|
||||
trap "_release_all_locks; rm -rf $TMP_DIR" EXIT
|
||||
|
||||
if ! command -v docker &>/dev/null; then
|
||||
error "Docker command not found"
|
||||
@@ -388,10 +391,6 @@ fi
|
||||
|
||||
info "Found $SERIES_COUNT series — fetching episode files..."
|
||||
|
||||
TMP_DIR="/tmp/sonarr_cleanup_$$"
|
||||
mkdir -p "$TMP_DIR"
|
||||
trap "rm -rf $TMP_DIR" EXIT
|
||||
|
||||
TRACKED_FILE="$TMP_DIR/tracked_paths.txt"
|
||||
> "$TRACKED_FILE"
|
||||
|
||||
|
||||
+52
-11
@@ -1,30 +1,71 @@
|
||||
#!/bin/bash
|
||||
# dev_install.sh — symlinks plugin files into unRAID WebUI for local development.
|
||||
# Run once after cloning. Re-run if the plugin directory is moved.
|
||||
# dev_install.sh — symlinks the OS-appropriate plugin into the web server for local development.
|
||||
# Run once after cloning. Re-run if the repo is moved.
|
||||
# Safe to re-run: removes stale symlink before recreating.
|
||||
#
|
||||
# Plugin layout:
|
||||
# Plugin/unraid/ ← Unraid plugin (PHP, served by emhttp at /usr/local/emhttp/plugins/)
|
||||
# Plugin/debian/ ← future
|
||||
# Plugin/<os>/ ← future
|
||||
#
|
||||
# Adding a new OS:
|
||||
# 1. Create Plugin/<os>/ with the OS-appropriate web app files
|
||||
# 2. Add detection + TARGET path below
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
PLUGIN_NAME="varaverk"
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
SOURCE="$SCRIPT_DIR/usr/local/emhttp/plugins/$PLUGIN_NAME"
|
||||
TARGET="/usr/local/emhttp/plugins/$PLUGIN_NAME"
|
||||
|
||||
if [ ! -d "$SOURCE" ]; then
|
||||
echo "ERROR: Source not found: $SOURCE"
|
||||
# ── Detect OS ────────────────────────────────────────────────────────────────
|
||||
|
||||
detect_os() {
|
||||
if [[ -f /etc/unraid-version ]]; then echo "unraid"
|
||||
elif [[ -f /etc/debian_version ]]; then echo "debian"
|
||||
elif [[ -f /etc/arch-release ]]; then echo "arch"
|
||||
else echo "unknown"; fi
|
||||
}
|
||||
|
||||
OS="${1:-$(detect_os)}" # accept override: ./dev_install.sh unraid
|
||||
|
||||
# ── OS-specific install target ────────────────────────────────────────────────
|
||||
|
||||
case "$OS" in
|
||||
unraid)
|
||||
SOURCE="$SCRIPT_DIR/unraid"
|
||||
TARGET="/usr/local/emhttp/plugins/$PLUGIN_NAME"
|
||||
;;
|
||||
debian|arch)
|
||||
echo "OS '$OS' detected but plugin target path not yet defined."
|
||||
echo "Add the TARGET= line for this OS in dev_install.sh."
|
||||
exit 1
|
||||
;;
|
||||
unknown|*)
|
||||
echo "Unsupported OS — could not detect from /etc/*-version markers."
|
||||
echo "Usage: $0 <os> (e.g. $0 unraid)"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
# ── Install ───────────────────────────────────────────────────────────────────
|
||||
|
||||
if [[ ! -d "$SOURCE" ]]; then
|
||||
echo "ERROR: Plugin source not found: $SOURCE"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -L "$TARGET" ]; then
|
||||
if [[ -L "$TARGET" ]]; then
|
||||
echo "Removing existing symlink: $TARGET"
|
||||
rm "$TARGET"
|
||||
elif [ -d "$TARGET" ]; then
|
||||
elif [[ -d "$TARGET" ]]; then
|
||||
echo "ERROR: $TARGET exists as a real directory — remove it manually first."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
ln -s "$SOURCE" "$TARGET"
|
||||
echo "Linked: $TARGET -> $SOURCE"
|
||||
echo ""
|
||||
echo "Plugin available in unRAID WebUI under Utilities → Varaverk."
|
||||
echo "Changes in $SOURCE take effect immediately (no restart needed)."
|
||||
echo "OS: $OS"
|
||||
echo "Source: $SOURCE"
|
||||
echo "Target: $TARGET"
|
||||
echo ""
|
||||
echo "Plugin linked. Changes in Plugin/$OS/ take effect immediately."
|
||||
|
||||
+3
-2
@@ -9,8 +9,9 @@ $pluginDir = "$docroot/plugins/$plugin";
|
||||
|
||||
// Determine active tab
|
||||
$tab = $_GET['tab'] ?? 'monitor';
|
||||
$validTabs = ['monitor', 'scheduler'];
|
||||
$validTabs = ['monitor', 'scheduler', 'partnership', 'fallback', 'watchdog', 'arrs'];
|
||||
if (!in_array($tab, $validTabs)) $tab = 'monitor';
|
||||
$tabLabels = ['monitor' => 'Monitor', 'scheduler' => 'Scheduler', 'partnership' => 'Partnership', 'fallback' => 'FallBack', 'watchdog' => 'Watchdog', 'arrs' => 'Arrs'];
|
||||
?>
|
||||
|
||||
<link rel="stylesheet" href="/plugins/<?=$plugin?>/css/varaverk.css">
|
||||
@@ -21,7 +22,7 @@ if (!in_array($tab, $validTabs)) $tab = 'monitor';
|
||||
<div id="vv-tabs">
|
||||
<?php foreach ($validTabs as $t): ?>
|
||||
<a href="?tab=<?=$t?>" class="vv-tab<?= $t === $tab ? ' active' : '' ?>">
|
||||
<?= ucfirst($t) ?>
|
||||
<?= $tabLabels[$t] ?? ucfirst($t) ?>
|
||||
</a>
|
||||
<?php endforeach; ?>
|
||||
</div>
|
||||
@@ -0,0 +1,4 @@
|
||||
<?php
|
||||
header('Content-Type: application/json');
|
||||
require_once dirname(__DIR__) . '/include/arrs.php';
|
||||
echo json_encode(vv_arrs_all());
|
||||
@@ -0,0 +1,121 @@
|
||||
<?php
|
||||
// Live board data: locks, recent errors, partner reachability.
|
||||
header('Content-Type: application/json');
|
||||
require_once dirname(__DIR__) . '/include/config.php';
|
||||
|
||||
$out = ['ok' => true];
|
||||
|
||||
// ── Active locks ──────────────────────────────────────────────────────────
|
||||
$lockDir = '/tmp/unraid_locks';
|
||||
$locks = [];
|
||||
if (is_dir($lockDir)) {
|
||||
foreach (glob($lockDir . '/*.lock') ?: [] as $lf) {
|
||||
$name = basename($lf, '.lock');
|
||||
$age = time() - (int)filemtime($lf);
|
||||
$content = trim(file_get_contents($lf) ?: '');
|
||||
// content is "PID:scriptname" — extract PID
|
||||
$pid = preg_match('/^(\d+)/', $content, $pm) ? $pm[1] : '';
|
||||
// Skip if PID is still alive (it's legitimately running)
|
||||
if ($pid && file_exists("/proc/$pid")) continue;
|
||||
$locks[] = ['name' => $name, 'file' => basename($lf), 'age' => $age];
|
||||
}
|
||||
}
|
||||
$out['locks'] = $locks;
|
||||
|
||||
// ── Recent errors ──────────────────────────────────────────────────────────
|
||||
// Scan recursively so subdirectory logs (Orchestrators/, Media/, etc.) are included.
|
||||
// e.script = relative path without extension = job ID base, e.g. "Orchestrators/transcode_management"
|
||||
$errors = [];
|
||||
if (is_dir(LOG_DIR)) {
|
||||
$cutoff = time() - 7 * 86400;
|
||||
$logBase = rtrim(LOG_DIR, '/') . '/';
|
||||
try {
|
||||
$ri = new RecursiveIteratorIterator(
|
||||
new RecursiveDirectoryIterator(LOG_DIR, RecursiveDirectoryIterator::SKIP_DOTS)
|
||||
);
|
||||
foreach ($ri as $lf) {
|
||||
if ($lf->getExtension() !== 'log') continue;
|
||||
if ($lf->getMTime() < $cutoff) continue;
|
||||
$rel = ltrim(str_replace($logBase, '', $lf->getPathname()), '/');
|
||||
$script = preg_replace('/\.log$/', '', $rel);
|
||||
|
||||
// Only report scripts whose last recorded run exited as warn or error.
|
||||
// This eliminates dry-run false positives (exit 0 = ok) and success-run
|
||||
// summaries that happen to contain words like "Failed: 0".
|
||||
$statFile = $logBase . $script . '.json';
|
||||
if (file_exists($statFile)) {
|
||||
$stat = json_decode(@file_get_contents($statFile), true);
|
||||
$status = $stat['status'] ?? '';
|
||||
if ($status !== 'warn' && $status !== 'error') continue;
|
||||
} else {
|
||||
continue; // no stat file — never ran through run_job.sh, skip
|
||||
}
|
||||
|
||||
$lines = array_slice(@file($lf->getPathname()) ?: [], -200);
|
||||
$lastErr = null;
|
||||
foreach (array_reverse($lines) as $raw) {
|
||||
$clean = preg_replace('/\033\[[0-9;]*[mK]/', '', rtrim($raw));
|
||||
if (!$clean) continue;
|
||||
if (preg_match('/\[(?:ERROR|WARN|CRITICAL|FAILED)\]/i', $clean) ||
|
||||
preg_match('/\b(?:ERROR|CRITICAL):\s/i', $clean) ||
|
||||
str_contains($clean, '✗') ||
|
||||
(str_contains($clean, '⚠') && !str_contains($clean, '♥'))) {
|
||||
$lastErr = mb_substr($clean, 0, 220);
|
||||
break;
|
||||
}
|
||||
}
|
||||
// Fall back to last non-blank line if no error pattern found in log
|
||||
if ($lastErr === null) {
|
||||
foreach (array_reverse($lines) as $raw) {
|
||||
$clean = preg_replace('/\033\[[0-9;]*[mK]/', '', rtrim($raw));
|
||||
if ($clean && !str_starts_with($clean, '──')) { $lastErr = mb_substr($clean, 0, 220); break; }
|
||||
}
|
||||
}
|
||||
if ($lastErr !== null)
|
||||
$errors[] = ['script' => $script, 'line' => $lastErr, 'ts' => (int)$lf->getMTime()];
|
||||
}
|
||||
} catch (Exception $e) {}
|
||||
usort($errors, fn($a, $b) => $b['ts'] - $a['ts']);
|
||||
}
|
||||
$out['errors'] = array_slice($errors, 0, 20);
|
||||
|
||||
// ── Partner reachability ───────────────────────────────────────────────────
|
||||
$cacheFile = '/tmp/vv_partner_cache.json';
|
||||
$cacheTtl = 30;
|
||||
$partnerData = null;
|
||||
|
||||
if (file_exists($cacheFile) && (time() - (int)filemtime($cacheFile)) < $cacheTtl) {
|
||||
$partnerData = json_decode(file_get_contents($cacheFile), true);
|
||||
} else {
|
||||
// Discover partner hostname dynamically from master.conf (works for any number of hosts)
|
||||
$vars = vv_conf_vars();
|
||||
$mine = vv_get_hostname();
|
||||
$partnerHost = null;
|
||||
foreach ($vars as $k => $v) {
|
||||
if (preg_match('/^HOST\d+$/', $k) && $v !== '' && strcasecmp($v, $mine) !== 0) {
|
||||
$partnerHost = $v; // first non-self host is the partner
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if ($partnerHost) {
|
||||
// Resolve via Tailscale — mirrors common.sh resolve_tailscale_ip().
|
||||
// Uses `tailscale ip -4` so it survives IP changes; never relies on local DNS.
|
||||
$partnerIp = vv_resolve_tailscale_ip($partnerHost);
|
||||
$target = $partnerIp ?: $partnerHost;
|
||||
$start = microtime(true);
|
||||
$result = shell_exec('ping -c1 -W2 ' . escapeshellarg($target) . ' 2>&1');
|
||||
$elapsed = (int)round((microtime(true) - $start) * 1000);
|
||||
$reached = str_contains((string)$result, '1 received')
|
||||
|| str_contains((string)$result, '1 packets received');
|
||||
$partnerData = [
|
||||
'host' => $partnerHost,
|
||||
'reachable' => $reached,
|
||||
'latency' => $reached ? $elapsed : null,
|
||||
];
|
||||
@file_put_contents($cacheFile, json_encode($partnerData));
|
||||
}
|
||||
}
|
||||
$out['partner'] = $partnerData;
|
||||
|
||||
echo json_encode($out);
|
||||
+2
-3
@@ -19,16 +19,15 @@ $logFile = vv_job_log_path($id);
|
||||
$logDir = dirname($logFile);
|
||||
if (!is_dir($logDir)) mkdir($logDir, 0755, true);
|
||||
|
||||
file_put_contents($logFile, "\n── " . date('Y-m-d H:i:s') . " [DRY RUN] ──────────────────────\n", FILE_APPEND);
|
||||
|
||||
$location = trim($_POST['location'] ?? '');
|
||||
if ($location && (!str_starts_with($location, '/') || str_contains($location, '..') || preg_match('/[\x00\n\r]/', $location))) {
|
||||
echo json_encode(['ok' => false, 'error' => 'Invalid location']);
|
||||
exit;
|
||||
}
|
||||
|
||||
$runner = dirname(__DIR__) . '/run_job.sh';
|
||||
$flags = vv_job_flags($id);
|
||||
$locArg = $location ? ' ' . escapeshellarg('--location=' . $location) : '';
|
||||
exec('nohup env DRY_RUN=1 bash ' . escapeshellarg($script) . ($flags ? " $flags" : '') . $locArg . ' >> ' . escapeshellarg($logFile) . ' 2>&1 </dev/null &');
|
||||
exec('nohup bash ' . escapeshellarg($runner) . ' ' . escapeshellarg($id) . ' ' . escapeshellarg($script) . ' --dry-run' . ($flags ? " $flags" : '') . ' --manual' . $locArg . ' >> ' . escapeshellarg($logFile) . ' 2>&1 </dev/null &');
|
||||
|
||||
echo json_encode(['ok' => true]);
|
||||
@@ -0,0 +1,4 @@
|
||||
<?php
|
||||
header('Content-Type: application/json');
|
||||
require_once dirname(__DIR__) . '/include/fallback.php';
|
||||
echo json_encode(vv_fb_all());
|
||||
@@ -0,0 +1,4 @@
|
||||
<?php
|
||||
header('Content-Type: application/json');
|
||||
require_once dirname(__DIR__) . '/include/partnership.php';
|
||||
echo json_encode(vv_partnership_all());
|
||||
@@ -0,0 +1,28 @@
|
||||
<?php
|
||||
header('Content-Type: application/json');
|
||||
|
||||
$logDir = '/var/log/varaverk';
|
||||
$runs = [];
|
||||
|
||||
try {
|
||||
$ri = new RecursiveIteratorIterator(
|
||||
new RecursiveDirectoryIterator($logDir, RecursiveDirectoryIterator::SKIP_DOTS)
|
||||
);
|
||||
foreach ($ri as $file) {
|
||||
if ($file->getExtension() !== 'json') continue;
|
||||
$d = @json_decode(@file_get_contents($file->getPathname()), true);
|
||||
if (!is_array($d) || empty($d['start']) || empty($d['status'])) continue;
|
||||
if ($d['status'] === 'running') continue;
|
||||
$id = (string)($d['id'] ?? '');
|
||||
$runs[] = [
|
||||
'id' => $id,
|
||||
'label' => basename(str_replace('.sh', '', $id)),
|
||||
'status' => $d['status'],
|
||||
'start' => (int)$d['start'],
|
||||
'dur' => isset($d['end']) ? max(0, (int)$d['end'] - (int)$d['start']) : 0,
|
||||
];
|
||||
}
|
||||
} catch (Exception $e) {}
|
||||
|
||||
usort($runs, fn($a, $b) => $b['start'] - $a['start']);
|
||||
echo json_encode(['ok' => true, 'runs' => array_slice($runs, 0, 24)]);
|
||||
@@ -19,16 +19,15 @@ $logFile = vv_job_log_path($id);
|
||||
$logDir = dirname($logFile);
|
||||
if (!is_dir($logDir)) mkdir($logDir, 0755, true);
|
||||
|
||||
file_put_contents($logFile, "\n── " . date('Y-m-d H:i:s') . " ──────────────────────\n", FILE_APPEND);
|
||||
|
||||
$location = trim($_POST['location'] ?? '');
|
||||
if ($location && (!str_starts_with($location, '/') || str_contains($location, '..') || preg_match('/[\x00\n\r]/', $location))) {
|
||||
echo json_encode(['ok' => false, 'error' => 'Invalid location']);
|
||||
exit;
|
||||
}
|
||||
|
||||
$runner = dirname(__DIR__) . '/run_job.sh';
|
||||
$flags = vv_job_flags($id);
|
||||
$locArg = $location ? ' ' . escapeshellarg('--location=' . $location) : '';
|
||||
exec('nohup bash ' . escapeshellarg($script) . ($flags ? " $flags" : '') . $locArg . ' >> ' . escapeshellarg($logFile) . ' 2>&1 </dev/null &');
|
||||
exec('nohup bash ' . escapeshellarg($runner) . ' ' . escapeshellarg($id) . ' ' . escapeshellarg($script) . ($flags ? " $flags" : '') . ' --manual' . $locArg . ' >> ' . escapeshellarg($logFile) . ' 2>&1 </dev/null &');
|
||||
|
||||
echo json_encode(['ok' => true]);
|
||||
+2
-2
@@ -10,7 +10,7 @@ if (!empty($_POST['batch'])) {
|
||||
$id = trim($e['id'] ?? '');
|
||||
$cron = trim($e['cron'] ?? '');
|
||||
if (!$id) continue;
|
||||
if ($cron && !in_array($cron, ['@array_start', '@array_stop'], true)
|
||||
if ($cron && !in_array($cron, ['array_start', 'array_stop'], true)
|
||||
&& !preg_match('/^(\S+\s+){4}\S+$/', $cron)) $cron = '';
|
||||
$clean[] = [
|
||||
'id' => $id,
|
||||
@@ -35,7 +35,7 @@ if (!$id) {
|
||||
}
|
||||
|
||||
// Basic cron validation — 5 fields, or known @event trigger, or empty
|
||||
if ($cron && !in_array($cron, ['@array_start', '@array_stop'], true)
|
||||
if ($cron && !in_array($cron, ['array_start', 'array_stop'], true)
|
||||
&& !preg_match('/^(\S+\s+){4}\S+$/', $cron)) {
|
||||
echo json_encode(['ok' => false, 'error' => 'Invalid cron expression']);
|
||||
exit;
|
||||
@@ -0,0 +1,54 @@
|
||||
<?php
|
||||
header('Content-Type: application/json');
|
||||
require_once dirname(__DIR__) . '/include/scheduler.php';
|
||||
|
||||
$id = trim($_GET['id'] ?? '');
|
||||
if (!$id || str_contains($id, '..') || !preg_match('/^[a-zA-Z0-9_.\/\-]+$/', $id)) {
|
||||
echo json_encode(['ok' => false, 'error' => 'Invalid id']);
|
||||
exit;
|
||||
}
|
||||
|
||||
$isScript = str_ends_with($id, '.sh');
|
||||
$name = basename($id, $isScript ? '.sh' : '');
|
||||
$dirName = basename(dirname($id)); // e.g. "Media", "Rsync", "Orchestrators"
|
||||
$slug = strtolower(str_replace(['_', '-'], ' ', $name));
|
||||
$parts = explode(' ', $slug);
|
||||
$first = $parts[0] ?? ''; // e.g. "radarr" from "radarr cleanup"
|
||||
|
||||
// Script header (bash scripts only)
|
||||
$path = SCRIPTS_DIR . '/' . $id;
|
||||
$header = ($isScript && file_exists($path)) ? vv_script_header_clean($path) : '';
|
||||
|
||||
// Section matcher: heading contains the full slug OR first meaningful word (>3 chars)
|
||||
$matcher = function(string $heading, bool $isIntro) use ($slug, $first): bool {
|
||||
if ($isIntro) return false;
|
||||
$h = strtolower(str_replace(['_', '-'], ' ', $heading));
|
||||
return str_contains($h, $slug)
|
||||
|| (strlen($first) > 3 && str_contains($h, $first));
|
||||
};
|
||||
|
||||
// Search main README/Manual + module-level files for this script's directory
|
||||
$scriptsDir = SCRIPTS_DIR;
|
||||
$searchFiles = [];
|
||||
foreach (['README', 'Manual'] as $docType) {
|
||||
$main = "$scriptsDir/$docType.md";
|
||||
if (file_exists($main)) $searchFiles[] = [$docType, $main];
|
||||
|
||||
$mod = "$scriptsDir/$docType-$dirName.md";
|
||||
if ($dirName && $dirName !== '.' && file_exists($mod) && $mod !== $main) {
|
||||
$searchFiles[] = ["$docType — $dirName", $mod];
|
||||
}
|
||||
}
|
||||
|
||||
$sections = [];
|
||||
foreach ($searchFiles as [$label, $file]) {
|
||||
$body = vv_readme_section($file, $matcher);
|
||||
if ($body) $sections[] = ['source' => $label, 'body' => $body];
|
||||
}
|
||||
|
||||
echo json_encode([
|
||||
'ok' => true,
|
||||
'name' => $name,
|
||||
'header' => $header,
|
||||
'sections' => $sections,
|
||||
]);
|
||||
@@ -0,0 +1,4 @@
|
||||
<?php
|
||||
header('Content-Type: application/json');
|
||||
require_once dirname(__DIR__) . '/include/watchdog.php';
|
||||
echo json_encode(vv_wd_all());
|
||||
+102
-18
@@ -55,12 +55,8 @@
|
||||
padding-left: 44px; box-sizing: border-box; width: 100%;
|
||||
white-space: nowrap; overflow: hidden; text-overflow: ellipsis;
|
||||
cursor: default; }
|
||||
.vv-cron { flex: 0 0 110px; width: 110px; background: #111; border: 1px solid #444; color: #ddd;
|
||||
padding: 4px 6px; border-radius: 4px; font-family: monospace; font-size: 13px; }
|
||||
.vv-cron-hint { font-size: 11px; font-family: monospace; color: #444; cursor: pointer;
|
||||
white-space: nowrap; flex-shrink: 0; padding: 3px 5px; border-radius: 3px;
|
||||
user-select: none; transition: color 0.15s; }
|
||||
.vv-cron-hint:hover { color: #888; background: rgba(255,255,255,0.04); }
|
||||
.vv-cron { flex: 0 0 80px; width: 80px; background: #111; border: 1px solid #444; color: #ddd;
|
||||
padding: 4px 6px; border-radius: 4px; font-family: monospace; font-size: 11px; }
|
||||
.vv-event-badge { flex: 0 0 auto; padding: 3px 8px; border-radius: 4px; font-size: 12px;
|
||||
background: #1a3a1a; border: 1px solid #2e6b2e; color: #6fcf6f;
|
||||
white-space: nowrap; font-weight: 500; }
|
||||
@@ -100,7 +96,7 @@
|
||||
#vv-sched-layout { display: flex; gap: 16px; align-items: stretch; }
|
||||
#vv-sched-left { flex: 1 1 0; min-width: 0; display: flex; flex-direction: column; align-self: flex-start; }
|
||||
#vv-sched-cards { flex: 1; }
|
||||
#vv-sched-right { display: none; flex: 1 1 0; min-width: 0; flex-direction: column; }
|
||||
#vv-sched-right { display: none; flex: 1 1 0; min-width: 0; flex-direction: column; overflow: hidden; align-self: flex-start; }
|
||||
#vv-sched-right.vv-panel-visible { display: flex; }
|
||||
.vv-log-card { flex: 1; display: flex; flex-direction: column; padding-bottom: 0; }
|
||||
.vv-log-right-pre { max-height: none; overflow-y: auto; }
|
||||
@@ -109,8 +105,13 @@
|
||||
@media (max-width: 900px) {
|
||||
#vv-sched-layout { flex-direction: column; align-items: stretch; }
|
||||
#vv-sched-left { flex: none; width: 100%; }
|
||||
#vv-sched-right { flex-direction: column; width: 100%; }
|
||||
.vv-log-right-pre { min-height: 260px; max-height: 340px; }
|
||||
#vv-sched-right { flex-direction: column; width: 100%; overflow: visible; }
|
||||
.vv-log-right-pre { min-height: 520px; max-height: 680px; }
|
||||
/* Toolbar: stack title row above buttons row, let buttons wrap */
|
||||
.vv-log-toolbar { flex-direction: column; align-items: flex-start; gap: 6px; }
|
||||
.vv-log-toolbar > div { flex-wrap: wrap; gap: 6px !important; }
|
||||
/* Array-event rows: badge already shows the event, cron input is redundant */
|
||||
.vv-job-row:has(.vv-event-badge) .vv-cron { display: none; }
|
||||
}
|
||||
|
||||
/* Plugin settings row (Advanced mode) */
|
||||
@@ -134,20 +135,25 @@
|
||||
}
|
||||
|
||||
/* Phone layout — scheduler row/actions fixes + monitor single-column */
|
||||
@media (max-width: 480px) {
|
||||
/* Phone portrait — hint gone, cron stays inline but compact */
|
||||
.vv-cron-hint { display: none !important; }
|
||||
.vv-job-row .vv-cron { flex: 0 0 68px; width: 68px; font-size: 10px; margin-left: 30px; }
|
||||
}
|
||||
|
||||
@media (max-width: 600px) {
|
||||
/* Allow action buttons to wrap rather than overflow the card */
|
||||
.vv-job-actions { flex-wrap: wrap; padding-left: 0; }
|
||||
/* Kill the inline margin-left:auto that pushes Advanced off-screen */
|
||||
.vv-advanced-toggle { margin-left: 0 !important; width: auto !important; }
|
||||
/* Let the cron input flex rather than hold a fixed 110px */
|
||||
.vv-cron { flex: 1 1 80px; width: auto; min-width: 80px; }
|
||||
/* Keep cron from growing — label owns remaining row space */
|
||||
.vv-cron { flex: 0 0 75px; width: 75px; }
|
||||
/* Slightly tighter label on narrow screens */
|
||||
.vv-job-label { font-size: 14px; }
|
||||
/* Footer buttons wrap instead of overflowing */
|
||||
.vv-sched-footer { flex-wrap: wrap; }
|
||||
/* Log toolbar wraps so Stop/Clear stay inside the card */
|
||||
.vv-log-toolbar { flex-wrap: wrap; align-items: flex-start; }
|
||||
.vv-log-toolbar > div { flex-wrap: wrap; }
|
||||
/* Log toolbar search — narrow on small screens */
|
||||
#vv-log-search { width: 80px; }
|
||||
/* Snapshot footer — smaller on mobile */
|
||||
.vv-snap-footer { gap: 10px !important; }
|
||||
.vv-snap-item { gap: 4px; }
|
||||
@@ -229,8 +235,16 @@
|
||||
border: 1px solid #333; border-radius: 4px; padding: 10px 12px; resize: none;
|
||||
line-height: 1.5; scroll-behavior: auto; tab-size: 2; width: 100%; box-sizing: border-box; }
|
||||
|
||||
/* Syntax-highlighted conf editor overlay */
|
||||
#vv-editor-wrap { position: relative; display: block; }
|
||||
/* Editor layout: gutter + inner area */
|
||||
#vv-editor-wrap { display: flex; }
|
||||
#vv-ln-gutter { width: 42px; min-width: 42px; flex-shrink: 0;
|
||||
background: #0a0a0a; border-right: 1px solid #1c1c1c;
|
||||
font-family: monospace; font-size: 12px; line-height: 1.5; tab-size: 2;
|
||||
color: #3a3a3a; text-align: right; padding: 10px 8px 10px 0;
|
||||
overflow: hidden; user-select: none; white-space: pre; }
|
||||
#vv-editor-inner { position: relative; flex: 1; overflow: hidden; }
|
||||
|
||||
/* Syntax-highlight overlay sits over the transparent textarea */
|
||||
#vv-hl-overlay { display: none; position: absolute; top: 1px; left: 1px; right: 1px; bottom: 1px;
|
||||
margin: 0; padding: 10px 12px; box-sizing: border-box;
|
||||
font-family: monospace; font-size: 12px; line-height: 1.5; tab-size: 2;
|
||||
@@ -271,6 +285,10 @@
|
||||
.vv-info-body { padding: 2px 10px 10px; }
|
||||
.vv-info-cols { margin: 0; padding-left: 16px; columns: 2; column-gap: 20px; column-fill: balance; }
|
||||
.vv-info-cols li { font-size: 12px; color: #aaa; margin-bottom: 5px; break-inside: avoid; line-height: 1.5; }
|
||||
.vv-info-cols .vv-info-sep { column-span: all; list-style: none; margin: 10px -4px 6px;
|
||||
padding: 4px 8px; background: #161a1d; border-left: 2px solid #1e6fa5;
|
||||
font-size: 10px; font-weight: bold; text-transform: uppercase;
|
||||
letter-spacing: .08em; color: #4a8ab5; break-inside: avoid; }
|
||||
.vv-info-cols li strong { color: #ccc; }
|
||||
.vv-info-cols code { background: #1a1a1a; padding: 0 4px; border-radius: 2px;
|
||||
font-size: 11px; color: #9ab; border: 1px solid #333; }
|
||||
@@ -286,6 +304,58 @@
|
||||
border-bottom: 1px solid #333;
|
||||
}
|
||||
|
||||
/* Run-status dot colours (tree + activity list) */
|
||||
.vv-stat-ok { color: #4caf50; }
|
||||
.vv-stat-warn { color: #ff9800; }
|
||||
.vv-stat-error{ color: #f44336; }
|
||||
.vv-stat-skip { color: #607d8b; }
|
||||
.vv-stat-none { color: #444; }
|
||||
|
||||
/* Cron humanizer hint — left of cron input; first to collapse under space pressure */
|
||||
.vv-cron-hint { font-size: 10px; color: #505050; white-space: nowrap;
|
||||
flex: 0 10 auto; min-width: 0; max-width: 130px;
|
||||
overflow: hidden; text-overflow: ellipsis;
|
||||
pointer-events: none; user-select: none; }
|
||||
|
||||
/* Recent Activity list */
|
||||
.vv-activity-list { display: flex; flex-direction: column; }
|
||||
.vv-activity-row { display: flex; align-items: center; gap: 6px; padding: 4px 0;
|
||||
border-bottom: 1px solid #1a1a1a; font-size: 11px; cursor: pointer; }
|
||||
.vv-activity-row:last-child { border-bottom: none; }
|
||||
.vv-activity-row:hover .vv-activity-label { color: #fff; }
|
||||
.vv-activity-dot { flex-shrink: 0; font-size: 9px; }
|
||||
.vv-activity-label { flex: 1; min-width: 0; overflow: hidden; text-overflow: ellipsis;
|
||||
white-space: nowrap; color: #ccc; }
|
||||
.vv-activity-ago { color: #555; font-size: 10px; white-space: nowrap; flex-shrink: 0; }
|
||||
.vv-activity-dur { color: #444; font-size: 10px; white-space: nowrap; flex-shrink: 0;
|
||||
min-width: 32px; text-align: right; }
|
||||
|
||||
/* Log search */
|
||||
#vv-log-search { width: 120px; font-size: 11px; padding: 2px 6px;
|
||||
background: #111; border: 1px solid #333; color: #ccc;
|
||||
border-radius: 3px; font-family: monospace; }
|
||||
#vv-log-search:focus { border-color: #555; outline: none; }
|
||||
mark { background: #5d4037; color: #ffcc80; border-radius: 2px; }
|
||||
.vv-log-dim { opacity: 0.2; }
|
||||
|
||||
/* Cron Calculator */
|
||||
.vv-calc-wrap { display: flex; flex-direction: column; gap: 8px; }
|
||||
.vv-calc-in { width: 100%; background: #111; border: 1px solid #333; color: #ccc;
|
||||
padding: 5px 8px; border-radius: 3px; font-family: monospace; font-size: 12px;
|
||||
box-sizing: border-box; }
|
||||
.vv-calc-in:focus { border-color: #555; outline: none; }
|
||||
.vv-calc-expr { font-family: monospace; font-size: 13px; color: #7cb8e8;
|
||||
background: #0d1117; padding: 4px 8px; border-radius: 3px; }
|
||||
.vv-calc-desc { font-size: 12px; color: #aaa; padding: 2px 2px 0; }
|
||||
.vv-calc-hint { font-size: 11px; color: #555; font-style: italic; }
|
||||
.vv-calc-runs-lbl { font-size: 10px; color: #555; text-transform: uppercase;
|
||||
letter-spacing: .05em; margin-top: 4px; }
|
||||
.vv-calc-runs { display: flex; flex-direction: column; gap: 2px; }
|
||||
.vv-calc-run-row { display: flex; gap: 8px; font-size: 11px; }
|
||||
.vv-calc-run-in { color: #4caf50; white-space: nowrap; min-width: 56px; }
|
||||
.vv-calc-run-at { color: #666; }
|
||||
.vv-calc-apply-btn { align-self: flex-start; margin-top: 2px; }
|
||||
|
||||
/* Board blocks — Next Runs, Errors, Locks, Partner, Disabled */
|
||||
.vv-board-placeholder { color: #555; font-size: 11px; padding: 3px 0; }
|
||||
|
||||
@@ -311,7 +381,10 @@
|
||||
.vv-err-row:last-child { border-bottom: none; }
|
||||
.vv-err-top { display: flex; align-items: baseline; gap: 8px; margin-bottom: 2px; }
|
||||
.vv-err-script { color: #e07070; font-size: 11px; font-weight: bold; }
|
||||
.vv-err-age { color: #555; font-size: 10px; }
|
||||
.vv-err-age { color: #555; font-size: 10px; flex: 1; padding-left: 6px; white-space: nowrap; }
|
||||
.vv-ack-btn { margin-left: auto; font-size: 10px; padding: 1px 6px;
|
||||
color: #555; border-color: #333; flex-shrink: 0; }
|
||||
.vv-ack-btn:hover { color: #aaa; border-color: #555; }
|
||||
.vv-err-line { color: #888; font-size: 11px; word-break: break-all; line-height: 1.4; }
|
||||
|
||||
/* Locks */
|
||||
@@ -404,6 +477,17 @@
|
||||
word-break: break-word; background: none; border: none; margin: 0;
|
||||
padding: 8px 12px; line-height: 1.5; }
|
||||
|
||||
/* Cog settings button next to script name */
|
||||
.vv-cog-btn { cursor:pointer; color:#444; font-size:.85em; margin-right:4px;
|
||||
line-height:1; vertical-align:middle; user-select:none; }
|
||||
.vv-cog-btn:hover { color:#aaa; }
|
||||
|
||||
/* Enriched script info blocks (advanced mode: header + docs + config) */
|
||||
.vv-sinfo-block { border-top: 1px solid #222; padding: 0; }
|
||||
.vv-sinfo-block:first-child { border-top: none; }
|
||||
.vv-sinfo-lbl { font-size: 10px; font-weight: bold; text-transform: uppercase;
|
||||
letter-spacing: .06em; color: #555; padding: 6px 12px 2px; }
|
||||
|
||||
/* Syntax highlight tokens — dark theme (VSCode-inspired) */
|
||||
.vv-hl-sep { color: #2d2d2d; }
|
||||
.vv-hl-shebang { color: #4a4a4a; }
|
||||
@@ -424,7 +508,7 @@
|
||||
/* Log panel */
|
||||
.vv-log-panel { margin-top: 10px; border-top: 1px solid #333; padding-top: 8px; }
|
||||
.vv-log-toolbar { display: flex; justify-content: space-between; align-items: center;
|
||||
margin-bottom: 4px; }
|
||||
flex-wrap: wrap; gap: 4px; margin-bottom: 4px; }
|
||||
.vv-log-ts { font-size: 11px; color: #666; }
|
||||
.vv-log-pre { background: #0d0d0d; border: 1px solid #333; border-radius: 4px;
|
||||
padding: 10px 12px; margin: 0; font-family: monospace; font-size: 12px;
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
#!/bin/bash
|
||||
# Varaverk: fire all jobs with cron=array_start (background — non-blocking).
|
||||
php -r "
|
||||
require_once '/usr/local/emhttp/plugins/varaverk/include/scheduler.php';
|
||||
\$s = vv_schedule_load();
|
||||
\$runner = '/usr/local/emhttp/plugins/varaverk/run_job.sh';
|
||||
foreach (\$s as \$id => \$e) {
|
||||
if (strncmp(\$id, '__', 2) === 0) continue;
|
||||
if ((\$e['cron'] ?? '') !== 'array_start') continue;
|
||||
if (empty(\$e['enabled'])) continue;
|
||||
\$script = SCRIPTS_DIR . '/' . \$id;
|
||||
if (!file_exists(\$script)) continue;
|
||||
\$flags = !empty(\$e['log_enabled']) ? ' --log' : '';
|
||||
exec('nohup bash ' . escapeshellarg(\$runner) . ' ' . escapeshellarg(\$id) . ' ' . escapeshellarg(\$script) . \$flags . ' > /dev/null 2>&1 &');
|
||||
}
|
||||
" 2>/dev/null
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
#!/bin/bash
|
||||
# Varaverk: fire all jobs with cron=array_stop (foreground — blocks until all done).
|
||||
php -r "
|
||||
require_once '/usr/local/emhttp/plugins/varaverk/include/scheduler.php';
|
||||
\$s = vv_schedule_load();
|
||||
\$runner = '/usr/local/emhttp/plugins/varaverk/run_job.sh';
|
||||
foreach (\$s as \$id => \$e) {
|
||||
if (strncmp(\$id, '__', 2) === 0) continue;
|
||||
if ((\$e['cron'] ?? '') !== 'array_stop') continue;
|
||||
if (empty(\$e['enabled'])) continue;
|
||||
\$script = SCRIPTS_DIR . '/' . \$id;
|
||||
if (!file_exists(\$script)) continue;
|
||||
\$flags = !empty(\$e['log_enabled']) ? ' --log' : '';
|
||||
passthru('bash ' . escapeshellarg(\$runner) . ' ' . escapeshellarg(\$id) . ' ' . escapeshellarg(\$script) . \$flags);
|
||||
}
|
||||
" 2>/dev/null
|
||||
|
Before Width: | Height: | Size: 103 B After Width: | Height: | Size: 103 B |
@@ -0,0 +1,278 @@
|
||||
<?php
|
||||
// Arr (Sonarr / Radarr / Lidarr) data helpers
|
||||
|
||||
require_once __DIR__ . '/config.php';
|
||||
|
||||
// ── Conf helpers ──────────────────────────────────────────────────────────────
|
||||
|
||||
function vv_arr_scalar(string $raw, string $key): string {
|
||||
return preg_match('/^\s*' . preg_quote($key, '/') . '\s*=\s*"?([^"\n]*)"?/m', $raw, $m)
|
||||
? trim($m[1]) : '';
|
||||
}
|
||||
|
||||
// Returns all configured hosts found in master.conf as ['host1'=>'hostname', ...]
|
||||
// Returns ['host1' => 'hostname', 'host2' => 'hostname', ...] from master.conf.
|
||||
// Matches both HOST1="name" and HOST1_NAME="name" (either convention).
|
||||
function vv_arr_known_hosts(): array {
|
||||
$vars = vv_conf_vars(); // already parses HOST1="val" correctly
|
||||
$hosts = [];
|
||||
foreach ($vars as $k => $v) {
|
||||
if (preg_match('/^HOST(\d+)$/', $k, $m) && $v !== '') {
|
||||
$hosts['host' . $m[1]] = $v;
|
||||
}
|
||||
}
|
||||
ksort($hosts);
|
||||
return $hosts ?: ['host1' => 'HOST1'];
|
||||
}
|
||||
|
||||
function vv_arr_node_names(): array {
|
||||
return array_map(fn($name) => $name, vv_arr_known_hosts());
|
||||
}
|
||||
|
||||
// ── Discovery ─────────────────────────────────────────────────────────────────
|
||||
|
||||
function vv_discover_arrs(): array {
|
||||
$nodes = [];
|
||||
$defs = [
|
||||
'sonarr' => ['SONARR_URL', 'SONARR_API_KEY', 'SONARR_TV_ROOT', 'v3'],
|
||||
'radarr' => ['RADARR_URL', 'RADARR_API_KEY', 'RADARR_MOVIES_ROOT', 'v3'],
|
||||
'lidarr' => ['LIDARR_URL', 'LIDARR_API_KEY', 'LIDARR_MUSIC_ROOT', 'v1'],
|
||||
];
|
||||
foreach (array_keys(vv_arr_known_hosts()) as $h) {
|
||||
$raw = vv_read_conf_raw($h . '.conf');
|
||||
if (!$raw) continue;
|
||||
$pfx = strtoupper($h) . '_';
|
||||
$get = fn($k) => vv_arr_scalar($raw, $pfx . $k);
|
||||
$arrs = [];
|
||||
foreach ($defs as $type => [$uk, $ak, $rk, $api]) {
|
||||
$url = $get($uk);
|
||||
$key = $get($ak);
|
||||
if ($url && $key && !str_contains($key, 'your-')) {
|
||||
$arrs[] = ['type' => $type, 'url' => $url, 'key' => $key,
|
||||
'root' => $get($rk), 'api' => $api];
|
||||
}
|
||||
}
|
||||
if ($arrs) $nodes[] = ['host' => $h, 'arrs' => $arrs];
|
||||
}
|
||||
return $nodes;
|
||||
}
|
||||
|
||||
// ── HTTP ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
function vv_arr_http(string $url, string $apiKey, string $path, int $timeout = 4): ?array {
|
||||
$ctx = stream_context_create(['http' => [
|
||||
'timeout' => $timeout,
|
||||
'header' => "X-Api-Key: $apiKey\r\nAccept: application/json\r\n",
|
||||
'ignore_errors' => true,
|
||||
]]);
|
||||
$raw = @file_get_contents(rtrim($url, '/') . $path, false, $ctx);
|
||||
return $raw ? (json_decode($raw, true) ?: null) : null;
|
||||
}
|
||||
|
||||
// ── Live arr data ─────────────────────────────────────────────────────────────
|
||||
|
||||
function vv_fetch_arr_live(array $arr): array {
|
||||
$url = $arr['url'];
|
||||
$key = $arr['key'];
|
||||
$base = '/api/' . $arr['api'];
|
||||
$type = $arr['type'];
|
||||
|
||||
$out = ['online' => false, 'version' => null, 'health' => [],
|
||||
'queue' => ['dl' => 0, 'warn' => 0, 'err' => 0], 'disk' => []];
|
||||
|
||||
$sys = vv_arr_http($url, $key, "$base/system/status");
|
||||
if (!$sys) return $out;
|
||||
$out['online'] = true;
|
||||
$out['version'] = $sys['version'] ?? null;
|
||||
|
||||
if ($type === 'sonarr') {
|
||||
$data = vv_arr_http($url, $key, "$base/series");
|
||||
if (is_array($data)) {
|
||||
$out['total'] = count($data);
|
||||
$out['monitored'] = count(array_filter($data, fn($x) => !empty($x['monitored'])));
|
||||
$out['episodes'] = array_sum(array_column($data, 'episodeFileCount'));
|
||||
}
|
||||
} elseif ($type === 'radarr') {
|
||||
$data = vv_arr_http($url, $key, "$base/movie");
|
||||
if (is_array($data)) {
|
||||
$out['total'] = count($data);
|
||||
$out['monitored'] = count(array_filter($data, fn($x) => !empty($x['monitored'])));
|
||||
$out['files'] = count(array_filter($data, fn($x) => !empty($x['hasFile'])));
|
||||
}
|
||||
} elseif ($type === 'lidarr') {
|
||||
$data = vv_arr_http($url, $key, "$base/artist");
|
||||
if (is_array($data)) {
|
||||
$out['total'] = count($data);
|
||||
$out['monitored'] = count(array_filter($data, fn($x) => !empty($x['monitored'])));
|
||||
$out['albums'] = array_sum(array_map(
|
||||
fn($a) => $a['statistics']['albumCount'] ?? $a['albumCount'] ?? 0, $data));
|
||||
}
|
||||
}
|
||||
|
||||
$q = vv_arr_http($url, $key, "$base/queue?page=1&pageSize=500");
|
||||
if (is_array($q)) {
|
||||
foreach (($q['records'] ?? $q) as $r) {
|
||||
if (!is_array($r)) continue;
|
||||
$s = $r['status'] ?? '';
|
||||
$tds = strtolower($r['trackedDownloadStatus'] ?? '');
|
||||
$tst = strtolower($r['trackedDownloadState'] ?? '');
|
||||
if ($s === 'downloading') $out['queue']['dl']++;
|
||||
if ($tds === 'warning' || $tst === 'downloadingstalled') $out['queue']['warn']++;
|
||||
if ($tds === 'error') $out['queue']['err']++;
|
||||
}
|
||||
}
|
||||
|
||||
$h = vv_arr_http($url, $key, "$base/health");
|
||||
if (is_array($h)) $out['health'] = $h;
|
||||
|
||||
$d = vv_arr_http($url, $key, "$base/diskspace");
|
||||
if (is_array($d)) $out['disk'] = $d;
|
||||
|
||||
return $out;
|
||||
}
|
||||
|
||||
// ── Log stats ─────────────────────────────────────────────────────────────────
|
||||
|
||||
function vv_arr_cleanup_stats(string $type): array {
|
||||
$slugs = ['sonarr' => 'Media/sonarr_cleanup',
|
||||
'radarr' => 'Media/radarr_cleanup',
|
||||
'lidarr' => 'Media/lidarr_cleanup'];
|
||||
$base = LOG_DIR . '/' . ($slugs[$type] ?? '');
|
||||
$out = ['last_run' => null, 'end' => null, 'status' => null,
|
||||
'tracked' => null, 'total' => null,
|
||||
'orphans' => 0, 'orphans_sz' => '0B', 'junk' => 0];
|
||||
|
||||
$jf = $base . '.json';
|
||||
if (!file_exists($jf)) return $out;
|
||||
$meta = json_decode(file_get_contents($jf), true) ?: [];
|
||||
$out['last_run'] = $meta['start'] ?? null;
|
||||
$out['end'] = $meta['end'] ?? null;
|
||||
$out['status'] = $meta['status'] ?? null;
|
||||
|
||||
$lf = $base . '.log';
|
||||
if (!file_exists($lf)) return $out;
|
||||
$log = file_get_contents($lf);
|
||||
$parts = preg_split('/━{3,}[^\n]*SUMMARY[^\n]*/u', $log);
|
||||
$blk = count($parts) > 1 ? end($parts) : $log;
|
||||
|
||||
if (preg_match('/Tracked:\s*([\d,]+)\s*files\s*\(([\d,]+)/u', $blk, $m)) {
|
||||
$out['tracked'] = (int)str_replace(',', '', $m[1]);
|
||||
$out['total'] = (int)str_replace(',', '', $m[2]);
|
||||
}
|
||||
if (preg_match('/Orphans:\s*([\d,]+)\s*files\s*\(([^)]+)\)/u', $blk, $m)) {
|
||||
$out['orphans'] = (int)str_replace(',', '', $m[1]);
|
||||
$out['orphans_sz'] = trim($m[2]);
|
||||
}
|
||||
if (preg_match('/Junk:\s*([\d,]+)\s*files/u', $blk, $m)) {
|
||||
$out['junk'] = (int)str_replace(',', '', $m[1]);
|
||||
}
|
||||
return $out;
|
||||
}
|
||||
|
||||
function vv_arr_discovery_stats(string $type): array {
|
||||
$slugs = ['sonarr' => 'Media/playback_aware_sonarr_discovery',
|
||||
'radarr' => 'Media/playback_aware_radarr_discovery',
|
||||
'lidarr' => 'Media/playback_aware_lidarr_discovery'];
|
||||
$base = LOG_DIR . '/' . ($slugs[$type] ?? '');
|
||||
$out = ['last_run' => null, 'status' => null, 'added' => null];
|
||||
|
||||
$jf = $base . '.json';
|
||||
if (!file_exists($jf)) return $out;
|
||||
$meta = json_decode(file_get_contents($jf), true) ?: [];
|
||||
$out['last_run'] = $meta['start'] ?? null;
|
||||
$out['status'] = $meta['status'] ?? null;
|
||||
|
||||
$lf = $base . '.log';
|
||||
if (file_exists($lf)) {
|
||||
$log = file_get_contents($lf);
|
||||
if (preg_match('/Added[:\s]+(\d+)/i', $log, $m)) $out['added'] = (int)$m[1];
|
||||
elseif (preg_match('/(\d+)\s+added/i', $log, $m)) $out['added'] = (int)$m[1];
|
||||
}
|
||||
return $out;
|
||||
}
|
||||
|
||||
function vv_arr_sync_stats(): array {
|
||||
$base = LOG_DIR . '/Media/arr_sync';
|
||||
$out = ['last_run' => null, 'status' => null, 'added' => null,
|
||||
'nodes' => null, 'blocklist_count' => null];
|
||||
|
||||
$jf = $base . '.json';
|
||||
if (file_exists($jf)) {
|
||||
$meta = json_decode(file_get_contents($jf), true) ?: [];
|
||||
$out['last_run'] = $meta['start'] ?? null;
|
||||
$out['status'] = $meta['status'] ?? null;
|
||||
}
|
||||
|
||||
$master = vv_read_conf_raw('master.conf');
|
||||
if (preg_match('/ARR_SYNC_BLOCKLIST\s*=\s*"?([^"\n#]+)"?/m', $master, $m)) {
|
||||
$blPath = trim($m[1]);
|
||||
if (file_exists($blPath)) {
|
||||
$out['blocklist_count'] = count(array_filter(
|
||||
file($blPath, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES)));
|
||||
}
|
||||
}
|
||||
|
||||
$lf = $base . '.log';
|
||||
if (file_exists($lf)) {
|
||||
$log = file_get_contents($lf);
|
||||
if (preg_match('/Total added[:\s]+(\d+)/i', $log, $m)) $out['added'] = (int)$m[1];
|
||||
if (preg_match('/Nodes?[:\s]+(\d+)/i', $log, $m)) $out['nodes'] = (int)$m[1];
|
||||
}
|
||||
return $out;
|
||||
}
|
||||
|
||||
function vv_arr_recovery_stats(): array {
|
||||
$base = LOG_DIR . '/Media/arrs_failed_stalled_recovery';
|
||||
$out = ['last_run' => null, 'status' => null, 'fixed' => 0, 'searched' => 0];
|
||||
|
||||
$jf = $base . '.json';
|
||||
if (!file_exists($jf)) return $out;
|
||||
$meta = json_decode(file_get_contents($jf), true) ?: [];
|
||||
$out['last_run'] = $meta['start'] ?? null;
|
||||
$out['status'] = $meta['status'] ?? null;
|
||||
|
||||
$lf = $base . '.log';
|
||||
if (file_exists($lf)) {
|
||||
$log = file_get_contents($lf);
|
||||
if (preg_match('/Removed[:\s]+(\d+)/i', $log, $m)) $out['fixed'] = (int)$m[1];
|
||||
if (preg_match('/Re-searched[:\s]+(\d+)/i',$log, $m)) $out['searched'] = (int)$m[1];
|
||||
}
|
||||
return $out;
|
||||
}
|
||||
|
||||
// ── Entry point ───────────────────────────────────────────────────────────────
|
||||
|
||||
function vv_arrs_all(): array {
|
||||
$currentHost = vv_detect_host();
|
||||
$names = vv_arr_node_names();
|
||||
$allNodes = vv_discover_arrs();
|
||||
$result = [];
|
||||
|
||||
foreach ($allNodes as $node) {
|
||||
$h = $node['host'];
|
||||
$isLocal = ($h === $currentHost || $currentHost === 'unknown');
|
||||
$nodeOut = ['host' => $h, 'name' => $names[$h] ?? strtoupper($h), 'local' => $isLocal, 'arrs' => []];
|
||||
|
||||
foreach ($node['arrs'] as $arr) {
|
||||
$entry = ['type' => $arr['type'], 'root' => $arr['root']];
|
||||
if ($isLocal) {
|
||||
$entry = array_merge($entry, vv_fetch_arr_live($arr));
|
||||
$entry['cleanup'] = vv_arr_cleanup_stats($arr['type']);
|
||||
$entry['discovery'] = vv_arr_discovery_stats($arr['type']);
|
||||
} else {
|
||||
$entry['online'] = false;
|
||||
$entry['remote'] = true;
|
||||
}
|
||||
$nodeOut['arrs'][] = $entry;
|
||||
}
|
||||
$result[] = $nodeOut;
|
||||
}
|
||||
|
||||
return [
|
||||
'nodes' => $result,
|
||||
'sync' => vv_arr_sync_stats(),
|
||||
'recovery' => vv_arr_recovery_stats(),
|
||||
'host' => $currentHost,
|
||||
'ts' => time(),
|
||||
];
|
||||
}
|
||||
+25
-9
@@ -14,14 +14,29 @@ function vv_get_hostname(): string {
|
||||
return trim(shell_exec('hostname -s') ?: '');
|
||||
}
|
||||
|
||||
// Mirror of common.sh resolve_tailscale_ip(): tries `tailscale ip -4` first (Tailscale manages
|
||||
// the mapping so this survives IP changes), falls back to parsing `tailscale status` text.
|
||||
function vv_resolve_tailscale_ip(string $hostname): string {
|
||||
$h = strtolower($hostname);
|
||||
$ip = trim(shell_exec('tailscale ip -4 ' . escapeshellarg($h) . ' 2>/dev/null') ?: '');
|
||||
if ($ip) return $ip;
|
||||
$out = shell_exec('tailscale status 2>/dev/null') ?: '';
|
||||
foreach (explode("\n", $out) as $line) {
|
||||
$cols = preg_split('/\s+/', trim($line));
|
||||
if (isset($cols[1]) && stripos($cols[1], $h . '.') === 0) return $cols[0];
|
||||
}
|
||||
return '';
|
||||
}
|
||||
|
||||
function vv_detect_host(): string {
|
||||
// Reads master.conf to find HOST1/HOST2 hostnames, returns 'host1', 'host2', or 'unknown'
|
||||
// Reads master.conf for HOST1="name" (or HOST1_NAME="name") and matches running hostname.
|
||||
// Returns 'host1', 'host2', 'host3', ... or 'unknown'. Works for any number of hosts.
|
||||
$master = vv_read_conf_raw('master.conf');
|
||||
preg_match('/^\s*HOST1_NAME\s*=\s*["\']?(\S+?)["\']?\s*$/m', $master, $m1);
|
||||
preg_match('/^\s*HOST2_NAME\s*=\s*["\']?(\S+?)["\']?\s*$/m', $master, $m2);
|
||||
preg_match_all('/^\s*(HOST\d+)(?:_NAME)?\s*=\s*["\']?(\S+?)["\']?\s*$/m', $master, $m);
|
||||
$hostname = vv_get_hostname();
|
||||
if (!empty($m1[1]) && strcasecmp($hostname, $m1[1]) === 0) return 'host1';
|
||||
if (!empty($m2[1]) && strcasecmp($hostname, $m2[1]) === 0) return 'host2';
|
||||
foreach ($m[1] as $i => $key) {
|
||||
if (strcasecmp($hostname, trim($m[2][$i])) === 0) return strtolower($key);
|
||||
}
|
||||
return 'unknown';
|
||||
}
|
||||
|
||||
@@ -44,10 +59,12 @@ function vv_get_conf_files(): array {
|
||||
$host = vv_detect_host();
|
||||
$files = [];
|
||||
if ($host === 'host1') {
|
||||
// Owner sees master.conf + their own host conf
|
||||
$files[] = 'master.conf';
|
||||
$files[] = 'host1.conf';
|
||||
} elseif ($host === 'host2') {
|
||||
$files[] = 'host2.conf';
|
||||
} elseif (preg_match('/^host(\d+)$/', $host)) {
|
||||
// Any other numbered host sees only their own conf
|
||||
$files[] = $host . '.conf';
|
||||
} else {
|
||||
// Unknown host — show all for dev/debug
|
||||
foreach (glob(CONF_DIR . '/*.conf') as $f) {
|
||||
@@ -62,8 +79,7 @@ function vv_conf_vars(): array {
|
||||
$vars = [];
|
||||
$files = ['master.conf'];
|
||||
$host = vv_detect_host();
|
||||
if ($host === 'host1') $files[] = 'host1.conf';
|
||||
if ($host === 'host2') $files[] = 'host2.conf';
|
||||
if (preg_match('/^host\d+$/', $host)) $files[] = $host . '.conf';
|
||||
|
||||
foreach ($files as $f) {
|
||||
$raw = vv_read_conf_raw($f);
|
||||
@@ -0,0 +1,184 @@
|
||||
<?php
|
||||
// Fallback tab data helpers
|
||||
|
||||
require_once __DIR__ . '/config.php';
|
||||
require_once __DIR__ . '/partnership.php'; // vv_pt_ssh(), vv_pt_ts_peers()
|
||||
|
||||
// ── Conf parsers ──────────────────────────────────────────────────────────────
|
||||
|
||||
function vv_fb_bash_array(string $raw, string $varname): array {
|
||||
if (!preg_match('/^\s*' . preg_quote($varname, '/') . '\s*=\s*\(\s*(.*?)\s*\)/ms', $raw, $m))
|
||||
return [];
|
||||
preg_match_all('/"([^"]*)"/', $m[1], $items);
|
||||
return array_values(array_filter($items[1]));
|
||||
}
|
||||
|
||||
function vv_fb_scalar(string $raw, string $varname): string {
|
||||
return preg_match('/^\s*' . preg_quote($varname, '/') . '\s*=\s*"?([^"\n]*)"?/m', $raw, $m)
|
||||
? trim($m[1]) : '';
|
||||
}
|
||||
|
||||
// ── State file ────────────────────────────────────────────────────────────────
|
||||
|
||||
function vv_fb_parse_state(string $text): array {
|
||||
$out = [
|
||||
'state' => 'UNKNOWN',
|
||||
'fallback_start' => 0,
|
||||
'handback_strikes' => 0,
|
||||
'tier2_started' => false,
|
||||
'tier3_started' => false,
|
||||
'tier4_started' => false,
|
||||
];
|
||||
foreach (explode("\n", $text) as $line) {
|
||||
$line = trim($line);
|
||||
if (!$line || !str_contains($line, '=')) continue;
|
||||
[$k, $v] = array_pad(explode('=', $line, 2), 2, '');
|
||||
$k = trim($k); $v = trim($v, '"\'');
|
||||
switch ($k) {
|
||||
case 'state': $out['state'] = $v; break;
|
||||
case 'fallback_start': $out['fallback_start'] = (int)$v; break;
|
||||
case 'handback_strikes': $out['handback_strikes'] = (int)$v; break;
|
||||
case 'tier2_started': $out['tier2_started'] = $v === 'true'; break;
|
||||
case 'tier3_started': $out['tier3_started'] = $v === 'true'; break;
|
||||
case 'tier4_started': $out['tier4_started'] = $v === 'true'; break;
|
||||
}
|
||||
}
|
||||
return $out;
|
||||
}
|
||||
|
||||
function vv_fb_local_state(): array {
|
||||
$path = '/boot/config/fallback_state.db';
|
||||
return vv_fb_parse_state(file_exists($path) ? file_get_contents($path) : '');
|
||||
}
|
||||
|
||||
function vv_fb_remote_state(string $ip, string $sshKey): array {
|
||||
$out = vv_pt_ssh($ip, $sshKey, 'cat /boot/config/fallback_state.db 2>/dev/null');
|
||||
return vv_fb_parse_state($out);
|
||||
}
|
||||
|
||||
// ── Running containers ────────────────────────────────────────────────────────
|
||||
|
||||
function vv_fb_local_running(): array {
|
||||
$out = shell_exec("docker ps --format '{{.Names}}' 2>/dev/null") ?: '';
|
||||
return array_values(array_filter(explode("\n", trim($out))));
|
||||
}
|
||||
|
||||
function vv_fb_remote_running(string $ip, string $sshKey): array {
|
||||
$out = vv_pt_ssh($ip, $sshKey, "docker ps --format '{{.Names}}' 2>/dev/null");
|
||||
return array_values(array_filter(explode("\n", trim($out))));
|
||||
}
|
||||
|
||||
// ── Covers — what a node runs for the other when it's down ───────────────────
|
||||
|
||||
function vv_fb_covers(string $covering, string $remote, string $coveringRaw, string $remoteRaw): array {
|
||||
$cu = strtoupper($covering); // HOST1
|
||||
$ru = strtoupper($remote); // HOST2
|
||||
$tiers = [];
|
||||
for ($t = 1; $t <= 4; $t++) {
|
||||
$tiers["tier$t"] = vv_fb_bash_array($coveringRaw, "FALLBACK_{$cu}_COVERS_{$ru}_TIER{$t}");
|
||||
}
|
||||
// Delays: how long the remote (covered) host must be down before each tier fires.
|
||||
// Stored in the *remote* host's conf as REMOTE_TIER*_DELAY.
|
||||
$tiers['delays'] = [
|
||||
'tier2' => (int)(vv_fb_scalar($remoteRaw, "{$ru}_TIER2_DELAY") ?: 240),
|
||||
'tier3' => (int)(vv_fb_scalar($remoteRaw, "{$ru}_TIER3_DELAY") ?: 720),
|
||||
'tier4' => (int)(vv_fb_scalar($remoteRaw, "{$ru}_TIER4_DELAY") ?: 1440),
|
||||
];
|
||||
return $tiers;
|
||||
}
|
||||
|
||||
// ── Known hosts ───────────────────────────────────────────────────────────────
|
||||
|
||||
function vv_fb_known_hosts(): array {
|
||||
$master = vv_read_conf_raw('master.conf');
|
||||
preg_match_all('/^\s*(HOST(\d+))(?:_NAME)?\s*=\s*["\']?(\S+?)["\']?\s*$/m', $master, $m);
|
||||
$hosts = [];
|
||||
foreach ($m[1] as $i => $key) {
|
||||
$num = $m[2][$i];
|
||||
$name = trim($m[3][$i]);
|
||||
$hosts['host' . $num] = $name;
|
||||
}
|
||||
ksort($hosts);
|
||||
return $hosts ?: ['host1' => 'HOST1'];
|
||||
}
|
||||
|
||||
// ── Main data builder ─────────────────────────────────────────────────────────
|
||||
|
||||
function vv_fb_all(): array {
|
||||
$currentHost = vv_detect_host();
|
||||
$hosts = vv_fb_known_hosts();
|
||||
$tsPeers = vv_pt_ts_peers();
|
||||
$handbackReq = (int)(vv_fb_scalar(vv_read_conf_raw('master.conf'), 'FALLBACK_HANDBACK_STRIKES') ?: 3);
|
||||
$fbEnabled = vv_fb_scalar(vv_read_conf_raw('master.conf'), 'FALLBACK_ENABLED') === 'true';
|
||||
|
||||
// Read all host conf raws upfront
|
||||
$raws = [];
|
||||
foreach (array_keys($hosts) as $slot) {
|
||||
$raws[$slot] = vv_read_conf_raw($slot . '.conf');
|
||||
}
|
||||
|
||||
// SSH key — from local host conf
|
||||
$myId = strtoupper($currentHost);
|
||||
$myRaw = $raws[$currentHost] ?? '';
|
||||
$mySshKey = vv_fb_scalar($myRaw, $myId . '_SSH_KEY');
|
||||
|
||||
$nodes = [];
|
||||
foreach ($hosts as $slot => $hostname) {
|
||||
$isMe = ($slot === $currentHost || $currentHost === 'unknown');
|
||||
$tsLabel = strtolower($hostname);
|
||||
$ts = $tsPeers[$tsLabel] ?? ['online' => null, 'active' => false, 'ip' => null];
|
||||
$ip = $ts['ip'] ?? null;
|
||||
|
||||
// State
|
||||
if ($isMe) {
|
||||
$state = vv_fb_local_state();
|
||||
} elseif ($ip && $mySshKey) {
|
||||
$state = vv_fb_remote_state($ip, $mySshKey);
|
||||
} else {
|
||||
$state = vv_fb_parse_state('');
|
||||
$state['state'] = $ts['online'] === false ? 'OFFLINE' : 'UNREACHABLE';
|
||||
}
|
||||
|
||||
// Running containers
|
||||
if ($isMe) {
|
||||
$running = vv_fb_local_running();
|
||||
} elseif ($ip && $mySshKey && $ts['online']) {
|
||||
$running = vv_fb_remote_running($ip, $mySshKey);
|
||||
} else {
|
||||
$running = [];
|
||||
}
|
||||
|
||||
// Covers: for a 2-node setup, each covers the other
|
||||
// For N nodes this would need a different approach — for now, assume 2-node
|
||||
$covers = null;
|
||||
foreach ($hosts as $otherSlot => $otherHostname) {
|
||||
if ($otherSlot === $slot) continue;
|
||||
$coveringRaw = $raws[$slot] ?? '';
|
||||
$remoteRaw = $raws[$otherSlot] ?? '';
|
||||
$covers = [
|
||||
'slot' => $otherSlot,
|
||||
'id' => strtoupper($otherSlot),
|
||||
'hostname' => $otherHostname,
|
||||
] + vv_fb_covers($slot, $otherSlot, $coveringRaw, $remoteRaw);
|
||||
break; // 2-node only
|
||||
}
|
||||
|
||||
$nodes[] = [
|
||||
'slot' => $slot,
|
||||
'id' => strtoupper($slot),
|
||||
'hostname' => $hostname,
|
||||
'is_me' => $isMe,
|
||||
'ts_online' => $ts['online'],
|
||||
'state' => $state,
|
||||
'running' => $running,
|
||||
'covers' => $covers,
|
||||
];
|
||||
}
|
||||
|
||||
return [
|
||||
'ts' => time(),
|
||||
'fb_enabled' => $fbEnabled,
|
||||
'handback_req' => $handbackReq,
|
||||
'nodes' => $nodes,
|
||||
];
|
||||
}
|
||||
+8
-3
@@ -306,8 +306,11 @@ function vv_partner_state(): array {
|
||||
if ($h) $tsPeers[$h] = (bool)($peer['Online'] ?? false);
|
||||
}
|
||||
|
||||
$hostIds = array_filter(array_keys($vars), fn($k) => preg_match('/^HOST\d+$/', $k) && ($vars[$k] ?? '') !== '');
|
||||
sort($hostIds);
|
||||
|
||||
$hosts = [];
|
||||
foreach (['HOST1', 'HOST2', 'HOST3', 'HOST4'] as $id) {
|
||||
foreach ($hostIds as $id) {
|
||||
$hostname = $vars[$id] ?? '';
|
||||
if (!$hostname) continue;
|
||||
$isMe = strcasecmp($hostname, $myName) === 0;
|
||||
@@ -364,8 +367,10 @@ function vv_fallback_active(): array {
|
||||
$myName = trim(shell_exec('hostname -s') ?: '');
|
||||
|
||||
// Identify which HOST id we are
|
||||
$allHostIds = array_filter(array_keys($vars), fn($k) => preg_match('/^HOST\d+$/', $k) && ($vars[$k] ?? '') !== '');
|
||||
sort($allHostIds);
|
||||
$myId = null;
|
||||
foreach (['HOST1', 'HOST2', 'HOST3', 'HOST4'] as $id) {
|
||||
foreach ($allHostIds as $id) {
|
||||
if (strcasecmp($vars[$id] ?? '', $myName) === 0) { $myId = $id; break; }
|
||||
}
|
||||
if (!$myId) return [];
|
||||
@@ -383,7 +388,7 @@ function vv_fallback_active(): array {
|
||||
$rawConf = vv_read_conf_raw($confFile);
|
||||
|
||||
$result = [];
|
||||
foreach (['HOST1', 'HOST2', 'HOST3', 'HOST4'] as $covered) {
|
||||
foreach ($allHostIds as $covered) {
|
||||
if ($covered === $myId) continue;
|
||||
$coveredHostname = $vars[$covered] ?? '';
|
||||
if (!$coveredHostname) continue;
|
||||
@@ -0,0 +1,178 @@
|
||||
<?php
|
||||
// Partnership page data helpers
|
||||
|
||||
require_once __DIR__ . '/config.php';
|
||||
require_once __DIR__ . '/arrs.php'; // vv_arr_known_hosts(), vv_arr_scalar()
|
||||
|
||||
// ── Config ────────────────────────────────────────────────────────────────────
|
||||
|
||||
function vv_pt_config(): array {
|
||||
$v = vv_conf_vars();
|
||||
return [
|
||||
'enabled' => ($v['PARTNERSHIP_ENABLED'] ?? 'false') === 'true',
|
||||
'owner_host' => $v['PARTNERSHIP_OWNER_HOST'] ?? '',
|
||||
'sync_min' => (int)($v['PARTNERSHIP_SYNC_INTERVAL'] ?? 15),
|
||||
'grace_hours' => (int)($v['PARTNERSHIP_GRACE_HOURS'] ?? 6),
|
||||
'offline_threshold' => (int)($v['PARTNERSHIP_OFFLINE_THRESHOLD'] ?? 30),
|
||||
'remove_tailscale' => ($v['PARTNERSHIP_REMOVE_TAILSCALE'] ?? 'true') === 'true',
|
||||
'folderview3' => ($v['PARTNERSHIP_FOLDERVIEW3'] ?? 'false') === 'true',
|
||||
'tailscale_configured' => !empty($v['TAILSCALE_API_KEY']) && !empty($v['TAILSCALE_TAILNET']),
|
||||
];
|
||||
}
|
||||
|
||||
// ── State file parser ─────────────────────────────────────────────────────────
|
||||
|
||||
function vv_pt_read_db(string $path): array {
|
||||
if (!file_exists($path)) return [];
|
||||
$out = [];
|
||||
foreach (file($path, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES) as $line) {
|
||||
[$k, $v] = array_pad(explode('=', trim($line), 2), 2, '');
|
||||
if ($k) $out[trim($k)] = trim($v, '"\'');
|
||||
}
|
||||
return $out;
|
||||
}
|
||||
|
||||
// ── Tailscale peers ───────────────────────────────────────────────────────────
|
||||
|
||||
function vv_pt_ts_peers(): array {
|
||||
$raw = shell_exec('tailscale status --json 2>/dev/null') ?: '{}';
|
||||
$data = json_decode($raw, true) ?: [];
|
||||
$peers = [];
|
||||
|
||||
// Self
|
||||
$self = $data['Self'] ?? [];
|
||||
$selfLabel = strtolower(explode('.', $self['DNSName'] ?? '')[0]);
|
||||
if ($selfLabel) {
|
||||
$peers[$selfLabel] = [
|
||||
'online' => true,
|
||||
'active' => true,
|
||||
'ip' => $self['TailscaleIPs'][0] ?? null,
|
||||
];
|
||||
}
|
||||
|
||||
// Peers
|
||||
foreach ($data['Peer'] ?? [] as $peer) {
|
||||
$label = strtolower(explode('.', $peer['DNSName'] ?? '')[0]);
|
||||
if (!$label) continue;
|
||||
$peers[$label] = [
|
||||
'online' => (bool)($peer['Online'] ?? false),
|
||||
'active' => (bool)($peer['Active'] ?? false),
|
||||
'ip' => $peer['TailscaleIPs'][0] ?? null,
|
||||
];
|
||||
}
|
||||
return $peers;
|
||||
}
|
||||
|
||||
// ── SSH helper — run a single command on a remote host ────────────────────────
|
||||
|
||||
function vv_pt_ssh(string $ip, string $sshKey, string $cmd, int $timeout = 4): string {
|
||||
if (!$ip || !$sshKey || !file_exists($sshKey)) return '';
|
||||
$full = sprintf(
|
||||
'ssh -i %s -o ConnectTimeout=%d -o StrictHostKeyChecking=no -o BatchMode=yes root@%s %s 2>/dev/null',
|
||||
escapeshellarg($sshKey), $timeout, escapeshellarg($ip), escapeshellarg($cmd)
|
||||
);
|
||||
return shell_exec($full) ?: '';
|
||||
}
|
||||
|
||||
// ── System info ───────────────────────────────────────────────────────────────
|
||||
|
||||
function vv_pt_local_system(): array {
|
||||
$ver = '';
|
||||
if (file_exists('/etc/unraid-version')) {
|
||||
preg_match('/VERSION="([^"]+)"/', file_get_contents('/etc/unraid-version'), $m);
|
||||
$ver = $m[1] ?? '';
|
||||
}
|
||||
$uptime = 0;
|
||||
if (file_exists('/proc/uptime')) {
|
||||
$uptime = (int)explode(' ', file_get_contents('/proc/uptime'))[0];
|
||||
}
|
||||
return ['unraid_version' => $ver, 'uptime_sec' => $uptime];
|
||||
}
|
||||
|
||||
function vv_pt_remote_system(string $ip, string $sshKey): array {
|
||||
$out = vv_pt_ssh($ip, $sshKey,
|
||||
'printf "%s\nUPTIME:%s\n" "$(cat /etc/unraid-version 2>/dev/null)" "$(cat /proc/uptime 2>/dev/null)"');
|
||||
$ver = '';
|
||||
preg_match('/VERSION="([^"]+)"/', $out, $m);
|
||||
if ($m) $ver = $m[1];
|
||||
$uptime = 0;
|
||||
if (preg_match('/UPTIME:([\d.]+)/', $out, $m)) $uptime = (int)$m[1];
|
||||
return ['unraid_version' => $ver, 'uptime_sec' => $uptime];
|
||||
}
|
||||
|
||||
// ── Per-node data ─────────────────────────────────────────────────────────────
|
||||
|
||||
function vv_pt_nodes(): array {
|
||||
$currentHost = vv_detect_host();
|
||||
$hosts = vv_arr_known_hosts(); // ['host1' => 'hostname', ...]
|
||||
$vars = vv_conf_vars();
|
||||
$tsPeers = vv_pt_ts_peers();
|
||||
$ownerSlot = strtolower($vars['PARTNERSHIP_OWNER_HOST'] ?? '');
|
||||
|
||||
// SSH key for this host
|
||||
$myId = strtoupper($currentHost);
|
||||
$myRaw = vv_read_conf_raw($currentHost . '.conf');
|
||||
$mySshKey = vv_arr_scalar($myRaw, $myId . '_SSH_KEY');
|
||||
|
||||
$nodes = [];
|
||||
foreach ($hosts as $slot => $hostname) {
|
||||
$isMe = ($slot === $currentHost || $currentHost === 'unknown');
|
||||
$isOwner = (strtolower($ownerSlot) === $slot);
|
||||
|
||||
// Tailscale
|
||||
$tsLabel = strtolower($hostname);
|
||||
$ts = $tsPeers[$tsLabel] ?? ['online' => null, 'active' => false, 'ip' => null];
|
||||
|
||||
// Fallback state
|
||||
$fbState = 'UNKNOWN';
|
||||
$fbPath = '/boot/config/fallback_state.db';
|
||||
if ($isMe) {
|
||||
$fb = vv_pt_read_db($fbPath);
|
||||
$fbState = $fb['state'] ?? 'UNKNOWN';
|
||||
} elseif ($ts['online'] && $ts['ip'] && $mySshKey) {
|
||||
$out = vv_pt_ssh($ts['ip'], $mySshKey, 'cat /boot/config/fallback_state.db 2>/dev/null');
|
||||
if ($out) {
|
||||
$fb = [];
|
||||
foreach (explode("\n", $out) as $line) {
|
||||
[$k, $v] = array_pad(explode('=', trim($line), 2), 2, '');
|
||||
if ($k) $fb[trim($k)] = trim($v, '"\'');
|
||||
}
|
||||
$fbState = $fb['state'] ?? 'UNKNOWN';
|
||||
}
|
||||
}
|
||||
|
||||
// Partnership DB — local only (each server writes its own)
|
||||
$dbPath = "/boot/config/partnership_{$hostname}.db";
|
||||
$ptDb = vv_pt_read_db($dbPath);
|
||||
|
||||
// System info
|
||||
$system = $isMe
|
||||
? vv_pt_local_system()
|
||||
: ($ts['online'] && $ts['ip'] && $mySshKey ? vv_pt_remote_system($ts['ip'], $mySshKey) : []);
|
||||
|
||||
$nodes[] = [
|
||||
'slot' => $slot,
|
||||
'id' => strtoupper($slot),
|
||||
'hostname' => $hostname,
|
||||
'is_me' => $isMe,
|
||||
'is_owner' => $isOwner,
|
||||
'ts_online' => $ts['online'],
|
||||
'ts_active' => $ts['active'],
|
||||
'ts_ip' => $ts['ip'],
|
||||
'fallback' => $fbState,
|
||||
'partnership' => $ptDb,
|
||||
'system' => $system,
|
||||
];
|
||||
}
|
||||
return $nodes;
|
||||
}
|
||||
|
||||
// ── Entry point ───────────────────────────────────────────────────────────────
|
||||
|
||||
function vv_partnership_all(): array {
|
||||
return [
|
||||
'config' => vv_pt_config(),
|
||||
'nodes' => vv_pt_nodes(),
|
||||
'ts' => time(),
|
||||
];
|
||||
}
|
||||
+40
-39
@@ -8,6 +8,10 @@ require_once __DIR__ . '/confform.php';
|
||||
define('SCHEDULE_FILE', '/boot/config/plugins/varaverk/schedule.json');
|
||||
define('CRON_FILE', '/boot/config/plugins/varaverk/varaverk.cron');
|
||||
|
||||
function vv_pretty_label(string $slug): string {
|
||||
return ucwords(str_replace('_', ' ', $slug));
|
||||
}
|
||||
|
||||
function vv_schedule_load(): array {
|
||||
if (!file_exists(SCHEDULE_FILE)) return [];
|
||||
$data = json_decode(file_get_contents(SCHEDULE_FILE), true);
|
||||
@@ -89,7 +93,7 @@ function vv_cron_rebuild(array $schedule): bool {
|
||||
foreach ($schedule as $entry) {
|
||||
if (empty($entry['enabled']) || empty($entry['cron']) || empty($entry['id'])) continue;
|
||||
// Event-triggered jobs are handled by static event scripts, not cron.
|
||||
if (str_starts_with($entry['cron'], '@array_')) continue;
|
||||
if (str_starts_with($entry['cron'], 'array_')) continue;
|
||||
$id = $entry['id'];
|
||||
// When a child's orch is enabled it is the sole trigger — suppress independent cron.
|
||||
if (isset($childToOrch[$id]) && !empty($schedule[$childToOrch[$id]]['enabled'])) continue;
|
||||
@@ -235,7 +239,7 @@ function vv_custom_scripts(): array {
|
||||
$entry = $schedule[$rel] ?? [];
|
||||
$scripts[] = [
|
||||
'id' => $rel,
|
||||
'label' => basename($path, '.sh'),
|
||||
'label' => vv_pretty_label(basename($path, '.sh')),
|
||||
'desc' => vv_script_description($path),
|
||||
'enabled' => (bool)($entry['enabled'] ?? false),
|
||||
'cron' => $entry['cron'] ?? '',
|
||||
@@ -284,7 +288,7 @@ function vv_script_library(): array {
|
||||
$parts = explode('/', $rel);
|
||||
if (count($parts) < 2 || in_array($parts[0], $exclude)) continue;
|
||||
if (in_array($rel, $orchIds) || isset($confMap[$rel])) continue;
|
||||
$library[] = ['id' => $rel, 'label' => basename($rel, '.sh')];
|
||||
$library[] = ['id' => $rel, 'label' => vv_pretty_label(basename($rel, '.sh'))];
|
||||
}
|
||||
} catch (Exception $e) {}
|
||||
usort($library, fn($a, $b) => strcmp($a['id'], $b['id']));
|
||||
@@ -298,52 +302,35 @@ function vv_folders_load(): array {
|
||||
return is_array($f) ? $f : [];
|
||||
}
|
||||
|
||||
// Walk the scripts repo and return the job tree:
|
||||
// hardcoded array-event entries first, then cron-scheduled orchestrators
|
||||
// Walk the scripts repo and return the job tree.
|
||||
// Type is derived from the saved cron value: array_start / array_stop → 'event', else 'orchestrator'.
|
||||
// Well-known event orchs get their cron seeded from $eventDefaults when not yet in schedule.json.
|
||||
// Any orch or custom script can carry array_start / array_stop as its cron — the event scripts fire all of them.
|
||||
function vv_job_tree(): array {
|
||||
$scriptsDir = SCRIPTS_DIR;
|
||||
$schedule = vv_schedule_load();
|
||||
|
||||
// Hardcoded array-event entries — always present, trigger via Unraid event scripts
|
||||
$eventDefs = [
|
||||
['id' => 'Orchestrators/array_started.sh', 'cron' => '@array_start', 'label' => 'Array Starting'],
|
||||
['id' => 'Orchestrators/array_stopping.sh', 'cron' => '@array_stop', 'label' => 'Array Stopping'],
|
||||
// First-run seeds only — applied when schedule.json has no entry for these IDs yet.
|
||||
$eventDefaults = [
|
||||
'Orchestrators/array_started.sh' => 'array_start',
|
||||
'Orchestrators/array_stopping.sh' => 'array_stop',
|
||||
];
|
||||
$orchs = [];
|
||||
$eventIds = [];
|
||||
foreach ($eventDefs as $ev) {
|
||||
$id = $ev['id'];
|
||||
$path = "$scriptsDir/$id";
|
||||
$entry = $schedule[$id] ?? [];
|
||||
$eventIds[] = $id;
|
||||
$orchs[] = [
|
||||
'id' => $id,
|
||||
'label' => $ev['label'],
|
||||
'desc' => vv_script_description($path),
|
||||
'type' => 'event',
|
||||
'enabled' => (bool)($entry['enabled'] ?? false),
|
||||
'cron' => $ev['cron'],
|
||||
'log_enabled' => (bool)($entry['log_enabled'] ?? false),
|
||||
'suggested_cron' => '',
|
||||
'suggested_label' => '',
|
||||
'children' => vv_script_children($path, $schedule),
|
||||
];
|
||||
}
|
||||
|
||||
// Cron-scheduled orchestrators — discovered by glob, event entries excluded
|
||||
$orchPattern = "$scriptsDir/Orchestrators/*.sh";
|
||||
foreach (glob($orchPattern) ?: [] as $path) {
|
||||
$orchs = [];
|
||||
foreach (glob("$scriptsDir/Orchestrators/*.sh") ?: [] as $path) {
|
||||
$id = 'Orchestrators/' . basename($path);
|
||||
if (in_array($id, $eventIds, true)) continue;
|
||||
$entry = $schedule[$id] ?? ['enabled' => false, 'cron' => ''];
|
||||
$suggested = vv_script_suggested_cron($path);
|
||||
$default = $eventDefaults[$id] ?? '';
|
||||
$entry = $schedule[$id] ?? ['enabled' => false, 'cron' => $default];
|
||||
$cron = $entry['cron'] ?? $default;
|
||||
$isEvent = str_starts_with($cron, 'array_');
|
||||
$suggested = $isEvent ? ['cron' => '', 'label' => ''] : vv_script_suggested_cron($path);
|
||||
$orchs[] = [
|
||||
'id' => $id,
|
||||
'label' => basename($path, '.sh'),
|
||||
'label' => vv_pretty_label(basename($path, '.sh')),
|
||||
'desc' => vv_script_description($path),
|
||||
'type' => 'orchestrator',
|
||||
'type' => $isEvent ? 'event' : 'orchestrator',
|
||||
'enabled' => (bool)($entry['enabled'] ?? false),
|
||||
'cron' => $entry['cron'] ?? '',
|
||||
'cron' => $cron,
|
||||
'log_enabled' => (bool)($entry['log_enabled'] ?? false),
|
||||
'suggested_cron' => $suggested['cron'],
|
||||
'suggested_label' => $suggested['label'],
|
||||
@@ -351,6 +338,20 @@ function vv_job_tree(): array {
|
||||
'conf_arrays' => vv_orch_conf_arrays($path),
|
||||
];
|
||||
}
|
||||
|
||||
// Events first (array_start before array_stop), then alphabetical by label.
|
||||
usort($orchs, function($a, $b) {
|
||||
$ae = $a['type'] === 'event' ? 0 : 1;
|
||||
$be = $b['type'] === 'event' ? 0 : 1;
|
||||
if ($ae !== $be) return $ae - $be;
|
||||
if ($ae === 0) {
|
||||
$as = str_contains($a['cron'], 'start') ? 0 : 1;
|
||||
$bs = str_contains($b['cron'], 'start') ? 0 : 1;
|
||||
if ($as !== $bs) return $as - $bs;
|
||||
}
|
||||
return strcmp($a['label'], $b['label']);
|
||||
});
|
||||
|
||||
return $orchs;
|
||||
}
|
||||
|
||||
@@ -492,7 +493,7 @@ function vv_script_children(string $orchPath, array $schedule): array {
|
||||
$suggested = vv_script_suggested_cron("$scriptsDir/$rel");
|
||||
$children[] = [
|
||||
'id' => $rel,
|
||||
'label' => basename($rel, '.sh'),
|
||||
'label' => vv_pretty_label(basename($rel, '.sh')),
|
||||
'desc' => vv_script_description("$scriptsDir/$rel"),
|
||||
'type' => 'script',
|
||||
'enabled' => (bool)($entry['enabled'] ?? false),
|
||||
@@ -0,0 +1,341 @@
|
||||
<?php
|
||||
// Watchdog tab data helpers
|
||||
|
||||
require_once __DIR__ . '/config.php';
|
||||
require_once __DIR__ . '/partnership.php'; // vv_pt_ssh(), vv_pt_ts_peers()
|
||||
|
||||
// ── Conf array parser (bash arrays) ──────────────────────────────────────────
|
||||
|
||||
function vv_wd_bash_array(string $raw, string $varname): array {
|
||||
if (!preg_match('/^\s*' . preg_quote($varname, '/') . '\s*=\s*\(\s*(.*?)\s*\)/ms', $raw, $m))
|
||||
return [];
|
||||
preg_match_all('/"([^"]*)"/', $m[1], $items);
|
||||
return array_values(array_filter($items[1]));
|
||||
}
|
||||
|
||||
function vv_wd_bash_assoc(string $raw, string $varname): array {
|
||||
// declare -A VARNAME=( ["key"]=val ["key2"]=val2 )
|
||||
if (!preg_match('/^\s*declare\s+-A\s+' . preg_quote($varname, '/') . '\s*=\s*\(\s*(.*?)\s*\)/ms', $raw, $m))
|
||||
return [];
|
||||
preg_match_all('/\["([^"]+)"\]\s*=\s*"?([^"\s\)]*)"?/', $m[1], $pairs);
|
||||
$out = [];
|
||||
foreach ($pairs[1] as $i => $k) $out[$k] = $pairs[2][$i];
|
||||
return $out;
|
||||
}
|
||||
|
||||
function vv_wd_scalar(string $raw, string $varname): string {
|
||||
return preg_match('/^\s*' . preg_quote($varname, '/') . '\s*=\s*"?([^"\n]*)"?/m', $raw, $m)
|
||||
? trim($m[1]) : '';
|
||||
}
|
||||
|
||||
// ── State file parsers ────────────────────────────────────────────────────────
|
||||
|
||||
// Parses key:value format (with optional key=value mixed in)
|
||||
function vv_wd_parse_kv(string $text): array {
|
||||
$out = [];
|
||||
foreach (explode("\n", $text) as $line) {
|
||||
$line = trim($line);
|
||||
if (!$line) continue;
|
||||
if (str_contains($line, ':')) {
|
||||
[$k, $v] = explode(':', $line, 2);
|
||||
$out[trim($k)] = trim($v);
|
||||
} elseif (str_contains($line, '=')) {
|
||||
[$k, $v] = explode('=', $line, 2);
|
||||
$out[trim($k)] = trim($v, '"\'');
|
||||
}
|
||||
}
|
||||
return $out;
|
||||
}
|
||||
|
||||
// Restart log: "container|timestamp" one per line
|
||||
function vv_wd_parse_restart_log(string $text, int $windowSeconds = 86400): array {
|
||||
$now = time();
|
||||
$cutoff = $now - $windowSeconds;
|
||||
$entries = [];
|
||||
foreach (explode("\n", trim($text)) as $line) {
|
||||
$line = trim($line);
|
||||
if (!$line || !str_contains($line, '|')) continue;
|
||||
[$name, $ts] = explode('|', $line, 2);
|
||||
$ts = (int)$ts;
|
||||
if ($ts >= $cutoff) $entries[] = ['name' => trim($name), 'ts' => $ts];
|
||||
}
|
||||
usort($entries, fn($a, $b) => $b['ts'] - $a['ts']);
|
||||
return $entries;
|
||||
}
|
||||
|
||||
// Skip list: one container name per line
|
||||
function vv_wd_parse_skiplist(string $text): array {
|
||||
return array_values(array_filter(array_map('trim', explode("\n", $text))));
|
||||
}
|
||||
|
||||
// Reboot log: one timestamp per line
|
||||
function vv_wd_parse_reboot_log(string $text, int $windowHrs = 12): array {
|
||||
$cutoff = time() - ($windowHrs * 3600);
|
||||
$entries = [];
|
||||
foreach (explode("\n", trim($text)) as $line) {
|
||||
$ts = (int)trim($line);
|
||||
if ($ts > 0 && $ts >= $cutoff) $entries[] = $ts;
|
||||
}
|
||||
rsort($entries);
|
||||
return $entries;
|
||||
}
|
||||
|
||||
// ── Local system snapshot ─────────────────────────────────────────────────────
|
||||
|
||||
function vv_wd_local_system(): array {
|
||||
// RAM
|
||||
$memRaw = file_exists('/proc/meminfo') ? file_get_contents('/proc/meminfo') : '';
|
||||
$memTotal = 0; $memAvail = 0;
|
||||
if (preg_match('/^MemTotal:\s+(\d+)/m', $memRaw, $m)) $memTotal = (int)$m[1] * 1024;
|
||||
if (preg_match('/^MemAvailable:\s+(\d+)/m', $memRaw, $m)) $memAvail = (int)$m[1] * 1024;
|
||||
|
||||
// Load + cores
|
||||
$loadRaw = file_exists('/proc/loadavg') ? file_get_contents('/proc/loadavg') : '0 0 0';
|
||||
$loadParts = explode(' ', trim($loadRaw));
|
||||
$load1 = (float)($loadParts[0] ?? 0);
|
||||
$cores = (int)(trim(shell_exec('nproc 2>/dev/null') ?: '1'));
|
||||
|
||||
// Uptime
|
||||
$uptimeRaw = file_exists('/proc/uptime') ? file_get_contents('/proc/uptime') : '0';
|
||||
$uptime = (int)explode(' ', $uptimeRaw)[0];
|
||||
|
||||
// Docker daemon alive
|
||||
$daemonOk = (trim(shell_exec('docker info >/dev/null 2>&1; echo $?') ?: '1') === '0');
|
||||
|
||||
// OOM count (from stability watchdog OOM file — just the prev cycle count)
|
||||
$oomFile = '/tmp/system_watchdog_oom.db';
|
||||
$oomCount = file_exists($oomFile) ? (int)trim(file_get_contents($oomFile)) : 0;
|
||||
|
||||
return [
|
||||
'mem_total' => $memTotal,
|
||||
'mem_avail' => $memAvail,
|
||||
'load1' => $load1,
|
||||
'cores' => $cores,
|
||||
'uptime' => $uptime,
|
||||
'daemon_ok' => $daemonOk,
|
||||
'oom_count' => $oomCount,
|
||||
];
|
||||
}
|
||||
|
||||
// ── Local state files ─────────────────────────────────────────────────────────
|
||||
|
||||
function vv_wd_local_states(string $restartLogPath): array {
|
||||
$rwRaw = @file_get_contents('/tmp/resource_watchdog_state.db') ?: '';
|
||||
$dockRaw = @file_get_contents('/tmp/container_watchdog_state.db') ?: '';
|
||||
$skipRaw = @file_get_contents('/boot/config/system_watchdog_failed.db') ?: '';
|
||||
$sysRaw = @file_get_contents('/tmp/system_watchdog_state.db') ?: '';
|
||||
$rebootRaw = @file_get_contents('/boot/config/system_watchdog_reboots.db')?: '';
|
||||
$restartRaw= @file_get_contents($restartLogPath) ?: '';
|
||||
|
||||
$rw = vv_wd_parse_kv($rwRaw);
|
||||
$dock = vv_wd_parse_kv($dockRaw);
|
||||
$sys = vv_wd_parse_kv($sysRaw);
|
||||
|
||||
// Container strikes: everything in docker state that isn't a flag
|
||||
$strikes = [];
|
||||
foreach ($dock as $k => $v) {
|
||||
if ($k !== 'daemon_strikes' && $k !== 'daemon_restarted_flag' && (int)$v > 0)
|
||||
$strikes[$k] = (int)$v;
|
||||
}
|
||||
|
||||
// Stability strikes: everything in sys state that isn't a flag key
|
||||
$sysStrikes = [];
|
||||
foreach ($sys as $k => $v) {
|
||||
if (!str_contains($k, '=') && (int)$v > 0)
|
||||
$sysStrikes[$k] = (int)$v;
|
||||
}
|
||||
|
||||
return [
|
||||
'rw_level' => (int)($rw['rm_action_level'] ?? 0),
|
||||
'rw_recover' => (int)($rw['rm_recover_cycles'] ?? 0),
|
||||
'rw_paused' => array_filter(explode(',', $rw['rm_paused_containers'] ?? '')),
|
||||
'rw_stopped' => array_filter(explode(',', $rw['rm_stopped_containers'] ?? '')),
|
||||
'mem_shutdown' => ($rw['mem_shutdown_active'] ?? 'false') === 'true',
|
||||
'daemon_strikes' => (int)($dock['daemon_strikes'] ?? 0),
|
||||
'daemon_restart' => ($dock['daemon_restarted_flag'] ?? 'false') === 'true',
|
||||
'ctr_strikes' => $strikes,
|
||||
'skiplist' => vv_wd_parse_skiplist($skipRaw),
|
||||
'sys_strikes' => $sysStrikes,
|
||||
'reboots' => vv_wd_parse_reboot_log($rebootRaw),
|
||||
'restarts' => vv_wd_parse_restart_log($restartRaw),
|
||||
];
|
||||
}
|
||||
|
||||
// ── Remote data via SSH ───────────────────────────────────────────────────────
|
||||
|
||||
function vv_wd_remote_data(string $ip, string $sshKey, string $restartLogPath): array {
|
||||
// Bundle into one SSH call
|
||||
$cmd = "printf 'UPTIME:%s\nLOAD:%s\nCORES:%s\nMEMTOTAL:%s\nMEMAVAIL:%s\nDAEMON:%s\nOOM:%s\n---RW---\n%s\n---DOCK---\n%s\n---SKIP---\n%s\n---SYS---\n%s\n---REBOOT---\n%s\n---RESTART---\n%s\n' "
|
||||
. '"$(cat /proc/uptime|cut -d\" \" -f1)" '
|
||||
. '"$(cat /proc/loadavg|cut -d\" \" -f1)" '
|
||||
. '"$(nproc)" '
|
||||
. '"$(grep -m1 MemTotal /proc/meminfo|awk \"{print \\\$2}\")" '
|
||||
. '"$(grep -m1 MemAvailable /proc/meminfo|awk \"{print \\\$2}\")" '
|
||||
. '"$(docker info >/dev/null 2>&1 && echo ok || echo err)" '
|
||||
. '"$(cat /tmp/system_watchdog_oom.db 2>/dev/null||echo 0)" '
|
||||
. '"$(cat /tmp/resource_watchdog_state.db 2>/dev/null)" '
|
||||
. '"$(cat /tmp/container_watchdog_state.db 2>/dev/null)" '
|
||||
. '"$(cat /boot/config/system_watchdog_failed.db 2>/dev/null)" '
|
||||
. '"$(cat /tmp/system_watchdog_state.db 2>/dev/null)" '
|
||||
. '"$(cat /boot/config/system_watchdog_reboots.db 2>/dev/null)" '
|
||||
. '"$(cat ' . escapeshellarg($restartLogPath) . ' 2>/dev/null)"';
|
||||
|
||||
$out = vv_pt_ssh($ip, $sshKey, $cmd, 8);
|
||||
if (!$out) return null;
|
||||
|
||||
// Parse sections
|
||||
$sections = preg_split('/^---\w+---$/m', $out);
|
||||
$header = $sections[0] ?? '';
|
||||
$rwRaw = $sections[1] ?? '';
|
||||
$dockRaw = $sections[2] ?? '';
|
||||
$skipRaw = $sections[3] ?? '';
|
||||
$sysRaw = $sections[4] ?? '';
|
||||
$rebootRaw= $sections[5] ?? '';
|
||||
$restartRaw=$sections[6] ?? '';
|
||||
|
||||
// Parse header lines
|
||||
$hdr = [];
|
||||
foreach (explode("\n", $header) as $line) {
|
||||
if (preg_match('/^(\w+):(.*)$/', trim($line), $m)) $hdr[$m[1]] = trim($m[2]);
|
||||
}
|
||||
|
||||
$rw = vv_wd_parse_kv($rwRaw);
|
||||
$dock = vv_wd_parse_kv($dockRaw);
|
||||
$sys = vv_wd_parse_kv($sysRaw);
|
||||
|
||||
$strikes = [];
|
||||
foreach ($dock as $k => $v) {
|
||||
if ($k !== 'daemon_strikes' && $k !== 'daemon_restarted_flag' && (int)$v > 0)
|
||||
$strikes[$k] = (int)$v;
|
||||
}
|
||||
$sysStrikes = [];
|
||||
foreach ($sys as $k => $v) {
|
||||
if (!str_contains($k, '=') && (int)$v > 0) $sysStrikes[$k] = (int)$v;
|
||||
}
|
||||
|
||||
$memTotal = (int)($hdr['MEMTOTAL'] ?? 0) * 1024;
|
||||
$memAvail = (int)($hdr['MEMAVAIL'] ?? 0) * 1024;
|
||||
|
||||
return [
|
||||
'system' => [
|
||||
'mem_total' => $memTotal,
|
||||
'mem_avail' => $memAvail,
|
||||
'load1' => (float)($hdr['LOAD'] ?? 0),
|
||||
'cores' => (int)($hdr['CORES'] ?? 1),
|
||||
'uptime' => (int)($hdr['UPTIME'] ?? 0),
|
||||
'daemon_ok' => ($hdr['DAEMON'] ?? '') === 'ok',
|
||||
'oom_count' => (int)($hdr['OOM'] ?? 0),
|
||||
],
|
||||
'states' => [
|
||||
'rw_level' => (int)($rw['rm_action_level'] ?? 0),
|
||||
'rw_recover' => (int)($rw['rm_recover_cycles'] ?? 0),
|
||||
'rw_paused' => array_filter(explode(',', $rw['rm_paused_containers'] ?? '')),
|
||||
'rw_stopped' => array_filter(explode(',', $rw['rm_stopped_containers'] ?? '')),
|
||||
'mem_shutdown' => ($rw['mem_shutdown_active'] ?? 'false') === 'true',
|
||||
'daemon_strikes'=> (int)($dock['daemon_strikes'] ?? 0),
|
||||
'daemon_restart'=> ($dock['daemon_restarted_flag'] ?? 'false') === 'true',
|
||||
'ctr_strikes' => $strikes,
|
||||
'skiplist' => vv_wd_parse_skiplist($skipRaw),
|
||||
'sys_strikes' => $sysStrikes,
|
||||
'reboots' => vv_wd_parse_reboot_log($rebootRaw),
|
||||
'restarts' => vv_wd_parse_restart_log($restartRaw),
|
||||
],
|
||||
];
|
||||
}
|
||||
|
||||
// ── Config inventory ──────────────────────────────────────────────────────────
|
||||
|
||||
function vv_wd_node_config(string $slot, string $raw, string $masterRaw): array {
|
||||
$id = strtoupper($slot);
|
||||
return [
|
||||
'monitored' => vv_wd_bash_assoc($raw, "{$id}_WATCHDOG_CONTAINERS"),
|
||||
'urls' => vv_wd_bash_assoc($raw, "{$id}_WATCHDOG_CONTAINER_URLS"),
|
||||
'required' => vv_wd_bash_array($raw, "{$id}_WATCHDOG_REQUIRED_CONTAINERS"),
|
||||
'ignore' => vv_wd_bash_array($raw, "{$id}_WATCHDOG_SCAN_IGNORE"),
|
||||
'pause_list' => vv_wd_bash_array($raw, "{$id}_RW_PAUSE_CONTAINERS"),
|
||||
'stop_list' => vv_wd_bash_array($raw, "{$id}_RW_STOP_CONTAINERS"),
|
||||
'critical' => vv_wd_bash_array($masterRaw, "RW_CRITICAL_CONTAINERS"),
|
||||
];
|
||||
}
|
||||
|
||||
// ── Entry point ───────────────────────────────────────────────────────────────
|
||||
|
||||
function vv_wd_all(): array {
|
||||
$currentHost = vv_detect_host();
|
||||
$tsPeers = vv_pt_ts_peers();
|
||||
$masterRaw = vv_read_conf_raw('master.conf');
|
||||
$restartLog = '/mnt/user/appdata/unraid_scripts/data/container_restart_history.db';
|
||||
|
||||
// Config thresholds from master.conf
|
||||
$cfg = [
|
||||
'rw_soft_gb' => (float)(vv_wd_scalar($masterRaw, 'RW_RAM_SOFT_GB') ?: 12),
|
||||
'rw_medium_gb' => (float)(vv_wd_scalar($masterRaw, 'RW_RAM_MEDIUM_GB') ?: 8),
|
||||
'rw_hard_gb' => (float)(vv_wd_scalar($masterRaw, 'RW_RAM_HARD_GB') ?: 6),
|
||||
'rw_recover_gb' => (float)(vv_wd_scalar($masterRaw, 'RW_RAM_RECOVER_GB') ?: 20),
|
||||
'rw_load_soft' => (float)(vv_wd_scalar($masterRaw, 'RW_LOAD_SOFT_MULTIPLIER') ?: 2.0),
|
||||
'rw_load_med' => (float)(vv_wd_scalar($masterRaw, 'RW_LOAD_MEDIUM_MULTIPLIER') ?: 3.0),
|
||||
'sys_mem_gb' => (float)(vv_wd_scalar($masterRaw, 'SYS_WATCHDOG_MEM_GB') ?: 4),
|
||||
'sys_strikes' => (int)(vv_wd_scalar($masterRaw, 'SYS_WATCHDOG_STRIKE_LIMIT') ?: 2),
|
||||
'reboot_limit' => (int)(vv_wd_scalar($masterRaw, 'SYS_WATCHDOG_REBOOT_LIMIT') ?: 3),
|
||||
'reboot_window' => (int)(vv_wd_scalar($masterRaw, 'SYS_WATCHDOG_REBOOT_WINDOW_HRS') ?: 12),
|
||||
'restart_limit' => (int)(vv_wd_scalar($masterRaw, 'WATCHDOG_CONTAINER_RESTART_LIMIT') ?: 3),
|
||||
'startup_grace' => (int)(vv_wd_scalar($masterRaw, 'WATCHDOG_STARTUP_GRACE') ?: 600),
|
||||
'soft_mem_pct' => (int)(vv_wd_scalar($masterRaw, 'SOFT_MEM_THRESHOLD') ?: 80),
|
||||
'soft_cpu_pct' => (int)(vv_wd_scalar($masterRaw, 'SOFT_CPU_THRESHOLD') ?: 80),
|
||||
'hard_cpu_pct' => (int)(vv_wd_scalar($masterRaw, 'HARD_CPU_THRESHOLD') ?: 85),
|
||||
'cpu_fail_lim' => (int)(vv_wd_scalar($masterRaw, 'CPU_FAIL_LIMIT') ?: 2),
|
||||
'resp_fail_lim' => (int)(vv_wd_scalar($masterRaw, 'RESP_FAIL_LIMIT') ?: 2),
|
||||
];
|
||||
|
||||
// SSH key from current host conf
|
||||
$myId = strtoupper($currentHost);
|
||||
$myRaw = vv_read_conf_raw($currentHost . '.conf');
|
||||
$mySshKey = vv_wd_scalar($myRaw, $myId . '_SSH_KEY');
|
||||
|
||||
// Known hosts
|
||||
preg_match_all('/^\s*(HOST(\d+))(?:_NAME)?\s*=\s*["\']?(\S+?)["\']?\s*$/m', $masterRaw, $m);
|
||||
$hosts = [];
|
||||
foreach ($m[1] as $i => $key) {
|
||||
$hosts['host' . $m[2][$i]] = trim($m[3][$i]);
|
||||
}
|
||||
ksort($hosts);
|
||||
if (!$hosts) $hosts = ['host1' => 'HOST1'];
|
||||
|
||||
$nodes = [];
|
||||
foreach ($hosts as $slot => $hostname) {
|
||||
$isMe = ($slot === $currentHost || $currentHost === 'unknown');
|
||||
$tsLabel = strtolower($hostname);
|
||||
$ts = $tsPeers[$tsLabel] ?? ['online' => null, 'active' => false, 'ip' => null];
|
||||
$ip = $ts['ip'] ?? null;
|
||||
$raw = vv_read_conf_raw($slot . '.conf');
|
||||
|
||||
if ($isMe) {
|
||||
$system = vv_wd_local_system();
|
||||
$states = vv_wd_local_states($restartLog);
|
||||
} elseif ($ip && $mySshKey && $ts['online']) {
|
||||
$remote = vv_wd_remote_data($ip, $mySshKey, $restartLog);
|
||||
$system = $remote['system'] ?? null;
|
||||
$states = $remote['states'] ?? null;
|
||||
} else {
|
||||
$system = null;
|
||||
$states = null;
|
||||
}
|
||||
|
||||
$nodes[] = [
|
||||
'slot' => $slot,
|
||||
'id' => strtoupper($slot),
|
||||
'hostname' => $hostname,
|
||||
'is_me' => $isMe,
|
||||
'ts_online' => $ts['online'],
|
||||
'system' => $system,
|
||||
'states' => $states,
|
||||
'config' => vv_wd_node_config($slot, $raw, $masterRaw),
|
||||
];
|
||||
}
|
||||
|
||||
return [
|
||||
'ts' => time(),
|
||||
'cfg' => $cfg,
|
||||
'nodes' => $nodes,
|
||||
];
|
||||
}
|
||||
@@ -0,0 +1,231 @@
|
||||
<style>
|
||||
.vv-arr-card { background:#161616;border:1px solid #2a2a2a;border-radius:6px;padding:10px;min-width:0; }
|
||||
.vv-arr-head { display:flex;justify-content:space-between;align-items:center;margin-bottom:8px; }
|
||||
.vv-arr-name { font-size:12px;font-weight:bold;text-transform:uppercase;color:#888;letter-spacing:.05em; }
|
||||
.vv-arr-status{ display:flex;align-items:center;gap:5px; }
|
||||
.vv-arr-dot { width:7px;height:7px;border-radius:50%;flex-shrink:0; }
|
||||
.vv-arr-ver { font-size:10px;color:#555; }
|
||||
.vv-arr-row { display:flex;justify-content:space-between;align-items:baseline;gap:8px;margin:2px 0; }
|
||||
.vv-arr-lbl { font-size:11px;color:#555;white-space:nowrap; }
|
||||
.vv-arr-val { font-size:12px;color:#bbb;text-align:right; }
|
||||
.vv-arr-sep { border:none;border-top:1px solid #222;margin:6px 0; }
|
||||
.vv-arr-sub { font-size:10px;color:#444;margin:1px 0 1px 2px; }
|
||||
.vv-arr-sub.warn { color:#e57; }
|
||||
.vv-arr-path { font-size:10px;color:#393939;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;margin-top:4px; }
|
||||
.vv-arr-node { grid-column:1/-1; }
|
||||
.vv-arr-node-head { display:flex;align-items:baseline;gap:8px;margin-bottom:10px; }
|
||||
.vv-arr-node-title{ font-size:12px;font-weight:bold;color:#666;letter-spacing:.06em;text-transform:uppercase; }
|
||||
.vv-arr-node-host { font-size:11px;color:#3a3a3a; }
|
||||
.vv-arr-cards { display:grid;grid-template-columns:repeat(auto-fill,minmax(180px,1fr));gap:10px; }
|
||||
.vv-arr-stat-row { display:flex;gap:16px;flex-wrap:wrap; }
|
||||
.vv-arr-stat { display:flex;flex-direction:column;gap:1px; }
|
||||
.vv-arr-stat-val { font-size:15px;font-weight:bold;color:#ccc; }
|
||||
.vv-arr-stat-lbl { font-size:10px;color:#555; }
|
||||
</style>
|
||||
|
||||
<div style="display:flex;justify-content:space-between;align-items:center;margin-bottom:12px;padding:0 2px;">
|
||||
<span style="font-size:13px;font-weight:bold;color:#888;text-transform:uppercase;letter-spacing:.06em;">Arrs</span>
|
||||
<span style="font-size:11px;color:#3a3a3a;" id="vv-arrs-ts"></span>
|
||||
</div>
|
||||
|
||||
<div id="vv-arrs-grid" style="display:grid;grid-template-columns:repeat(8,1fr);gap:12px;">
|
||||
<div style="grid-column:1/-1;color:#444;font-size:12px;padding:16px 0;text-align:center;">Loading…</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
(function() {
|
||||
|
||||
function _relTime(ts) {
|
||||
if (!ts) return '—';
|
||||
const d = Math.floor(Date.now() / 1000) - ts;
|
||||
if (d < 60) return 'just now';
|
||||
if (d < 3600) return Math.floor(d / 60) + 'm ago';
|
||||
if (d < 86400) return Math.floor(d / 3600) + 'h ago';
|
||||
if (d < 172800) return 'yesterday';
|
||||
return Math.floor(d / 86400) + 'd ago';
|
||||
}
|
||||
|
||||
function _n(v) { return v == null ? '—' : Number(v).toLocaleString(); }
|
||||
|
||||
function _bytes(b) {
|
||||
if (!b) return '—';
|
||||
const u = ['B','KB','MB','GB','TB']; let i = 0;
|
||||
while (b >= 1024 && i < u.length - 1) { b /= 1024; i++; }
|
||||
return b.toFixed(1) + ' ' + u[i];
|
||||
}
|
||||
|
||||
function _row(lbl, val) {
|
||||
return `<div class="vv-arr-row"><span class="vv-arr-lbl">${lbl}</span><span class="vv-arr-val">${val}</span></div>`;
|
||||
}
|
||||
|
||||
function _arrCard(arr) {
|
||||
const names = {sonarr:'Sonarr', radarr:'Radarr', lidarr:'Lidarr'};
|
||||
const name = names[arr.type] || arr.type;
|
||||
const online = arr.online;
|
||||
const dotCol = online ? '#4caf50' : (arr.remote ? '#444' : '#c62828');
|
||||
const verStr = online ? (arr.version || 'online') : (arr.remote ? 'not reachable' : 'offline');
|
||||
|
||||
let body = '';
|
||||
|
||||
if (online) {
|
||||
// Library
|
||||
if (arr.type === 'sonarr') {
|
||||
body += _row('Series', _n(arr.total));
|
||||
body += _row('Monitored', _n(arr.monitored));
|
||||
body += _row('Episodes', _n(arr.episodes));
|
||||
} else if (arr.type === 'radarr') {
|
||||
body += _row('Movies', _n(arr.total));
|
||||
body += _row('Monitored', _n(arr.monitored));
|
||||
body += _row('Files', _n(arr.files));
|
||||
} else if (arr.type === 'lidarr') {
|
||||
body += _row('Artists', _n(arr.total));
|
||||
body += _row('Monitored', _n(arr.monitored));
|
||||
if (arr.albums) body += _row('Albums', _n(arr.albums));
|
||||
}
|
||||
|
||||
// Disk — show largest entry
|
||||
if (arr.disk && arr.disk.length) {
|
||||
const d = arr.disk.reduce((a,b) => (b.totalSpace||0) > (a.totalSpace||0) ? b : a);
|
||||
if (d.freeSpace && d.totalSpace) {
|
||||
body += _row('Free', _bytes(d.freeSpace) + ' / ' + _bytes(d.totalSpace));
|
||||
}
|
||||
}
|
||||
|
||||
// Queue
|
||||
const q = arr.queue || {};
|
||||
const qDl = q.dl || 0;
|
||||
const qWrn = q.warn || 0;
|
||||
const qErr = q.err || 0;
|
||||
const qTot = qDl + qWrn + qErr;
|
||||
const qCol = qErr ? '#f44' : (qWrn ? '#ff9800' : '#bbb');
|
||||
body += `<hr class="vv-arr-sep">`;
|
||||
const qStr = qTot === 0
|
||||
? `<span style="color:#3a3a3a">idle</span>`
|
||||
: `<span style="color:${qCol}">${qDl}↓ ${qWrn}⚠ ${qErr}✗</span>`;
|
||||
body += _row('Queue', qStr);
|
||||
|
||||
// Health
|
||||
if (arr.health && arr.health.length) {
|
||||
const col = arr.health.some(h => h.type === 'error') ? '#f44' : '#ff9800';
|
||||
body += `<div class="vv-arr-sub warn" style="color:${col}">` +
|
||||
arr.health.slice(0,2).map(h => h.message || h.type).join('<br>') + '</div>';
|
||||
}
|
||||
|
||||
// Cleanup
|
||||
const cl = arr.cleanup || {};
|
||||
if (cl.last_run) {
|
||||
body += `<hr class="vv-arr-sep">`;
|
||||
const clOk = cl.status === 'ok'
|
||||
? '<span style="color:#4caf50">✓</span>'
|
||||
: '<span style="color:#f44">✗</span>';
|
||||
body += _row('Cleanup', _relTime(cl.last_run) + ' ' + clOk);
|
||||
if (cl.tracked != null) {
|
||||
body += `<div class="vv-arr-sub">${_n(cl.tracked)} files · ${_n(cl.total)} items</div>`;
|
||||
const orph = cl.orphans || 0, junk = cl.junk || 0;
|
||||
if (orph || junk) {
|
||||
body += `<div class="vv-arr-sub warn">${orph} orphans · ${junk} junk</div>`;
|
||||
} else {
|
||||
body += `<div class="vv-arr-sub">0 orphans · 0 junk</div>`;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Discovery
|
||||
const disc = arr.discovery || {};
|
||||
if (disc.last_run) {
|
||||
if (!cl.last_run) body += `<hr class="vv-arr-sep">`;
|
||||
const added = disc.added != null ? ` <span style="color:#6fcf97">+${disc.added}</span>` : '';
|
||||
body += _row('Discovery', _relTime(disc.last_run) + added);
|
||||
}
|
||||
}
|
||||
|
||||
body += `<div class="vv-arr-path">${arr.root || '—'}</div>`;
|
||||
|
||||
return `<div class="vv-arr-card">
|
||||
<div class="vv-arr-head">
|
||||
<span class="vv-arr-name">${name}</span>
|
||||
<span class="vv-arr-status">
|
||||
<span class="vv-arr-dot" style="background:${dotCol}"></span>
|
||||
<span class="vv-arr-ver">${verStr}</span>
|
||||
</span>
|
||||
</div>
|
||||
${body}
|
||||
</div>`;
|
||||
}
|
||||
|
||||
function _nodeSection(node, ownerHost) {
|
||||
const isOwner = node.host === ownerHost;
|
||||
const badge = isOwner ? ' <span style="color:#3a4a3a;font-size:10px;">owner</span>' : '';
|
||||
const cards = node.arrs.map(_arrCard).join('');
|
||||
return `<div class="vv-card vv-arr-node">
|
||||
<div class="vv-arr-node-head">
|
||||
<span class="vv-arr-node-title">${node.host.toUpperCase()}</span>
|
||||
<span class="vv-arr-node-host">${node.name}${badge}</span>
|
||||
</div>
|
||||
<div class="vv-arr-cards">${cards}</div>
|
||||
</div>`;
|
||||
}
|
||||
|
||||
function _syncCard(sync) {
|
||||
let body = '';
|
||||
if (!sync.last_run) {
|
||||
body = '<div style="color:#3a3a3a;font-size:12px;">Never run</div>';
|
||||
} else {
|
||||
const ok = sync.status === 'ok'
|
||||
? '<span style="color:#4caf50">✓</span>'
|
||||
: '<span style="color:#f44">✗</span>';
|
||||
body += _row('Last run', _relTime(sync.last_run) + ' ' + ok);
|
||||
if (sync.added != null) body += _row('Added', `<span style="color:#6fcf97">+${_n(sync.added)}</span>`);
|
||||
if (sync.nodes != null) body += _row('Nodes', _n(sync.nodes));
|
||||
}
|
||||
if (sync.blocklist_count != null) {
|
||||
body += `<hr class="vv-arr-sep">`;
|
||||
body += _row('Blocklist', _n(sync.blocklist_count) + ' entries');
|
||||
}
|
||||
return `<div class="vv-card" style="grid-column:span 4;">
|
||||
<h3>Arrs Sync</h3>${body}</div>`;
|
||||
}
|
||||
|
||||
function _recoveryCard(rec) {
|
||||
let body = '';
|
||||
if (!rec.last_run) {
|
||||
body = '<div style="color:#3a3a3a;font-size:12px;">Never run</div>';
|
||||
} else {
|
||||
const ok = rec.status === 'ok'
|
||||
? '<span style="color:#4caf50">✓</span>'
|
||||
: '<span style="color:#f44">✗</span>';
|
||||
body += _row('Last run', _relTime(rec.last_run) + ' ' + ok);
|
||||
body += _row('Fixed', _n(rec.fixed));
|
||||
body += _row('Re-searched',_n(rec.searched));
|
||||
}
|
||||
return `<div class="vv-card" style="grid-column:span 4;">
|
||||
<h3>Failed / Stalled Recovery</h3>${body}</div>`;
|
||||
}
|
||||
|
||||
function _render(data) {
|
||||
let html = '';
|
||||
for (const node of (data.nodes || [])) {
|
||||
html += _nodeSection(node, data.host);
|
||||
}
|
||||
html += _syncCard(data.sync || {});
|
||||
html += _recoveryCard(data.recovery || {});
|
||||
|
||||
document.getElementById('vv-arrs-grid').innerHTML = html || '<div style="grid-column:1/-1;color:#444;font-size:12px;padding:16px 0;text-align:center;">No arrs configured.</div>';
|
||||
|
||||
const ts = data.ts ? new Date(data.ts * 1000).toLocaleString([],
|
||||
{month:'numeric',day:'numeric',year:'numeric',hour:'2-digit',minute:'2-digit',second:'2-digit'}) : '';
|
||||
document.getElementById('vv-arrs-ts').textContent = ts ? 'Updated: ' + ts : '';
|
||||
}
|
||||
|
||||
function vvArrsLoad() {
|
||||
fetch('/plugins/varaverk/api/arrs.php')
|
||||
.then(r => r.json())
|
||||
.then(_render)
|
||||
.catch(() => {});
|
||||
}
|
||||
|
||||
vvArrsLoad();
|
||||
setInterval(vvArrsLoad, 60000);
|
||||
|
||||
})();
|
||||
</script>
|
||||
@@ -0,0 +1,243 @@
|
||||
<style>
|
||||
.vv-fb-active { background:#1a1200;border:1px solid #5a3800;border-radius:6px;padding:12px 14px; }
|
||||
.vv-fb-active-h { display:flex;align-items:baseline;gap:10px;margin-bottom:8px; }
|
||||
.vv-fb-badge { font-size:11px;font-weight:bold;letter-spacing:.06em;padding:2px 7px;border-radius:3px;flex-shrink:0; }
|
||||
.vv-fb-badge.fb { background:#5a3800;color:#ffb74d; }
|
||||
.vv-fb-badge.norm { background:#1a2a1a;color:#4caf50; }
|
||||
.vv-fb-badge.dark { background:#2a1a2a;color:#9c27b0; }
|
||||
.vv-fb-badge.nonet{ background:#1a1a2a;color:#5c7cfa; }
|
||||
.vv-fb-meta { display:flex;gap:18px;flex-wrap:wrap;margin-bottom:10px; }
|
||||
.vv-fb-meta-item{ display:flex;flex-direction:column;gap:1px; }
|
||||
.vv-fb-meta-val { font-size:17px;font-weight:bold;color:#ffb74d; }
|
||||
.vv-fb-meta-lbl { font-size:10px;color:#5a4020; }
|
||||
.vv-fb-ctrs { display:flex;gap:6px;flex-wrap:wrap;margin-top:6px; }
|
||||
.vv-fb-ctr { font-size:11px;padding:2px 8px;border-radius:3px;background:#2a1e00;color:#ffb74d;border:1px solid #4a2e00; }
|
||||
.vv-fb-ctr.running { background:#1a2a1a;color:#6fcf97;border-color:#2d4a2d; }
|
||||
.vv-fb-ctr.stopped { background:#2a1a1a;color:#e57;border-color:#4a2020;opacity:.7; }
|
||||
.vv-fb-node { grid-column:span 4; }
|
||||
.vv-fb-node-h { display:flex;align-items:baseline;gap:8px;margin-bottom:10px; }
|
||||
.vv-fb-node-id { font-size:12px;font-weight:bold;color:#666;letter-spacing:.06em;text-transform:uppercase; }
|
||||
.vv-fb-node-nm { font-size:11px;color:#3a3a3a; }
|
||||
.vv-fb-arrow { font-size:11px;color:#333; }
|
||||
.vv-fb-covers { font-size:11px;color:#3a3a3a; }
|
||||
.vv-fb-tier { margin-bottom:8px; }
|
||||
.vv-fb-tier-h { display:flex;align-items:baseline;gap:6px;margin-bottom:4px; }
|
||||
.vv-fb-tier-lbl { font-size:11px;font-weight:bold;color:#555;text-transform:uppercase;letter-spacing:.04em; }
|
||||
.vv-fb-tier-delay { font-size:10px;color:#3a3a3a; }
|
||||
.vv-fb-tier-pills { display:flex;gap:5px;flex-wrap:wrap; }
|
||||
.vv-fb-pill { font-size:11px;padding:2px 7px;border-radius:3px;background:#1e1e1e;color:#777;border:1px solid #2a2a2a; }
|
||||
.vv-fb-pill.active-t { background:#1a2010;color:#8bc34a;border-color:#2d3a1d; }
|
||||
.vv-fb-pill.empty { color:#333;font-style:italic; }
|
||||
.vv-fb-state-dot { width:6px;height:6px;border-radius:50%;flex-shrink:0;margin-top:3px; }
|
||||
.vv-fb-sep { border:none;border-top:1px solid #222;margin:8px 0; }
|
||||
.vv-fb-disabled { grid-column:1/-1;color:#3a3a3a;font-size:12px;padding:20px 0;text-align:center; }
|
||||
</style>
|
||||
|
||||
<div style="display:flex;justify-content:space-between;align-items:center;margin-bottom:12px;padding:0 2px;">
|
||||
<span style="font-size:13px;font-weight:bold;color:#888;text-transform:uppercase;letter-spacing:.06em;">FallBack</span>
|
||||
<span style="font-size:11px;color:#3a3a3a;" id="vv-fb-ts"></span>
|
||||
</div>
|
||||
|
||||
<div id="vv-fb-grid" style="display:grid;grid-template-columns:repeat(8,1fr);gap:12px;">
|
||||
<div style="grid-column:1/-1;color:#444;font-size:12px;padding:16px 0;text-align:center;">Loading…</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
(function() {
|
||||
|
||||
function _dur(startTs) {
|
||||
if (!startTs || startTs === 0) return '—';
|
||||
const s = Math.floor(Date.now() / 1000) - startTs;
|
||||
if (s < 60) return s + 's';
|
||||
if (s < 3600) return Math.floor(s / 60) + 'm';
|
||||
const h = Math.floor(s / 3600), m = Math.floor((s % 3600) / 60);
|
||||
return m ? h + 'h ' + m + 'm' : h + 'h';
|
||||
}
|
||||
|
||||
function _fmtDelay(min) {
|
||||
if (!min) return 'immediate';
|
||||
if (min < 60) return min + 'min';
|
||||
const h = Math.floor(min / 60), m = min % 60;
|
||||
return m ? h + 'h ' + m + 'm' : h + 'h';
|
||||
}
|
||||
|
||||
function _activeTier(state) {
|
||||
if (!state) return 0;
|
||||
if (state.tier4_started) return 4;
|
||||
if (state.tier3_started) return 3;
|
||||
if (state.tier2_started) return 2;
|
||||
return 1;
|
||||
}
|
||||
|
||||
function _stateBadge(st) {
|
||||
const map = {
|
||||
NORMAL: ['norm', 'NORMAL'],
|
||||
FALLBACK: ['fb', 'FALLBACK'],
|
||||
NO_INTERNET: ['nonet', 'NO INTERNET'],
|
||||
DARK: ['dark', 'DARK'],
|
||||
OFFLINE: ['dark', 'OFFLINE'],
|
||||
UNREACHABLE: ['dark', 'UNREACHABLE'],
|
||||
UNKNOWN: ['dark', 'UNKNOWN'],
|
||||
};
|
||||
const [cls, label] = map[st] || ['dark', st];
|
||||
return `<span class="vv-fb-badge ${cls}">${label}</span>`;
|
||||
}
|
||||
|
||||
function _stateDot(st) {
|
||||
const col = {
|
||||
NORMAL:'#4caf50', FALLBACK:'#ffb74d',
|
||||
NO_INTERNET:'#5c7cfa', DARK:'#9c27b0',
|
||||
OFFLINE:'#555', UNREACHABLE:'#555', UNKNOWN:'#333',
|
||||
}[st] || '#333';
|
||||
return `<span class="vv-fb-state-dot" style="background:${col}"></span>`;
|
||||
}
|
||||
|
||||
function _activeCard(nodes, handbackReq) {
|
||||
const active = nodes.filter(n => n.state && n.state.state === 'FALLBACK');
|
||||
if (!active.length) return '';
|
||||
|
||||
return active.map(covering => {
|
||||
const st = covering.state;
|
||||
const tier = _activeTier(st);
|
||||
const cov = covering.covers;
|
||||
const covered = cov ? cov.hostname : '?';
|
||||
|
||||
// All containers that should be running at current tier
|
||||
let expected = [...(cov?.tier1 || [])];
|
||||
if (tier >= 2) expected = expected.concat(cov?.tier2 || []);
|
||||
if (tier >= 3) expected = expected.concat(cov?.tier3 || []);
|
||||
if (tier >= 4) expected = expected.concat(cov?.tier4 || []);
|
||||
const runningSet = new Set(covering.running || []);
|
||||
|
||||
const ctrPills = expected.length
|
||||
? expected.map(c => {
|
||||
const cls = runningSet.has(c) ? 'running' : 'stopped';
|
||||
const sym = runningSet.has(c) ? '▲' : '▼';
|
||||
return `<span class="vv-fb-ctr ${cls}">${sym} ${c}</span>`;
|
||||
}).join('')
|
||||
: '<span style="color:#5a4020;font-size:11px;">No containers configured for this tier</span>';
|
||||
|
||||
return `<div class="vv-card vv-fb-active" style="grid-column:1/-1;">
|
||||
<div class="vv-fb-active-h">
|
||||
${_stateBadge('FALLBACK')}
|
||||
<span style="font-size:12px;color:#aa7020;">${covering.id} covering ${cov?.id || '?'} (${covered})</span>
|
||||
</div>
|
||||
<div class="vv-fb-meta">
|
||||
<div class="vv-fb-meta-item">
|
||||
<span class="vv-fb-meta-val">${_dur(st.fallback_start)}</span>
|
||||
<span class="vv-fb-meta-lbl">DURATION</span>
|
||||
</div>
|
||||
<div class="vv-fb-meta-item">
|
||||
<span class="vv-fb-meta-val">Tier ${tier}</span>
|
||||
<span class="vv-fb-meta-lbl">ACTIVE TIER</span>
|
||||
</div>
|
||||
<div class="vv-fb-meta-item">
|
||||
<span class="vv-fb-meta-val">${st.handback_strikes} / ${handbackReq}</span>
|
||||
<span class="vv-fb-meta-lbl">HANDBACK STRIKES</span>
|
||||
</div>
|
||||
<div class="vv-fb-meta-item">
|
||||
<span class="vv-fb-meta-val">${expected.length}</span>
|
||||
<span class="vv-fb-meta-lbl">CONTAINERS</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="vv-fb-ctrs">${ctrPills}</div>
|
||||
</div>`;
|
||||
}).join('');
|
||||
}
|
||||
|
||||
function _tierSection(tiers, activeTier, delays) {
|
||||
const defs = [
|
||||
{ n: 1, key: 'tier1', label: 'Tier 1', delay: 0 },
|
||||
{ n: 2, key: 'tier2', label: 'Tier 2', delay: delays?.tier2 },
|
||||
{ n: 3, key: 'tier3', label: 'Tier 3', delay: delays?.tier3 },
|
||||
{ n: 4, key: 'tier4', label: 'Tier 4', delay: delays?.tier4 },
|
||||
];
|
||||
return defs.map(({ n, key, label, delay }) => {
|
||||
const containers = tiers[key] || [];
|
||||
const isActive = activeTier >= n;
|
||||
const pills = containers.length
|
||||
? containers.map(c => `<span class="vv-fb-pill${isActive ? ' active-t' : ''}">${c}</span>`).join('')
|
||||
: `<span class="vv-fb-pill empty">none</span>`;
|
||||
const delayStr = n === 1 ? 'immediate' : _fmtDelay(delay);
|
||||
return `<div class="vv-fb-tier">
|
||||
<div class="vv-fb-tier-h">
|
||||
<span class="vv-fb-tier-lbl">${label}</span>
|
||||
<span class="vv-fb-tier-delay">${delayStr}</span>
|
||||
</div>
|
||||
<div class="vv-fb-tier-pills">${pills}</div>
|
||||
</div>`;
|
||||
}).join('');
|
||||
}
|
||||
|
||||
function _nodeCard(node) {
|
||||
const st = node.state || {};
|
||||
const state = st.state || 'UNKNOWN';
|
||||
const cov = node.covers;
|
||||
const active = _activeTier(state === 'FALLBACK' ? st : null);
|
||||
|
||||
const covTarget = cov
|
||||
? `<span class="vv-fb-arrow">→</span><span class="vv-fb-covers">covers ${cov.id} (${cov.hostname})</span>`
|
||||
: '';
|
||||
|
||||
const tierSection = cov
|
||||
? _tierSection(cov, active, cov.delays)
|
||||
: '<div style="color:#3a3a3a;font-size:11px;">No coverage configured</div>';
|
||||
|
||||
return `<div class="vv-card vv-fb-node">
|
||||
<div class="vv-fb-node-h">
|
||||
${_stateDot(state)}
|
||||
<span class="vv-fb-node-id">${node.id}</span>
|
||||
<span class="vv-fb-node-nm">${node.hostname}</span>
|
||||
${covTarget}
|
||||
<span style="flex:1"></span>
|
||||
${_stateBadge(state)}
|
||||
</div>
|
||||
<hr class="vv-fb-sep">
|
||||
${tierSection}
|
||||
</div>`;
|
||||
}
|
||||
|
||||
function _render(data) {
|
||||
if (!data.fb_enabled) {
|
||||
document.getElementById('vv-fb-grid').innerHTML =
|
||||
'<div class="vv-fb-disabled">FALLBACK_ENABLED=false — fallback monitoring is disabled</div>';
|
||||
return;
|
||||
}
|
||||
|
||||
const nodes = data.nodes || [];
|
||||
let html = '';
|
||||
|
||||
// Top: active fallback card (if any)
|
||||
html += _activeCard(nodes, data.handback_req || 3);
|
||||
|
||||
// Per-node cards
|
||||
for (const node of nodes) {
|
||||
html += _nodeCard(node);
|
||||
}
|
||||
|
||||
if (!html) {
|
||||
html = '<div class="vv-fb-disabled">No nodes configured.</div>';
|
||||
}
|
||||
|
||||
document.getElementById('vv-fb-grid').innerHTML = html;
|
||||
|
||||
const ts = data.ts
|
||||
? new Date(data.ts * 1000).toLocaleString([], {
|
||||
month:'numeric', day:'numeric', year:'numeric',
|
||||
hour:'2-digit', minute:'2-digit', second:'2-digit'})
|
||||
: '';
|
||||
document.getElementById('vv-fb-ts').textContent = ts ? 'Updated: ' + ts : '';
|
||||
}
|
||||
|
||||
function vvFbLoad() {
|
||||
fetch('/plugins/varaverk/api/fallback.php')
|
||||
.then(r => r.json())
|
||||
.then(_render)
|
||||
.catch(() => {});
|
||||
}
|
||||
|
||||
vvFbLoad();
|
||||
setInterval(vvFbLoad, 30000);
|
||||
|
||||
})();
|
||||
</script>
|
||||
+32
-45
@@ -27,36 +27,19 @@
|
||||
<div id="vv-network-body">Loading...</div>
|
||||
</div>
|
||||
|
||||
<!-- Row 2: Scripts | Partner & Fallback (span 3) | Containers & VMs (span 4) -->
|
||||
<!-- Row 2: Scripts | Fallback (span 1) | Partner (span 2) | Containers & VMs (span 4) -->
|
||||
<div class="vv-card" id="vv-scripts-card" style="grid-column:span 1;">
|
||||
<h3>Scripts</h3>
|
||||
<div id="vv-scripts-body">Loading...</div>
|
||||
</div>
|
||||
|
||||
<div class="vv-card" id="vv-partner" style="grid-column:span 3;">
|
||||
<h3 style="display:flex;align-items:center;justify-content:space-between;">
|
||||
Partner & Fallback
|
||||
<svg width="32" height="18" viewBox="0 0 32 18" fill="none" xmlns="http://www.w3.org/2000/svg" style="opacity:0.45;flex-shrink:0;">
|
||||
<!-- Server left -->
|
||||
<rect x="0.5" y="2" width="9" height="14" rx="1.5" stroke="#888" stroke-width="1" fill="#1a1a1a"/>
|
||||
<line x1="0.5" y1="5.5" x2="9.5" y2="5.5" stroke="#555" stroke-width="0.6"/>
|
||||
<circle cx="2.5" cy="3.8" r="0.8" fill="#4caf50"/>
|
||||
<line x1="2" y1="8.5" x2="8" y2="8.5" stroke="#333" stroke-width="0.6"/>
|
||||
<line x1="2" y1="10.5" x2="8" y2="10.5" stroke="#333" stroke-width="0.6"/>
|
||||
<line x1="2" y1="12.5" x2="6" y2="12.5" stroke="#333" stroke-width="0.6"/>
|
||||
<!-- Arrows between -->
|
||||
<line x1="10.5" y1="8" x2="21.5" y2="8" stroke="#555" stroke-width="0.8"/>
|
||||
<polyline points="13,5.5 10.5,8 13,10.5" stroke="#555" stroke-width="0.8" fill="none"/>
|
||||
<polyline points="19,5.5 21.5,8 19,10.5" stroke="#555" stroke-width="0.8" fill="none"/>
|
||||
<!-- Server right -->
|
||||
<rect x="22.5" y="2" width="9" height="14" rx="1.5" stroke="#888" stroke-width="1" fill="#1a1a1a"/>
|
||||
<line x1="22.5" y1="5.5" x2="31.5" y2="5.5" stroke="#555" stroke-width="0.6"/>
|
||||
<circle cx="24.5" cy="3.8" r="0.8" fill="#4caf50"/>
|
||||
<line x1="24" y1="8.5" x2="30" y2="8.5" stroke="#333" stroke-width="0.6"/>
|
||||
<line x1="24" y1="10.5" x2="30" y2="10.5" stroke="#333" stroke-width="0.6"/>
|
||||
<line x1="24" y1="12.5" x2="28" y2="12.5" stroke="#333" stroke-width="0.6"/>
|
||||
</svg>
|
||||
</h3>
|
||||
<div class="vv-card" id="vv-fallback" style="grid-column:span 1;">
|
||||
<h3>Fallback</h3>
|
||||
<div id="vv-fallback-body">Loading...</div>
|
||||
</div>
|
||||
|
||||
<div class="vv-card" id="vv-partner" style="grid-column:span 2;">
|
||||
<h3>Partner</h3>
|
||||
<div id="vv-partner-body">Loading...</div>
|
||||
</div>
|
||||
|
||||
@@ -462,20 +445,9 @@ function vvPollMonitor() {
|
||||
<rect x="28" y="60" width="5" height="2" rx="1" fill="#333"/>
|
||||
</svg>`;
|
||||
|
||||
// ── Partner & Fallback ───────────────────────────────────────────────────
|
||||
// ── Partner ──────────────────────────────────────────────────────────────
|
||||
const pt = d.partner ?? {};
|
||||
const ptHosts = pt.hosts ?? [];
|
||||
const fb = d.fallback ?? {};
|
||||
const fbActive = d.fallback_active ?? [];
|
||||
const state = (fb.state ?? 'UNKNOWN').toUpperCase();
|
||||
const stateMap = {
|
||||
NORMAL: ['#4caf50', '✓ Nominal'],
|
||||
FAILOVER: ['#f44336', '⚠ Failover active'],
|
||||
NO_INTERNET: ['#ff9800', '⚡ Internet lost'],
|
||||
DARK: ['#9e9e9e', '◌ Dark mode'],
|
||||
UNKNOWN: ['#444', '— No state file'],
|
||||
};
|
||||
const [sColor, sLabel] = stateMap[state] ?? ['#444', state];
|
||||
const ptStatus = pt.enabled
|
||||
? `<span style="color:#4caf50;">enabled</span> · sync every ${pt.sync_min}min`
|
||||
: `<span style="color:#555;">disabled</span>`;
|
||||
@@ -497,32 +469,47 @@ function vvPollMonitor() {
|
||||
<span style="color:${dot};font-size:10px;white-space:nowrap;">● ${label}</span>
|
||||
</div>`;
|
||||
});
|
||||
document.getElementById('vv-partner-body').innerHTML = ptHtml;
|
||||
|
||||
ptHtml += `<div style="font-size:11px;color:${sColor};margin:6px 0;padding-top:6px;border-top:1px solid #2a2a2a;">${sLabel}`;
|
||||
// ── Fallback ──────────────────────────────────────────────────────────────
|
||||
const fb = d.fallback ?? {};
|
||||
const fbActive = d.fallback_active ?? [];
|
||||
const state = (fb.state ?? 'UNKNOWN').toUpperCase();
|
||||
const stateMap = {
|
||||
NORMAL: ['#4caf50', '✓ Nominal'],
|
||||
FAILOVER: ['#f44336', '⚠ Failover active'],
|
||||
NO_INTERNET: ['#ff9800', '⚡ Internet lost'],
|
||||
DARK: ['#9e9e9e', '◌ Dark mode'],
|
||||
UNKNOWN: ['#444', '— No state file'],
|
||||
};
|
||||
const [sColor, sLabel] = stateMap[state] ?? ['#444', state];
|
||||
|
||||
let fbHtml = `<div style="font-size:11px;color:${sColor};margin-bottom:8px;">${sLabel}`;
|
||||
if (state === 'FAILOVER') {
|
||||
const start = parseInt(fb.failover_start ?? 0);
|
||||
if (start > 0) {
|
||||
const sec = Math.floor(Date.now() / 1000) - start;
|
||||
ptHtml += ` · ${Math.floor(sec/3600)}h ${Math.floor((sec%3600)/60)}m`;
|
||||
fbHtml += ` · ${Math.floor(sec/3600)}h ${Math.floor((sec%3600)/60)}m`;
|
||||
}
|
||||
}
|
||||
ptHtml += `</div>`;
|
||||
fbHtml += `</div>`;
|
||||
|
||||
if (fbActive.length) {
|
||||
fbActive.forEach(group => {
|
||||
ptHtml += `<div style="font-size:10px;color:#888;margin-top:6px;margin-bottom:4px;">Running for ${group.hostname}</div>`;
|
||||
fbHtml += `<div style="font-size:10px;color:#888;margin-bottom:4px;">Running for ${group.hostname}</div>`;
|
||||
group.containers.forEach(c => {
|
||||
const img = c.image.includes('/') ? c.image.split('/').pop() : c.image;
|
||||
ptHtml += `<div style="display:flex;justify-content:space-between;align-items:center;
|
||||
fbHtml += `<div style="display:flex;justify-content:space-between;align-items:center;
|
||||
background:#1a1a1a;border-radius:4px;padding:4px 8px;margin-bottom:4px;">
|
||||
<span style="color:#ccc;font-size:11px;font-weight:500;">${c.name}</span>
|
||||
<span style="color:#555;font-size:9px;margin-left:8px;white-space:nowrap;">${img}</span>
|
||||
</div>`;
|
||||
});
|
||||
});
|
||||
} else if (state === 'NORMAL') {
|
||||
fbHtml += `<div style="font-size:10px;color:#333;">No containers active</div>`;
|
||||
}
|
||||
|
||||
document.getElementById('vv-partner-body').innerHTML = ptHtml;
|
||||
document.getElementById('vv-fallback-body').innerHTML = fbHtml;
|
||||
|
||||
// ── CPU ─────────────────────────────────────────────────────────────────
|
||||
const cpu = d.cpu ?? {};
|
||||
@@ -895,7 +882,7 @@ function vvPollMonitor() {
|
||||
}
|
||||
|
||||
vvPollMonitor();
|
||||
setInterval(vvPollMonitor, 5000);
|
||||
setInterval(vvPollMonitor, 2000);
|
||||
|
||||
// Pin pools card width to CPU card width across rows
|
||||
function vvSyncCardWidths() {
|
||||
@@ -0,0 +1,290 @@
|
||||
<style>
|
||||
.vv-pt-grid { display:grid; gap:12px; }
|
||||
.vv-pt-node { background:#161616; border:1px solid #2a2a2a; border-radius:6px; padding:12px; min-width:0; }
|
||||
.vv-pt-node.me { border-color:#2a3a2a; }
|
||||
.vv-pt-node-head { display:flex; justify-content:space-between; align-items:flex-start; margin-bottom:10px; }
|
||||
.vv-pt-hostname { font-size:13px; font-weight:bold; color:#ccc; }
|
||||
.vv-pt-slot { font-size:10px; color:#444; margin-top:1px; }
|
||||
.vv-pt-tags { display:flex; gap:4px; flex-wrap:wrap; justify-content:flex-end; }
|
||||
.vv-pt-tag { font-size:8px; padding:1px 5px; border-radius:3px; white-space:nowrap; }
|
||||
.vv-pt-tag.me { background:#1a3a1a; color:#4caf50; }
|
||||
.vv-pt-tag.owner { background:#1a2a3a; color:#4a9eff; }
|
||||
.vv-pt-tag.mirror { background:#2a2a1a; color:#ff9800; }
|
||||
.vv-pt-row { display:flex; justify-content:space-between; align-items:baseline; margin:3px 0; }
|
||||
.vv-pt-lbl { font-size:11px; color:#555; }
|
||||
.vv-pt-val { font-size:11px; color:#bbb; text-align:right; }
|
||||
.vv-pt-sep { border:none; border-top:1px solid #222; margin:8px 0; }
|
||||
.vv-pt-state { font-size:12px; font-weight:500; }
|
||||
.vv-pt-dot { width:8px; height:8px; border-radius:50%; display:inline-block; flex-shrink:0; margin-right:5px; }
|
||||
.vv-pt-actions { display:flex; gap:10px; flex-wrap:wrap; margin-top:4px; }
|
||||
.vv-pt-action-btn { padding:5px 16px; border:none; border-radius:4px; cursor:pointer;
|
||||
font-size:12px; font-weight:500; }
|
||||
.vv-pt-action-btn.run { background:#1a3a1a; color:#6fcf97; border:1px solid #2e6b2e; }
|
||||
.vv-pt-action-btn.warn { background:#3a1a1a; color:#f88; border:1px solid #6b2e2e; }
|
||||
.vv-pt-action-btn.info { background:#1a2a3a; color:#7ab; border:1px solid #2e4a6b; }
|
||||
.vv-pt-action-btn:disabled { opacity:0.4; cursor:default; }
|
||||
.vv-pt-transfer-note { font-size:10px; color:#555; margin-top:6px; font-family:monospace; }
|
||||
</style>
|
||||
|
||||
<div style="display:flex;justify-content:space-between;align-items:center;margin-bottom:12px;padding:0 2px;">
|
||||
<span style="font-size:13px;font-weight:bold;color:#888;text-transform:uppercase;letter-spacing:.06em;">Partnership</span>
|
||||
<span style="font-size:11px;color:#3a3a3a;" id="vv-pt-ts"></span>
|
||||
</div>
|
||||
|
||||
<!-- Config bar -->
|
||||
<div class="vv-card" style="margin-bottom:12px;" id="vv-pt-config-card">
|
||||
<div id="vv-pt-config-body" style="color:#444;font-size:12px;">Loading…</div>
|
||||
</div>
|
||||
|
||||
<!-- Node grid -->
|
||||
<div id="vv-pt-nodes" class="vv-pt-grid" style="margin-bottom:12px;">
|
||||
<div style="color:#444;font-size:12px;padding:16px 0;text-align:center;">Loading…</div>
|
||||
</div>
|
||||
|
||||
<!-- Actions -->
|
||||
<div class="vv-card" id="vv-pt-actions-card">
|
||||
<h3>Actions</h3>
|
||||
<div id="vv-pt-actions-body" style="color:#444;font-size:12px;">Loading…</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
(function() {
|
||||
|
||||
// ── Helpers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
function _relTime(ts) {
|
||||
if (!ts) return '—';
|
||||
const d = Math.floor(Date.now() / 1000) - ts;
|
||||
if (d < 60) return 'just now';
|
||||
if (d < 3600) return Math.floor(d / 60) + 'm ago';
|
||||
if (d < 86400) return Math.floor(d / 3600) + 'h ago';
|
||||
if (d < 172800) return 'yesterday';
|
||||
return Math.floor(d / 86400) + 'd ago';
|
||||
}
|
||||
|
||||
function _uptime(sec) {
|
||||
if (!sec) return '—';
|
||||
const d = Math.floor(sec / 86400);
|
||||
const h = Math.floor((sec % 86400) / 3600);
|
||||
const m = Math.floor((sec % 3600) / 60);
|
||||
return d > 0 ? `${d}d ${h}h ${m}m` : h > 0 ? `${h}h ${m}m` : `${m}m`;
|
||||
}
|
||||
|
||||
function _row(lbl, val) {
|
||||
return `<div class="vv-pt-row"><span class="vv-pt-lbl">${lbl}</span><span class="vv-pt-val">${val}</span></div>`;
|
||||
}
|
||||
|
||||
// ── Config bar ────────────────────────────────────────────────────────────────
|
||||
|
||||
function _renderConfig(cfg) {
|
||||
const dot = cfg.enabled ? '#4caf50' : '#555';
|
||||
const label = cfg.enabled
|
||||
? `<span style="color:#4caf50;">enabled</span>`
|
||||
: `<span style="color:#555;">disabled</span>`;
|
||||
const owner = cfg.owner_host || '—';
|
||||
const items = [
|
||||
label,
|
||||
`Owner: <span style="color:#ccc;">${owner}</span>`,
|
||||
`Sync: ${cfg.sync_min}min`,
|
||||
`Grace: ${cfg.grace_hours}h`,
|
||||
`Offline threshold: ${cfg.offline_threshold}d`,
|
||||
cfg.remove_tailscale ? 'Tailscale removal: on' : 'Tailscale removal: off',
|
||||
cfg.tailscale_configured ? '' : '<span style="color:#ff9800;">⚠ Tailscale API key not set</span>',
|
||||
].filter(Boolean);
|
||||
|
||||
return `<div style="display:flex;align-items:center;gap:6px;flex-wrap:wrap;">
|
||||
<span class="vv-pt-dot" style="background:${dot}"></span>
|
||||
${items.map(i => `<span style="font-size:11px;color:#666;">${i}</span>`)
|
||||
.join('<span style="color:#333;">·</span>')}
|
||||
</div>`;
|
||||
}
|
||||
|
||||
// ── Node card ─────────────────────────────────────────────────────────────────
|
||||
|
||||
const fbColors = {
|
||||
NORMAL: '#4caf50',
|
||||
FAILOVER: '#f44336',
|
||||
NO_INTERNET: '#ff9800',
|
||||
DARK: '#9e9e9e',
|
||||
UNKNOWN: '#444',
|
||||
};
|
||||
const fbLabels = {
|
||||
NORMAL: '✓ Nominal',
|
||||
FAILOVER: '⚠ Failover',
|
||||
NO_INTERNET: '⚡ No internet',
|
||||
DARK: '◌ Dark mode',
|
||||
UNKNOWN: '— Unknown',
|
||||
};
|
||||
const ptStates = {
|
||||
ACTIVE: ['#4caf50', 'Active'],
|
||||
INACTIVE: ['#555', 'Inactive'],
|
||||
PENDING: ['#ff9800', 'Pending'],
|
||||
};
|
||||
|
||||
function _nodeCard(node) {
|
||||
const tsOnline = node.ts_online;
|
||||
const dotCol = tsOnline === null ? '#555' : tsOnline ? '#4caf50' : '#f44336';
|
||||
const dotTip = tsOnline === null ? 'unknown' : tsOnline ? (node.ts_active ? 'active' : 'idle') : 'offline';
|
||||
|
||||
const tags = [
|
||||
node.is_me ? '<span class="vv-pt-tag me">US</span>' : '',
|
||||
node.is_owner ? '<span class="vv-pt-tag owner">OWNER</span>' : '<span class="vv-pt-tag mirror">MIRROR</span>',
|
||||
].join('');
|
||||
|
||||
// System info
|
||||
const sys = node.system || {};
|
||||
const ver = sys.unraid_version || '—';
|
||||
const uptime = sys.uptime_sec ? _uptime(sys.uptime_sec) : '—';
|
||||
|
||||
// Fallback
|
||||
const fb = (node.fallback || 'UNKNOWN').toUpperCase();
|
||||
const fbCol = fbColors[fb] || '#444';
|
||||
const fbLbl = fbLabels[fb] || fb;
|
||||
|
||||
// Partnership DB
|
||||
const pt = node.partnership || {};
|
||||
const ptState = (pt.state || '').toUpperCase();
|
||||
const [ptCol, ptLbl] = ptStates[ptState] || ['#444', ptState || '—'];
|
||||
const ptUpdated = pt.updated ? _relTime(parseInt(pt.updated)) : null;
|
||||
|
||||
let body = '';
|
||||
|
||||
// Network
|
||||
body += `<div style="display:flex;align-items:center;gap:6px;margin-bottom:8px;">
|
||||
<span class="vv-pt-dot" style="background:${dotCol}"></span>
|
||||
<span style="font-size:11px;color:${dotCol};">${dotTip}</span>`;
|
||||
if (node.ts_ip) body += `<span style="font-size:10px;color:#444;margin-left:4px;">${node.ts_ip}</span>`;
|
||||
body += `</div>`;
|
||||
|
||||
// System
|
||||
body += _row('unRAID', ver);
|
||||
body += _row('Uptime', uptime);
|
||||
|
||||
body += `<hr class="vv-pt-sep">`;
|
||||
|
||||
// States
|
||||
body += `<div class="vv-pt-row">
|
||||
<span class="vv-pt-lbl">Fallback</span>
|
||||
<span class="vv-pt-state" style="color:${fbCol};">${fbLbl}</span>
|
||||
</div>`;
|
||||
body += `<div class="vv-pt-row">
|
||||
<span class="vv-pt-lbl">Partnership</span>
|
||||
<span class="vv-pt-state" style="color:${ptCol};">${ptLbl}${ptUpdated ? `<span style="font-size:9px;color:#444;margin-left:4px;">${ptUpdated}</span>` : ''}</span>
|
||||
</div>`;
|
||||
|
||||
if (pt.reason) {
|
||||
body += `<div style="font-size:10px;color:#555;margin-top:3px;text-align:right;">${pt.reason}</div>`;
|
||||
}
|
||||
|
||||
return `<div class="vv-pt-node${node.is_me ? ' me' : ''}">
|
||||
<div class="vv-pt-node-head">
|
||||
<div>
|
||||
<div class="vv-pt-hostname">${node.hostname}</div>
|
||||
<div class="vv-pt-slot">${node.id}</div>
|
||||
</div>
|
||||
<div class="vv-pt-tags">${tags}</div>
|
||||
</div>
|
||||
${body}
|
||||
</div>`;
|
||||
}
|
||||
|
||||
// ── Actions ───────────────────────────────────────────────────────────────────
|
||||
|
||||
function _renderActions(nodes, cfg) {
|
||||
// Onboard: only show when partnership is disabled (not yet set up)
|
||||
// Offboard: only when enabled
|
||||
// Transfer: only when enabled + we are owner
|
||||
const isOwner = nodes.some(n => n.is_me && n.is_owner);
|
||||
const remotes = nodes.filter(n => !n.is_me);
|
||||
const remoteOpts = remotes.map(n =>
|
||||
`<option value="${n.slot}">${n.id} — ${n.hostname}</option>`).join('');
|
||||
|
||||
let html = '<div class="vv-pt-actions">';
|
||||
|
||||
// Onboard
|
||||
html += `<button class="vv-pt-action-btn run" onclick="vvPtOnboard(this)"
|
||||
title="Run partnership_onboard.sh — auto-detects role (run on mirror first, then owner)">
|
||||
▶ Onboard
|
||||
</button>`;
|
||||
|
||||
// Offboard
|
||||
html += `<button class="vv-pt-action-btn warn" onclick="vvPtOffboard(this)"
|
||||
${!cfg.enabled ? 'title="No active partnership to offboard" style="opacity:.35;cursor:default;"' : ''}>
|
||||
▶ Offboard
|
||||
</button>`;
|
||||
|
||||
html += '</div>';
|
||||
|
||||
// Transfer — show manual command, too destructive to one-click
|
||||
if (cfg.enabled && isOwner) {
|
||||
const confirmStr = 'i-understand-this-transfers-ownership';
|
||||
html += `<div style="margin-top:14px;padding-top:10px;border-top:1px solid #1e1e1e;">
|
||||
<span style="font-size:11px;color:#555;">Transfer ownership — run manually from the current owner:</span>
|
||||
<div class="vv-pt-transfer-note" style="margin-top:4px;padding:6px 10px;background:#111;border-radius:4px;color:#666;">
|
||||
bash Partnership/partnership_transfer.sh --confirm=${confirmStr}
|
||||
</div>
|
||||
</div>`;
|
||||
}
|
||||
|
||||
return html;
|
||||
}
|
||||
|
||||
// ── Main render ───────────────────────────────────────────────────────────────
|
||||
|
||||
function _render(data) {
|
||||
const cfg = data.config || {};
|
||||
const nodes = data.nodes || [];
|
||||
|
||||
// Config bar
|
||||
document.getElementById('vv-pt-config-body').innerHTML = _renderConfig(cfg);
|
||||
|
||||
// Node grid — columns based on count
|
||||
const cols = nodes.length <= 2 ? nodes.length : nodes.length <= 4 ? 2 : 3;
|
||||
const grid = document.getElementById('vv-pt-nodes');
|
||||
grid.style.gridTemplateColumns = `repeat(${cols}, 1fr)`;
|
||||
grid.innerHTML = nodes.length
|
||||
? nodes.map(_nodeCard).join('')
|
||||
: '<div style="color:#444;font-size:12px;padding:12px 0;">No hosts found in master.conf.</div>';
|
||||
|
||||
// Actions
|
||||
document.getElementById('vv-pt-actions-body').innerHTML = _renderActions(nodes, cfg);
|
||||
|
||||
// Timestamp
|
||||
const ts = data.ts ? new Date(data.ts * 1000).toLocaleString([],
|
||||
{month:'numeric',day:'numeric',year:'numeric',hour:'2-digit',minute:'2-digit',second:'2-digit'}) : '';
|
||||
document.getElementById('vv-pt-ts').textContent = ts ? 'Updated: ' + ts : '';
|
||||
}
|
||||
|
||||
// ── Actions ───────────────────────────────────────────────────────────────────
|
||||
|
||||
function vvPtOnboard(btn) {
|
||||
if (!confirm('Run partnership_onboard.sh?\n\nRun on the MIRROR first, then on the OWNER.')) return;
|
||||
btn.disabled = true;
|
||||
btn.textContent = '⟳ Starting…';
|
||||
vvRunById('Partnership/partnership_onboard.sh');
|
||||
setTimeout(() => { btn.disabled = false; btn.textContent = '▶ Onboard'; }, 4000);
|
||||
}
|
||||
|
||||
function vvPtOffboard(btn) {
|
||||
if (btn.style.opacity === '0.35' || btn.style.cursor === 'default') return;
|
||||
if (!confirm('Run partnership_offboard.sh?\n\nThis will end the partnership, reconfigure WebUIs, and revoke SSH access.\n\nContinue?')) return;
|
||||
btn.disabled = true;
|
||||
btn.textContent = '⟳ Starting…';
|
||||
vvRunById('Partnership/partnership_offboard.sh');
|
||||
setTimeout(() => { btn.disabled = false; btn.textContent = '▶ Offboard'; }, 4000);
|
||||
}
|
||||
|
||||
// ── Poll ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
function vvPtLoad() {
|
||||
fetch('/plugins/varaverk/api/partnership.php')
|
||||
.then(r => r.json())
|
||||
.then(_render)
|
||||
.catch(() => {});
|
||||
}
|
||||
|
||||
vvPtLoad();
|
||||
setInterval(vvPtLoad, 10000);
|
||||
|
||||
})();
|
||||
</script>
|
||||
+694
-127
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,370 @@
|
||||
<style>
|
||||
.vv-wd-card { background:#161616;border:1px solid #2a2a2a;border-radius:6px;padding:10px;min-width:0; }
|
||||
.vv-wd-sec { font-size:10px;font-weight:bold;color:#444;letter-spacing:.07em;text-transform:uppercase;margin-bottom:6px; }
|
||||
.vv-wd-row { display:flex;justify-content:space-between;align-items:baseline;gap:6px;margin:2px 0; }
|
||||
.vv-wd-lbl { font-size:11px;color:#444;white-space:nowrap; }
|
||||
.vv-wd-val { font-size:12px;color:#bbb;text-align:right; }
|
||||
.vv-wd-sep { border:none;border-top:1px solid #1e1e1e;margin:6px 0; }
|
||||
.vv-wd-pill { font-size:10px;padding:1px 6px;border-radius:2px;background:#1e1e1e;color:#666;border:1px solid #272727; }
|
||||
.vv-wd-pill.ok { background:#0d1f0d;color:#4caf50;border-color:#1a3a1a; }
|
||||
.vv-wd-pill.warn { background:#1f1500;color:#ffb74d;border-color:#3a2800; }
|
||||
.vv-wd-pill.err { background:#200d0d;color:#ef5350;border-color:#3a1a1a; }
|
||||
.vv-wd-pill-row { display:flex;flex-wrap:wrap;gap:4px;margin-top:4px; }
|
||||
.vv-wd-bar { height:5px;border-radius:2px;background:#1e1e1e;margin-top:3px;overflow:hidden; }
|
||||
.vv-wd-bar-fill{ height:100%;border-radius:2px;transition:width .3s; }
|
||||
.vv-wd-badge { font-size:11px;font-weight:bold;padding:2px 8px;border-radius:3px; }
|
||||
.vv-wd-badge.ok { background:#0d1f0d;color:#4caf50; }
|
||||
.vv-wd-badge.soft { background:#1f1f00;color:#cddc39; }
|
||||
.vv-wd-badge.med { background:#1f1000;color:#ffb74d; }
|
||||
.vv-wd-badge.hard { background:#200d0d;color:#ef5350; }
|
||||
.vv-wd-node-h { display:flex;align-items:center;gap:8px;margin-bottom:10px; }
|
||||
.vv-wd-node-id { font-size:12px;font-weight:bold;color:#666;letter-spacing:.06em;text-transform:uppercase; }
|
||||
.vv-wd-dot { width:6px;height:6px;border-radius:50%;flex-shrink:0; }
|
||||
.vv-wd-strike-row { display:flex;align-items:baseline;gap:6px;margin:2px 0; }
|
||||
.vv-wd-strike-name{ font-size:11px;color:#777;flex:1; }
|
||||
.vv-wd-strike-cnt { font-size:11px;font-weight:bold;color:#ffb74d; }
|
||||
.vv-wd-reboot-ts { font-size:11px;color:#555;margin:1px 0; }
|
||||
.vv-wd-ctr-row { display:flex;justify-content:space-between;align-items:baseline;margin:2px 0; }
|
||||
.vv-wd-ctr-name{ font-size:11px;color:#888; }
|
||||
.vv-wd-ctr-lim { font-size:11px;color:#555; }
|
||||
.vv-wd-pressure{ grid-column:1/-1;border-color:#3a2000;background:#1a1000; }
|
||||
</style>
|
||||
|
||||
<div style="display:flex;justify-content:space-between;align-items:center;margin-bottom:12px;padding:0 2px;">
|
||||
<span style="font-size:13px;font-weight:bold;color:#888;text-transform:uppercase;letter-spacing:.06em;">Watchdog</span>
|
||||
<span style="font-size:11px;color:#3a3a3a;" id="vv-wd-ts"></span>
|
||||
</div>
|
||||
|
||||
<div id="vv-wd-grid" style="display:grid;grid-template-columns:repeat(8,1fr);gap:12px;">
|
||||
<div style="grid-column:1/-1;color:#444;font-size:12px;padding:16px 0;text-align:center;">Loading…</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
(function() {
|
||||
|
||||
const GB = 1073741824;
|
||||
|
||||
function _relTime(ts) {
|
||||
if (!ts) return '—';
|
||||
const d = Math.floor(Date.now() / 1000) - ts;
|
||||
if (d < 60) return 'just now';
|
||||
if (d < 3600) return Math.floor(d / 60) + 'm ago';
|
||||
if (d < 86400)return Math.floor(d / 3600) + 'h ' + Math.floor((d % 3600) / 60) + 'm ago';
|
||||
return Math.floor(d / 86400) + 'd ago';
|
||||
}
|
||||
|
||||
function _fmtBytes(b) {
|
||||
if (b >= GB) return (b / GB).toFixed(1) + ' GB';
|
||||
if (b >= 1048576) return (b / 1048576).toFixed(0) + ' MB';
|
||||
return (b / 1024).toFixed(0) + ' KB';
|
||||
}
|
||||
|
||||
function _dur(s) {
|
||||
const d = Math.floor(s / 86400), h = Math.floor((s % 86400) / 3600), m = Math.floor((s % 3600) / 60);
|
||||
if (d) return d + 'd ' + h + 'h';
|
||||
if (h) return h + 'h ' + m + 'm';
|
||||
return m + 'm';
|
||||
}
|
||||
|
||||
function _row(lbl, val) {
|
||||
return `<div class="vv-wd-row"><span class="vv-wd-lbl">${lbl}</span><span class="vv-wd-val">${val}</span></div>`;
|
||||
}
|
||||
|
||||
function _bar(pct, col) {
|
||||
return `<div class="vv-wd-bar"><div class="vv-wd-bar-fill" style="width:${Math.min(pct,100)}%;background:${col}"></div></div>`;
|
||||
}
|
||||
|
||||
function _pill(label, cls) {
|
||||
return `<span class="vv-wd-pill ${cls}">${label}</span>`;
|
||||
}
|
||||
|
||||
function _levelLabel(level) {
|
||||
return ['OK', 'SOFT', 'MEDIUM', 'HARD'][level] || '?';
|
||||
}
|
||||
function _levelCls(level) {
|
||||
return ['ok', 'soft', 'med', 'hard'][level] || 'ok';
|
||||
}
|
||||
|
||||
// ── Pressure alert card ───────────────────────────────────────────────────────
|
||||
function _pressureCard(node) {
|
||||
const st = node.states;
|
||||
if (!st || st.rw_level === 0) return '';
|
||||
const level = st.rw_level;
|
||||
const cls = _levelCls(level);
|
||||
const label = _levelLabel(level);
|
||||
|
||||
const paused = (st.rw_paused || []).filter(Boolean);
|
||||
const stopped = (st.rw_stopped || []).filter(Boolean);
|
||||
const pausedHtml = paused.length ? paused.map(c => _pill(c, 'warn')).join('') : '';
|
||||
const stoppedHtml = stopped.length ? stopped.map(c => _pill(c, 'err')).join('') : '';
|
||||
|
||||
return `<div class="vv-wd-card vv-wd-pressure" style="grid-column:1/-1">
|
||||
<div style="display:flex;align-items:center;gap:10px;margin-bottom:8px;">
|
||||
<span class="vv-wd-badge ${cls}">PRESSURE ${label}</span>
|
||||
<span style="font-size:11px;color:#7a5020;">${node.id} (${node.hostname})</span>
|
||||
${st.mem_shutdown ? `<span class="vv-wd-badge hard" style="margin-left:auto;">MEM SHUTDOWN ACTIVE</span>` : ''}
|
||||
</div>
|
||||
${paused.length ? `<div style="margin-bottom:4px;"><span class="vv-wd-lbl">Paused:</span> <span class="vv-wd-pill-row" style="display:inline-flex;">${pausedHtml}</span></div>` : ''}
|
||||
${stopped.length ? `<div><span class="vv-wd-lbl">Stopped:</span> <span class="vv-wd-pill-row" style="display:inline-flex;">${stoppedHtml}</span></div>` : ''}
|
||||
</div>`;
|
||||
}
|
||||
|
||||
// ── System health card ────────────────────────────────────────────────────────
|
||||
function _systemCard(node, cfg) {
|
||||
const sys = node.system;
|
||||
if (!sys) {
|
||||
return `<div class="vv-wd-card" style="grid-column:span 2;">
|
||||
<div class="vv-wd-node-h">
|
||||
<span class="vv-wd-dot" style="background:#444"></span>
|
||||
<span class="vv-wd-node-id">${node.id}</span>
|
||||
<span style="font-size:11px;color:#3a3a3a;">${node.hostname}</span>
|
||||
<span class="vv-wd-badge" style="margin-left:auto;background:#1a1a1a;color:#444">UNREACHABLE</span>
|
||||
</div>
|
||||
</div>`;
|
||||
}
|
||||
|
||||
const memGb = sys.mem_avail / GB;
|
||||
const memTotGb = sys.mem_total / GB;
|
||||
const usedPct = memTotGb > 0 ? ((memTotGb - memGb) / memTotGb) * 100 : 0;
|
||||
const memCol = memGb < cfg.sys_mem_gb ? '#ef5350'
|
||||
: memGb < cfg.rw_hard_gb ? '#ef5350'
|
||||
: memGb < cfg.rw_medium_gb ? '#ffb74d'
|
||||
: memGb < cfg.rw_soft_gb ? '#cddc39' : '#4caf50';
|
||||
|
||||
const loadPct = sys.cores > 0 ? (sys.load1 / (sys.cores * cfg.rw_load_med)) * 100 : 0;
|
||||
const loadCol = sys.load1 > sys.cores * cfg.rw_load_med ? '#ef5350'
|
||||
: sys.load1 > sys.cores * cfg.rw_load_soft ? '#ffb74d'
|
||||
: '#4caf50';
|
||||
|
||||
const daemonDot = sys.daemon_ok ? '#4caf50' : '#ef5350';
|
||||
|
||||
const st = node.states || {};
|
||||
const level = st.rw_level || 0;
|
||||
const dotCol = level >= 3 ? '#ef5350' : level >= 2 ? '#ffb74d' : level >= 1 ? '#cddc39' : '#4caf50';
|
||||
|
||||
return `<div class="vv-wd-card" style="grid-column:span 2;">
|
||||
<div class="vv-wd-node-h">
|
||||
<span class="vv-wd-dot" style="background:${dotCol}"></span>
|
||||
<span class="vv-wd-node-id">${node.id}</span>
|
||||
<span style="font-size:11px;color:#3a3a3a;">${node.hostname}</span>
|
||||
<span class="vv-wd-badge ${_levelCls(level)}" style="margin-left:auto;">${_levelLabel(level)}</span>
|
||||
</div>
|
||||
<div class="vv-wd-sec">System</div>
|
||||
${_row('RAM free', `<span style="color:${memCol}">${_fmtBytes(sys.mem_avail)}</span> / ${_fmtBytes(sys.mem_total)}`)}
|
||||
${_bar(usedPct, memCol)}
|
||||
<div style="display:flex;justify-content:space-between;margin-top:1px;font-size:10px;color:#333">
|
||||
<span>free</span>
|
||||
<span>${cfg.rw_soft_gb}G soft · ${cfg.rw_hard_gb}G hard · ${cfg.sys_mem_gb}G reboot</span>
|
||||
</div>
|
||||
<div style="height:5px"></div>
|
||||
${_row('Load avg', `<span style="color:${loadCol}">${sys.load1.toFixed(2)}</span> / ${sys.cores} cores`)}
|
||||
${_bar(loadPct, loadCol)}
|
||||
<div style="height:5px"></div>
|
||||
${_row('Uptime', _dur(sys.uptime))}
|
||||
${_row('Docker', `<span style="color:${daemonDot}">${sys.daemon_ok ? 'daemon ok' : 'daemon err'}</span>`)}
|
||||
${sys.oom_count > 0 ? _row('OOM kills', `<span style="color:#ef5350">${sys.oom_count}</span>`) : _row('OOM kills', '<span style="color:#333">0</span>')}
|
||||
</div>`;
|
||||
}
|
||||
|
||||
// ── Docker watchdog state card ────────────────────────────────────────────────
|
||||
function _dockerCard(node, cfg) {
|
||||
const st = node.states;
|
||||
if (!st) return `<div class="vv-wd-card" style="grid-column:span 2;"><div class="vv-wd-sec">Docker Watchdog</div><div style="color:#3a3a3a;font-size:11px;padding:8px 0;">No data</div></div>`;
|
||||
|
||||
const strikes = Object.entries(st.ctr_strikes || {});
|
||||
const skiplist = st.skiplist || [];
|
||||
const restarts = (st.restarts || []).slice(0, 10);
|
||||
const daemonCls = st.daemon_strikes > 0 ? 'err' : 'ok';
|
||||
const allOk = strikes.length === 0 && skiplist.length === 0 && st.daemon_strikes === 0;
|
||||
|
||||
// Group restarts by container for last-24h summary
|
||||
const rCounts = {};
|
||||
for (const r of restarts) {
|
||||
rCounts[r.name] = (rCounts[r.name] || 0) + 1;
|
||||
}
|
||||
|
||||
let strikesHtml = '';
|
||||
if (strikes.length === 0 && st.daemon_strikes === 0) {
|
||||
strikesHtml = '<div style="color:#333;font-size:11px;">0 active strikes</div>';
|
||||
} else {
|
||||
if (st.daemon_strikes > 0) {
|
||||
strikesHtml += `<div class="vv-wd-strike-row"><span class="vv-wd-strike-name">daemon</span><span class="vv-wd-strike-cnt">${st.daemon_strikes}</span></div>`;
|
||||
}
|
||||
for (const [name, cnt] of strikes) {
|
||||
strikesHtml += `<div class="vv-wd-strike-row"><span class="vv-wd-strike-name">${name}</span><span class="vv-wd-strike-cnt">${cnt} / ${cfg.cpu_fail_lim}</span></div>`;
|
||||
}
|
||||
}
|
||||
|
||||
let skipHtml = '';
|
||||
if (skiplist.length === 0) {
|
||||
skipHtml = '<div style="color:#333;font-size:11px;">empty</div>';
|
||||
} else {
|
||||
skipHtml = `<div class="vv-wd-pill-row">${skiplist.map(c => _pill(c, 'err')).join('')}</div>`;
|
||||
}
|
||||
|
||||
let restartHtml = '';
|
||||
const rcEntries = Object.entries(rCounts);
|
||||
if (rcEntries.length === 0) {
|
||||
restartHtml = '<div style="color:#333;font-size:11px;">none (24h)</div>';
|
||||
} else {
|
||||
restartHtml = rcEntries.map(([n, c]) =>
|
||||
`<div class="vv-wd-row"><span class="vv-wd-lbl">${n}</span><span class="vv-wd-val" style="color:${c >= cfg.restart_limit ? '#ef5350' : '#ffb74d'}">${c}×</span></div>`
|
||||
).join('');
|
||||
}
|
||||
|
||||
return `<div class="vv-wd-card" style="grid-column:span 3;">
|
||||
<div class="vv-wd-sec">Docker Watchdog</div>
|
||||
<div style="display:flex;gap:6px;margin-bottom:8px;">
|
||||
${_pill(st.daemon_restart ? 'daemon restarted' : 'daemon ok', daemonCls)}
|
||||
${allOk ? _pill('all clear', 'ok') : ''}
|
||||
</div>
|
||||
<div class="vv-wd-sec">Strikes</div>
|
||||
${strikesHtml}
|
||||
<hr class="vv-wd-sep">
|
||||
<div class="vv-wd-sec">Skip list</div>
|
||||
${skipHtml}
|
||||
<hr class="vv-wd-sep">
|
||||
<div class="vv-wd-sec">Restarts (24h)</div>
|
||||
${restartHtml}
|
||||
</div>`;
|
||||
}
|
||||
|
||||
// ── Stability / reboot card ───────────────────────────────────────────────────
|
||||
function _stabilityCard(node, cfg) {
|
||||
const st = node.states;
|
||||
if (!st) return `<div class="vv-wd-card" style="grid-column:span 1;"><div class="vv-wd-sec">Stability</div><div style="color:#3a3a3a;font-size:11px;padding:8px 0;">No data</div></div>`;
|
||||
|
||||
const reboots = st.reboots || [];
|
||||
const sysStr = Object.entries(st.sys_strikes || {});
|
||||
const rebootCls = reboots.length >= cfg.reboot_limit ? 'err' : reboots.length > 0 ? 'warn' : 'ok';
|
||||
|
||||
let sysHtml = '';
|
||||
if (sysStr.length === 0) {
|
||||
sysHtml = '<div style="color:#333;font-size:11px;">0 active strikes</div>';
|
||||
} else {
|
||||
sysHtml = sysStr.map(([k, v]) =>
|
||||
`<div class="vv-wd-strike-row"><span class="vv-wd-strike-name" style="font-size:10px;">${k.replace(/_/g,' ')}</span><span class="vv-wd-strike-cnt">${v}</span></div>`
|
||||
).join('');
|
||||
}
|
||||
|
||||
const rebootHtml = reboots.length === 0
|
||||
? '<div style="color:#333;font-size:11px;">none (12h)</div>'
|
||||
: reboots.map(ts => `<div class="vv-wd-reboot-ts">${_relTime(ts)}</div>`).join('');
|
||||
|
||||
return `<div class="vv-wd-card" style="grid-column:span 3;">
|
||||
<div class="vv-wd-sec">Stability</div>
|
||||
<div style="display:flex;gap:6px;margin-bottom:8px;">
|
||||
${_pill(`${reboots.length} / ${cfg.reboot_limit} reboots`, rebootCls)}
|
||||
${_pill(`${cfg.reboot_window}h window`, '')}
|
||||
</div>
|
||||
<div class="vv-wd-sec">Strikes</div>
|
||||
${sysHtml}
|
||||
<hr class="vv-wd-sep">
|
||||
<div class="vv-wd-sec">Reboot history</div>
|
||||
${rebootHtml}
|
||||
</div>`;
|
||||
}
|
||||
|
||||
// ── Config inventory card ─────────────────────────────────────────────────────
|
||||
function _configCard(node) {
|
||||
const cfg = node.config || {};
|
||||
const mon = Object.entries(cfg.monitored || {});
|
||||
const req = cfg.required || [];
|
||||
const ign = cfg.ignore || [];
|
||||
const paus = cfg.pause_list|| [];
|
||||
const stop = cfg.stop_list || [];
|
||||
const crit = cfg.critical || [];
|
||||
|
||||
const monHtml = mon.length === 0
|
||||
? '<div style="color:#333;font-size:11px;">none</div>'
|
||||
: mon.map(([name, mb]) => {
|
||||
const gb = (mb / 1024).toFixed(0);
|
||||
return `<div class="vv-wd-ctr-row"><span class="vv-wd-ctr-name">${name}</span><span class="vv-wd-ctr-lim">${gb} GB</span></div>`;
|
||||
}).join('');
|
||||
|
||||
const reqHtml = req.length === 0
|
||||
? '<div style="color:#333;font-size:11px;">none</div>'
|
||||
: `<div class="vv-wd-pill-row">${req.map(c => _pill(c, crit.includes(c) ? '' : '')).join('')}</div>`;
|
||||
|
||||
const ignHtml = ign.length === 0
|
||||
? '<div style="color:#3a3a3a;font-size:11px;">none</div>'
|
||||
: `<div class="vv-wd-pill-row">${ign.map(c => _pill(c, '')).join('')}</div>`;
|
||||
|
||||
const pausHtml = paus.length === 0
|
||||
? '<div style="color:#3a3a3a;font-size:11px;">none</div>'
|
||||
: `<div class="vv-wd-pill-row">${paus.map(c => _pill(c, 'warn')).join('')}</div>`;
|
||||
|
||||
const stopHtml = stop.length === 0
|
||||
? '<div style="color:#3a3a3a;font-size:11px;">none</div>'
|
||||
: `<div class="vv-wd-pill-row">${stop.map(c => _pill(c, 'err')).join('')}</div>`;
|
||||
|
||||
return `<div class="vv-wd-card" style="grid-column:span 4;">
|
||||
<div style="display:flex;gap:6px;align-items:center;margin-bottom:8px;">
|
||||
<span class="vv-wd-node-id">${node.id}</span>
|
||||
<span style="font-size:11px;color:#3a3a3a;">${node.hostname}</span>
|
||||
</div>
|
||||
<div style="display:grid;grid-template-columns:1fr 1fr;gap:10px;">
|
||||
<div>
|
||||
<div class="vv-wd-sec">Mem limits (Tier 1)</div>
|
||||
${monHtml}
|
||||
<div style="height:8px"></div>
|
||||
<div class="vv-wd-sec">Required</div>
|
||||
${reqHtml}
|
||||
</div>
|
||||
<div>
|
||||
<div class="vv-wd-sec">Pause at medium pressure</div>
|
||||
${pausHtml}
|
||||
<div style="height:8px"></div>
|
||||
<div class="vv-wd-sec">Stop at hard pressure</div>
|
||||
${stopHtml}
|
||||
<div style="height:8px"></div>
|
||||
<div class="vv-wd-sec">Scan ignore</div>
|
||||
${ignHtml}
|
||||
</div>
|
||||
</div>
|
||||
</div>`;
|
||||
}
|
||||
|
||||
// ── Main render ───────────────────────────────────────────────────────────────
|
||||
function _render(data) {
|
||||
const nodes = data.nodes || [];
|
||||
const cfg = data.cfg || {};
|
||||
let html = '';
|
||||
|
||||
// Pressure alerts (full width, per node)
|
||||
for (const node of nodes) html += _pressureCard(node);
|
||||
|
||||
// System + watchdog state rows per node
|
||||
for (const node of nodes) {
|
||||
html += _systemCard(node, cfg);
|
||||
html += _dockerCard(node, cfg);
|
||||
html += _stabilityCard(node, cfg);
|
||||
}
|
||||
|
||||
// Config inventory per node
|
||||
for (const node of nodes) html += _configCard(node);
|
||||
|
||||
if (!html) html = '<div style="grid-column:1/-1;color:#444;font-size:12px;padding:16px 0;text-align:center;">No nodes configured.</div>';
|
||||
|
||||
document.getElementById('vv-wd-grid').innerHTML = html;
|
||||
|
||||
const ts = data.ts
|
||||
? new Date(data.ts * 1000).toLocaleString([], {
|
||||
month:'numeric', day:'numeric', year:'numeric',
|
||||
hour:'2-digit', minute:'2-digit', second:'2-digit'})
|
||||
: '';
|
||||
document.getElementById('vv-wd-ts').textContent = ts ? 'Updated: ' + ts : '';
|
||||
}
|
||||
|
||||
function vvWdLoad() {
|
||||
fetch('/plugins/varaverk/api/watchdog.php')
|
||||
.then(r => r.json())
|
||||
.then(_render)
|
||||
.catch(() => {});
|
||||
}
|
||||
|
||||
vvWdLoad();
|
||||
setInterval(vvWdLoad, 30000);
|
||||
|
||||
})();
|
||||
</script>
|
||||
@@ -0,0 +1,99 @@
|
||||
#!/bin/bash
|
||||
# Varaverk job runner — wraps script execution with JSON status tracking.
|
||||
# Called by /etc/cron.d/varaverk for every scheduled job.
|
||||
#
|
||||
# Usage: bash run_job.sh <job_id> <script_path> [flags...]
|
||||
#
|
||||
# Flags consumed by run_job.sh (stripped before passing to script):
|
||||
# --manual — marks a UI-triggered run; writes a sentinel on completion so
|
||||
# the next cron fire is suppressed if it falls within the job's
|
||||
# own cron interval (prevents double-firing after manual run).
|
||||
#
|
||||
# Writes: /var/log/varaverk/<id>.json — status, timestamps, exit code, pid
|
||||
# /var/log/varaverk/<id>.log — appended per run, trimmed to LOG_MAX_LINES
|
||||
# /var/log/varaverk/<id>.manual_ts — sentinel: epoch of last manual completion
|
||||
#
|
||||
# Status values: running → ok (exit 0) | warn (exit 1) | error (exit 2+)
|
||||
|
||||
JOB_ID="$1"
|
||||
SCRIPT="$2"
|
||||
shift 2
|
||||
|
||||
LOG_DIR="/var/log/varaverk"
|
||||
BASE="${JOB_ID%.sh}"
|
||||
LOG_FILE="$LOG_DIR/$BASE.log"
|
||||
STAT_FILE="$LOG_DIR/$BASE.json"
|
||||
MANUAL_TS_FILE="$LOG_DIR/$BASE.manual_ts"
|
||||
LOG_MAX_LINES=1000
|
||||
|
||||
# Strip --manual from script args — it's for run_job.sh only
|
||||
MANUAL=false
|
||||
SCRIPT_ARGS=()
|
||||
for _arg in "$@"; do
|
||||
[[ "$_arg" == "--manual" ]] && MANUAL=true || SCRIPT_ARGS+=("$_arg")
|
||||
done
|
||||
unset _arg
|
||||
|
||||
mkdir -p "$(dirname "$LOG_FILE")" "$(dirname "$STAT_FILE")"
|
||||
|
||||
# Cron invocation: skip if a manual run completed recently within this job's own interval.
|
||||
if [[ "$MANUAL" == false && -f "$MANUAL_TS_FILE" ]]; then
|
||||
LAST_MANUAL=$(cat "$MANUAL_TS_FILE" 2>/dev/null || echo 0)
|
||||
ELAPSED=$(( $(date +%s) - LAST_MANUAL ))
|
||||
|
||||
# Only suppress for interval-based crons (*/N * * * *).
|
||||
# Static schedules like "30 2 * * 0" run at their appointed time and are never suppressed.
|
||||
INTERVAL=0
|
||||
SCHEDULE_FILE="/boot/config/plugins/varaverk/schedule.json"
|
||||
if [[ -f "$SCHEDULE_FILE" ]]; then
|
||||
CRON_EXPR=$(php -r "
|
||||
\$s = json_decode(file_get_contents('$SCHEDULE_FILE'), true) ?: [];
|
||||
echo \$s['$JOB_ID']['cron'] ?? '';
|
||||
" 2>/dev/null)
|
||||
if [[ "$CRON_EXPR" =~ ^\*/([0-9]+) ]]; then
|
||||
INTERVAL=$(( ${BASH_REMATCH[1]} * 60 ))
|
||||
fi
|
||||
fi
|
||||
|
||||
if (( INTERVAL > 0 && ELAPSED < INTERVAL )); then
|
||||
printf '\n── %s [SKIPPED — ran manually %ds ago, interval %ds] ────────\n' \
|
||||
"$(date '+%Y-%m-%d %H:%M:%S')" "$ELAPSED" "$INTERVAL" >> "$LOG_FILE"
|
||||
printf '{"id":"%s","status":"skipped","manual_elapsed":%s,"interval":%s}\n' \
|
||||
"$JOB_ID" "$ELAPSED" "$INTERVAL" > "$STAT_FILE"
|
||||
rm -f "$MANUAL_TS_FILE"
|
||||
exit 0
|
||||
fi
|
||||
rm -f "$MANUAL_TS_FILE"
|
||||
fi
|
||||
|
||||
START=$(date +%s)
|
||||
printf '{"id":"%s","status":"running","start":%s,"pid":%s}\n' \
|
||||
"$JOB_ID" "$START" "$$" > "$STAT_FILE"
|
||||
|
||||
printf '\n── %s ────────────────────────────────────────────────\n' \
|
||||
"$(date '+%Y-%m-%d %H:%M:%S')" >> "$LOG_FILE"
|
||||
|
||||
bash "$SCRIPT" "${SCRIPT_ARGS[@]}" >> "$LOG_FILE" 2>&1
|
||||
EC=$?
|
||||
|
||||
END=$(date +%s)
|
||||
if [ "$EC" -eq 0 ]; then STATUS="ok"
|
||||
elif [ "$EC" -eq 1 ]; then STATUS="warn"
|
||||
else STATUS="error"
|
||||
fi
|
||||
|
||||
printf '{"id":"%s","status":"%s","start":%s,"end":%s,"exit":%s}\n' \
|
||||
"$JOB_ID" "$STATUS" "$START" "$END" "$EC" > "$STAT_FILE"
|
||||
|
||||
# Write manual sentinel so cron can skip the next fire within the interval
|
||||
if [[ "$MANUAL" == true ]]; then
|
||||
echo "$END" > "$MANUAL_TS_FILE"
|
||||
fi
|
||||
|
||||
# Trim to last LOG_MAX_LINES to prevent unbounded growth
|
||||
line_count=$(wc -l < "$LOG_FILE" 2>/dev/null || echo 0)
|
||||
if [ "$line_count" -gt "$LOG_MAX_LINES" ]; then
|
||||
tail -n "$LOG_MAX_LINES" "$LOG_FILE" > "${LOG_FILE}.tmp" && mv "${LOG_FILE}.tmp" "$LOG_FILE"
|
||||
fi
|
||||
|
||||
exit $EC
|
||||
@@ -1,88 +0,0 @@
|
||||
<?php
|
||||
// Live board data: locks, recent errors, partner reachability.
|
||||
header('Content-Type: application/json');
|
||||
require_once dirname(__DIR__) . '/include/config.php';
|
||||
|
||||
$out = ['ok' => true];
|
||||
|
||||
// ── Active locks ──────────────────────────────────────────────────────────
|
||||
$lockDir = '/tmp/unraid_locks';
|
||||
$locks = [];
|
||||
if (is_dir($lockDir)) {
|
||||
foreach (glob($lockDir . '/*.lock') ?: [] as $lf) {
|
||||
$name = basename($lf, '.lock');
|
||||
$age = time() - (int)filemtime($lf);
|
||||
$content = trim(file_get_contents($lf) ?: '');
|
||||
// content is "PID:scriptname" — extract PID
|
||||
$pid = preg_match('/^(\d+)/', $content, $pm) ? $pm[1] : '';
|
||||
// Skip if PID is still alive (it's legitimately running)
|
||||
if ($pid && file_exists("/proc/$pid")) continue;
|
||||
$locks[] = ['name' => $name, 'file' => basename($lf), 'age' => $age];
|
||||
}
|
||||
}
|
||||
$out['locks'] = $locks;
|
||||
|
||||
// ── Recent errors ──────────────────────────────────────────────────────────
|
||||
$errors = [];
|
||||
if (is_dir(LOG_DIR)) {
|
||||
$cutoff = time() - 7 * 86400; // only logs touched in last 7 days
|
||||
foreach (glob(LOG_DIR . '/*.log') ?: [] as $lf) {
|
||||
if (filemtime($lf) < $cutoff) continue;
|
||||
$script = basename($lf, '.log');
|
||||
$lines = array_slice(@file($lf) ?: [], -200);
|
||||
$lastErr = null;
|
||||
foreach (array_reverse($lines) as $raw) {
|
||||
// Strip ANSI escape codes
|
||||
$clean = preg_replace('/\033\[[0-9;]*[mK]/', '', rtrim($raw));
|
||||
if (!$clean) continue;
|
||||
if (preg_match('/\[(?:ERROR|WARN|CRITICAL|FAILED)\]/i', $clean) ||
|
||||
preg_match('/\b(?:ERROR|CRITICAL|FAILED):\s/i', $clean) ||
|
||||
str_contains($clean, '✗') ||
|
||||
(str_contains($clean, '⚠') && !str_contains($clean, '♥'))) {
|
||||
$lastErr = mb_substr($clean, 0, 220);
|
||||
break;
|
||||
}
|
||||
}
|
||||
if ($lastErr !== null) {
|
||||
$errors[] = ['script' => $script, 'line' => $lastErr, 'ts' => (int)filemtime($lf)];
|
||||
}
|
||||
}
|
||||
usort($errors, fn($a, $b) => $b['ts'] - $a['ts']);
|
||||
}
|
||||
$out['errors'] = array_slice($errors, 0, 20);
|
||||
|
||||
// ── Partner reachability ───────────────────────────────────────────────────
|
||||
$cacheFile = '/tmp/vv_partner_cache.json';
|
||||
$cacheTtl = 30;
|
||||
$partnerData = null;
|
||||
|
||||
if (file_exists($cacheFile) && (time() - (int)filemtime($cacheFile)) < $cacheTtl) {
|
||||
$partnerData = json_decode(file_get_contents($cacheFile), true);
|
||||
} else {
|
||||
$confRaw = vv_read_conf_raw('master.conf');
|
||||
preg_match('/^\s*HOST1(?:_NAME)?\s*=\s*["\']?(\S+?)["\']?\s*(?:#.*)?$/m', $confRaw, $m1);
|
||||
preg_match('/^\s*HOST2(?:_NAME)?\s*=\s*["\']?(\S+?)["\']?\s*(?:#.*)?$/m', $confRaw, $m2);
|
||||
$host1 = $m1[1] ?? '';
|
||||
$host2 = $m2[1] ?? '';
|
||||
$mine = vv_get_hostname();
|
||||
$partnerHost = null;
|
||||
if ($mine && $host1 && strcasecmp($mine, $host1) === 0) $partnerHost = $host2;
|
||||
if ($mine && $host2 && strcasecmp($mine, $host2) === 0) $partnerHost = $host1;
|
||||
|
||||
if ($partnerHost) {
|
||||
$start = microtime(true);
|
||||
$result = shell_exec('ping -c1 -W1 ' . escapeshellarg($partnerHost) . ' 2>&1');
|
||||
$elapsed = (int)round((microtime(true) - $start) * 1000);
|
||||
$reached = str_contains((string)$result, '1 received')
|
||||
|| str_contains((string)$result, '1 packets received');
|
||||
$partnerData = [
|
||||
'host' => $partnerHost,
|
||||
'reachable' => $reached,
|
||||
'latency' => $reached ? $elapsed : null,
|
||||
];
|
||||
@file_put_contents($cacheFile, json_encode($partnerData));
|
||||
}
|
||||
}
|
||||
$out['partner'] = $partnerData;
|
||||
|
||||
echo json_encode($out);
|
||||
@@ -1,13 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Varaverk: run array_started.sh if enabled in the schedule (background — non-blocking).
|
||||
php -r "
|
||||
require_once '/usr/local/emhttp/plugins/varaverk/include/scheduler.php';
|
||||
\$s = vv_schedule_load();
|
||||
\$e = \$s['Orchestrators/array_started.sh'] ?? [];
|
||||
if (empty(\$e['enabled'])) exit(0);
|
||||
\$runner = '/usr/local/emhttp/plugins/varaverk/run_job.sh';
|
||||
\$script = SCRIPTS_DIR . '/Orchestrators/array_started.sh';
|
||||
if (!file_exists(\$script)) exit(0);
|
||||
\$flags = !empty(\$e['log_enabled']) ? ' --log' : '';
|
||||
exec('nohup bash ' . escapeshellarg(\$runner) . ' Orchestrators/array_started.sh ' . escapeshellarg(\$script) . \$flags . ' > /dev/null 2>&1 &');
|
||||
" 2>/dev/null
|
||||
@@ -1,13 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Varaverk: run array_stopping.sh if enabled in the schedule (foreground — blocks until done).
|
||||
php -r "
|
||||
require_once '/usr/local/emhttp/plugins/varaverk/include/scheduler.php';
|
||||
\$s = vv_schedule_load();
|
||||
\$e = \$s['Orchestrators/array_stopping.sh'] ?? [];
|
||||
if (empty(\$e['enabled'])) exit(0);
|
||||
\$runner = '/usr/local/emhttp/plugins/varaverk/run_job.sh';
|
||||
\$script = SCRIPTS_DIR . '/Orchestrators/array_stopping.sh';
|
||||
if (!file_exists(\$script)) exit(0);
|
||||
\$flags = !empty(\$e['log_enabled']) ? ' --log' : '';
|
||||
passthru('bash ' . escapeshellarg(\$runner) . ' Orchestrators/array_stopping.sh ' . escapeshellarg(\$script) . \$flags);
|
||||
" 2>/dev/null
|
||||
@@ -1,49 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Varaverk job runner — wraps script execution with JSON status tracking.
|
||||
# Called by /etc/cron.d/varaverk for every scheduled job.
|
||||
#
|
||||
# Usage: bash run_job.sh <job_id> <script_path> [flags...]
|
||||
#
|
||||
# Writes: /var/log/varaverk/<id>.json — status, timestamps, exit code, pid
|
||||
# /var/log/varaverk/<id>.log — appended per run, trimmed to LOG_MAX_LINES
|
||||
#
|
||||
# Status values: running → ok (exit 0) | warn (exit 1) | error (exit 2+)
|
||||
|
||||
JOB_ID="$1"
|
||||
SCRIPT="$2"
|
||||
shift 2
|
||||
|
||||
LOG_DIR="/var/log/varaverk"
|
||||
BASE="${JOB_ID%.sh}"
|
||||
LOG_FILE="$LOG_DIR/$BASE.log"
|
||||
STAT_FILE="$LOG_DIR/$BASE.json"
|
||||
LOG_MAX_LINES=1000
|
||||
|
||||
mkdir -p "$(dirname "$LOG_FILE")" "$(dirname "$STAT_FILE")"
|
||||
|
||||
START=$(date +%s)
|
||||
printf '{"id":"%s","status":"running","start":%s,"pid":%s}\n' \
|
||||
"$JOB_ID" "$START" "$$" > "$STAT_FILE"
|
||||
|
||||
printf '\n── %s ────────────────────────────────────────────────\n' \
|
||||
"$(date '+%Y-%m-%d %H:%M:%S')" >> "$LOG_FILE"
|
||||
|
||||
bash "$SCRIPT" "$@" >> "$LOG_FILE" 2>&1
|
||||
EC=$?
|
||||
|
||||
END=$(date +%s)
|
||||
if [ "$EC" -eq 0 ]; then STATUS="ok"
|
||||
elif [ "$EC" -eq 1 ]; then STATUS="warn"
|
||||
else STATUS="error"
|
||||
fi
|
||||
|
||||
printf '{"id":"%s","status":"%s","start":%s,"end":%s,"exit":%s}\n' \
|
||||
"$JOB_ID" "$STATUS" "$START" "$END" "$EC" > "$STAT_FILE"
|
||||
|
||||
# Trim to last LOG_MAX_LINES to prevent unbounded growth
|
||||
line_count=$(wc -l < "$LOG_FILE" 2>/dev/null || echo 0)
|
||||
if [ "$line_count" -gt "$LOG_MAX_LINES" ]; then
|
||||
tail -n "$LOG_MAX_LINES" "$LOG_FILE" > "${LOG_FILE}.tmp" && mv "${LOG_FILE}.tmp" "$LOG_FILE"
|
||||
fi
|
||||
|
||||
exit $EC
|
||||
@@ -175,11 +175,11 @@ fi
|
||||
|
||||
cleanup_location() {
|
||||
local location="$1" label="$2" max_age="$3"
|
||||
local files_removed=0 bytes_freed=0 files_skipped=0 files_active=0
|
||||
local files_removed=0 bytes_freed=0 files_skipped=0 files_active=0 files_streaming=0
|
||||
|
||||
if [[ ! -d "$location" ]]; then
|
||||
log "$label does not exist — skipping"
|
||||
LOCATION_REMOVED=0 LOCATION_FREED="0B" LOCATION_SKIPPED=0 LOCATION_ACTIVE=0
|
||||
LOCATION_REMOVED=0 LOCATION_FREED="0B" LOCATION_SKIPPED=0 LOCATION_ACTIVE=0 LOCATION_STREAMING=0
|
||||
return
|
||||
fi
|
||||
|
||||
@@ -195,7 +195,8 @@ cleanup_location() {
|
||||
while IFS= read -r open_file; do
|
||||
[[ -n "$open_file" ]] && OPEN_FILES_MAP["$open_file"]=1
|
||||
done < <(timeout 15 lsof +D "$location" 2>/dev/null | awk 'NR>1 {print $9}' | sort -u)
|
||||
log "${#OPEN_FILES_MAP[@]} files currently open in $label"
|
||||
files_streaming=${#OPEN_FILES_MAP[@]}
|
||||
log "$files_streaming files currently open in $label"
|
||||
fi
|
||||
|
||||
# Process aged files
|
||||
@@ -247,19 +248,20 @@ cleanup_location() {
|
||||
freed_human="0B"
|
||||
fi
|
||||
|
||||
log "$label — removed $files_removed files ($freed_human freed) | active: $files_active | skipped: $files_skipped"
|
||||
log "$label — removed $files_removed files ($freed_human freed) | streaming: $files_streaming | aged+open: $files_active | skipped: $files_skipped"
|
||||
|
||||
LOCATION_REMOVED=$files_removed
|
||||
LOCATION_FREED=$freed_human
|
||||
LOCATION_SKIPPED=$files_skipped
|
||||
LOCATION_ACTIVE=$files_active
|
||||
LOCATION_STREAMING=$files_streaming
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Transcode Cleanup ━━━
|
||||
# ==============================================================================================
|
||||
START=$(date +%s)
|
||||
TOTAL_REMOVED=0 TOTAL_SKIPPED=0 TOTAL_ACTIVE=0
|
||||
TOTAL_REMOVED=0 TOTAL_SKIPPED=0 TOTAL_ACTIVE=0 TOTAL_STREAMING=0
|
||||
RAMDISK_FREED="0B" SSD_FREED="0B"
|
||||
|
||||
# Cleanup ramdisk
|
||||
@@ -268,6 +270,7 @@ if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
|
||||
TOTAL_REMOVED=$(( TOTAL_REMOVED + LOCATION_REMOVED ))
|
||||
TOTAL_SKIPPED=$(( TOTAL_SKIPPED + LOCATION_SKIPPED ))
|
||||
TOTAL_ACTIVE=$(( TOTAL_ACTIVE + LOCATION_ACTIVE ))
|
||||
TOTAL_STREAMING=$(( TOTAL_STREAMING + LOCATION_STREAMING ))
|
||||
RAMDISK_FREED=$LOCATION_FREED
|
||||
else
|
||||
log "Ramdisk not mounted — skipping ramdisk cleanup"
|
||||
@@ -279,6 +282,7 @@ if [[ -d "$TRANSCODE_SSD" ]]; then
|
||||
TOTAL_REMOVED=$(( TOTAL_REMOVED + LOCATION_REMOVED ))
|
||||
TOTAL_SKIPPED=$(( TOTAL_SKIPPED + LOCATION_SKIPPED ))
|
||||
TOTAL_ACTIVE=$(( TOTAL_ACTIVE + LOCATION_ACTIVE ))
|
||||
TOTAL_STREAMING=$(( TOTAL_STREAMING + LOCATION_STREAMING ))
|
||||
SSD_FREED=$LOCATION_FREED
|
||||
else
|
||||
log "SSD fallback not found — skipping SSD cleanup"
|
||||
@@ -307,7 +311,8 @@ echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_RAM Ramdisk freed: $RAMDISK_FREED"
|
||||
echo "$ICON_DISK SSD freed: $SSD_FREED"
|
||||
echo "$ICON_TRASH Removed: $TOTAL_REMOVED files"
|
||||
echo "$ICON_RUNNING Active: $TOTAL_ACTIVE files (open — protected)"
|
||||
echo "$ICON_RUNNING Streaming: $TOTAL_STREAMING files (open by Emby — untouched)"
|
||||
echo "$ICON_SHIELD Protected: $TOTAL_ACTIVE aged files saved by open-file check"
|
||||
echo "$ICON_TRASH Skipped: $TOTAL_SKIPPED files"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
|
||||
|
||||
@@ -259,6 +259,50 @@ fi
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Conf Upgrade ━━━
|
||||
# ==============================================================================================
|
||||
# Merges new conf structure into the live conf files after every pull.
|
||||
# New keys → added with template defaults (user fills in once).
|
||||
# Removed keys → dropped. Existing values → always preserved.
|
||||
# Silent when already up to date — no overhead on unchanged pulls.
|
||||
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Conf Upgrade ━━━"
|
||||
|
||||
UPGRADE_SCRIPT="$TARGET_DIR/Deployment/conf_upgrade.sh"
|
||||
TMPL_DIR="$TARGET_DIR/Deployment/conf_templates"
|
||||
CONF_DIR="$TARGET_DIR/Configurations"
|
||||
|
||||
if [[ ! -f "$UPGRADE_SCRIPT" ]]; then
|
||||
log "conf_upgrade.sh not found — skipping (pre-deployment-folder repo)"
|
||||
elif [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would upgrade master.conf and ${MY_ID,,}.conf"
|
||||
elif [[ "$SYNC_SUCCESS" == true ]]; then
|
||||
_DRY=""
|
||||
|
||||
# master.conf
|
||||
if [[ -f "$TMPL_DIR/master.conf" && -f "$CONF_DIR/master.conf" ]]; then
|
||||
bash "$UPGRADE_SCRIPT" \
|
||||
--template "$TMPL_DIR/master.conf" \
|
||||
--target "$CONF_DIR/master.conf" \
|
||||
--backup $_DRY
|
||||
else
|
||||
warn "master.conf template or target not found — skipping"
|
||||
fi
|
||||
|
||||
# This server's host conf only — sparse checkout ensures we have it
|
||||
HOST_CONF="$CONF_DIR/${MY_ID,,}.conf"
|
||||
if [[ -f "$TMPL_DIR/host.conf.template" && -f "$HOST_CONF" ]]; then
|
||||
bash "$UPGRADE_SCRIPT" \
|
||||
--template "$TMPL_DIR/host.conf.template" \
|
||||
--target "$HOST_CONF" \
|
||||
--backup $_DRY
|
||||
else
|
||||
warn "${MY_ID,,}.conf or host.conf.template not found — skipping"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
|
||||
Reference in New Issue
Block a user