From c377ddfcca9c57903d0bee0debcad27c07df2a28 Mon Sep 17 00:00:00 2001 From: Gmer4Lfe Date: Sat, 1 Aug 2026 22:44:23 -0400 Subject: [PATCH] Complete the header template across Partnership, Kernel, Deployment and Plugin Finishes the pass: every script now documents its safeguards, and the deliberate absences in the sourced libraries are recorded so they are not "corrected" later. --- Orchestrators/watchdog_orchestrator.sh | 4 +- Partnership/gitea_ssh_setup.sh | 25 +++ Partnership/onboard_cancel.sh | 96 +++++++++++ Partnership/partnership_offboard.sh | 73 +++++++++ Partnership/partnership_transfer.sh | 25 +++ Partnership/share_setup.sh | 105 +++++++++++- Partnership/ssh_setup.sh | 151 ++++++++++++++---- Plugin/build.sh | 67 ++++++++ Plugin/plugin_setup.sh | 43 +++++ Plugin/unraid/Partnership/containers.sh | 76 +++++++++ Plugin/unraid/System_Essentials/mover_stop.sh | 22 +++ .../System_Essentials/php_fpm_max_children.sh | 13 ++ .../System_Essentials/unraid_api_key_renew.sh | 33 ++++ .../System_Essentials/user_scripts_stop.sh | 34 ++++ Plugin/unraid/Tools/api_cache_writer.sh | 52 ++++++ Plugin/unraid/Tools/recreate_shares.sh | 35 ++++ .../unraid/Tools/remote_arr_cache_writer.sh | 38 +++++ Plugin/unraid/Tools/storage_migrate.sh | 45 ++++++ .../Watchdogs/System/webgui_watchdog.sh | 23 +++ Plugin/unraid/adapter.sh | 78 +++++++++ Plugin/unraid/user_script_plug-in.sh | 76 +++++++++ Watchdogs/resource_watchdog.sh | 2 +- common.sh | 94 +++++++++++ load_config.sh | 59 +++++++ 24 files changed, 1237 insertions(+), 32 deletions(-) diff --git a/Orchestrators/watchdog_orchestrator.sh b/Orchestrators/watchdog_orchestrator.sh index 0c1a05c..bb0a74a 100755 --- a/Orchestrators/watchdog_orchestrator.sh +++ b/Orchestrators/watchdog_orchestrator.sh @@ -55,7 +55,7 @@ # # Lock Acquisition # acquire_lock in strict mode — if the previous cycle is still running, this one exits -# rather than queuing. At a one-minute cadence a waiting lock would pile up cycles +# rather than queuing. At a 15-minute cadence a waiting lock would pile up cycles # behind a slow watchdog and eventually run them all at once. # # Host Detection @@ -63,7 +63,7 @@ # # Empty Job List Guard # Exits with an error and a notification if WATCHDOG_ORCHESTRATOR_SCRIPTS is empty. -# Without it the cycle reports "0/0 passed" and exits 0 every minute — indistinguishable +# Without it the cycle reports "0/0 passed" and exits 0 every cycle — indistinguishable # from a healthy run, while nothing at all is being monitored. # # Array Check diff --git a/Partnership/gitea_ssh_setup.sh b/Partnership/gitea_ssh_setup.sh index 8658ba5..48b4c92 100755 --- a/Partnership/gitea_ssh_setup.sh +++ b/Partnership/gitea_ssh_setup.sh @@ -41,6 +41,31 @@ # Skipped if entry already exists — use --force to overwrite # # ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# Separate Key From the Rsync Key +# Gitea authentication uses its own keypair rather than reusing the rsync automation key. +# The two have different lifecycles and different blast radii — re-keying Gitea should +# never break cross-host rsync, and revoking a partner's rsync access should not lock this +# host out of its own repository. +# +# Idempotent Registration +# Both generation and registration are skipped when already satisfied, and registration is +# matched on the public key itself rather than on a label. A key registered under a +# different title is still the same key, and re-registering it would leave duplicate +# entries accumulating in Gitea on every re-run. +# +# Locate, Do Not Assume +# The API endpoint is resolved at runtime — container-local first, then GITEA_DOMAIN. Same +# reasoning as git_pull_execute.sh: Gitea's address changes with container restarts and +# migrations, so hardcoding it guarantees an eventual break. +# +# Force Is Explicit +# Regeneration invalidates the key already registered in Gitea, so it requires --force. A +# bare re-run can never cost this host its repository access. +# +# ============================================================================================== # OPERATIONAL SAFEGUARDS # ============================================================================================== # diff --git a/Partnership/onboard_cancel.sh b/Partnership/onboard_cancel.sh index 1d89934..c5449a0 100755 --- a/Partnership/onboard_cancel.sh +++ b/Partnership/onboard_cancel.sh @@ -8,6 +8,102 @@ # Removes SSH keys between HOST1 and HOST2 in the specified direction. # Safe to run at any phase. Clears related setup.db flags. # +# The escape hatch for a half-finished onboard: partnership setup is multi-phase, and an +# attempt abandoned midway leaves keys installed and phase flags set. This unwinds that so +# onboarding can be started cleanly rather than resumed from an unknown state. +# +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== +# +# Per direction requested: +# +# h1 (HOST1 → HOST2) +# 1. SSH to HOST2 and delete HOST1's public key line from its authorized_keys +# 2. Clear HOST2's phase / key-ready flags from the setup db +# 3. Delete the local HOST1 key pair +# +# h2 (HOST2 → HOST1) +# Remove HOST2's public key from HOST1's authorized_keys, identifying the key by +# HOST2's hostname in the key comment. +# +# both — run each direction in turn. +# +# Key removal is matched on the key blob or hostname comment, never on line number. +# +# ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# Safe at Any Phase +# Onboarding is multi-phase and can fail anywhere in it. This runs against whatever state +# exists rather than requiring a known starting point — a missing key or an absent flag is +# a no-op, not an error. Cancelling twice is harmless. +# +# Direction Is Explicit +# Removing a key is not symmetric: it breaks authentication for whichever side loses it. +# The direction must be stated, and defaults to h1 (this host's own outbound key) rather +# than to both, so an unqualified run cannot sever the partner's access to you. +# +# Unwind, Do Not Repair +# The job is to return to a clean pre-onboard state so onboarding can be re-run from the +# top. It deliberately does not attempt to salvage or resume a partial setup — a known +# empty state is worth more than a guessed-at partial one. +# +# Keys and Flags Together +# Removing the key without clearing the setup-db flags would leave onboarding believing a +# phase had completed. Both are cleared in the same pass for that reason. +# +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# Root Enforcement +# Deletes key pairs from /root/.ssh and edits authorized_keys on both ends as root. +# +# Lock Acquisition +# acquire_lock prevents this racing an in-progress onboard, which would otherwise be +# installing the very keys this is removing. +# +# Host Detection +# detect_hosts() resolves MY_ID / REMOTE_ID so the direction flags map to real hosts. +# +# Direction Default +# Defaults to h1 — never removes the partner's inbound key unless explicitly asked. +# +# Targeted Key Removal +# authorized_keys lines are matched by key blob or hostname comment. Nothing is removed +# positionally, so an unrelated key can never be deleted because it sat on a given line. +# +# SSH Timeout +# The remote edit is wrapped in SSH_TIMEOUT — an unreachable partner fails fast rather +# than hanging a cancel that still has local cleanup to do. +# +# Idempotent +# Absent keys and absent flags are skipped silently. Re-running is safe. +# +# Dry Run Support +# --dry-run reports every key and flag it would remove, and removes none. +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master.conf +# +# SSH_TIMEOUT +# Bounds the remote authorized_keys edit. +# +# host*.conf +# +# SSH_KEY +# Key used to reach the partner, and the local pair deleted in the h1 direction. +# +# HOST* — hostnames, used to identify which key comment belongs to which side. +# +# Setup state lives in the platform setup db (platform_setup_db_path) — the phase and +# key-ready flags cleared here are the same ones partnership_onboard.sh sets. +# # ============================================================================================== # RUNTIME MODES # ============================================================================================== diff --git a/Partnership/partnership_offboard.sh b/Partnership/partnership_offboard.sh index 0334383..317e611 100755 --- a/Partnership/partnership_offboard.sh +++ b/Partnership/partnership_offboard.sh @@ -37,6 +37,79 @@ # Step 8: SSH revocation — revoke keys both directions, write state, signal owner # # ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# Sync Before Severing +# The final sync (owner step 2) runs before any state change, so the mirror leaves with +# current Critical-Data rather than a snapshot from whenever the last scheduled sync +# happened. Once keys are revoked there is no second chance to move data. +# +# Stop the Sync First +# Step 1 on both paths halts rsync before anything else. A sync running through a +# partnership teardown would be writing to a partner that is having its access removed +# underneath it. +# +# Revoke Last, Not First +# SSH keys and Emby admin are revoked at the end. Every earlier step needs working remote +# access — revoking up front would strand the remaining cleanup on the far side and leave +# the mirror holding containers nobody can remove. +# +# Both Sides Land Somewhere Valid +# Each path restarts the host's own parked containers before finishing. Offboarding must +# leave two working standalone servers, not one working server and one stripped of the +# coverage it was relying on. +# +# Role Detected, Not Declared +# Owner and mirror run different sequences, and the role is derived rather than passed in. +# A human choosing the wrong path would run the owner's remote-cleanup steps against a +# server that never deployed anything. +# +# Blocklist Is the Enforcement +# Writing INACTIVE state is not enough on its own — a stale cron or a script mid-flight +# could still attempt a sync. The mirror is blocklisted so rsync.sh refuses it outright, +# independently of whatever any config still says. +# +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# Root Enforcement +# Container removal, conf edits and SSH key revocation all require root. +# +# Lock Acquisition +# acquire_lock "strict" — an offboard is not resumable partway, so a second instance is +# refused rather than queued behind the first. +# +# Host Detection +# detect_hosts() resolves MY_ID / REMOTE_ID, which the role detection builds on. +# +# Docker Presence Check +# Verified before any container removal is attempted. +# +# Ordered Teardown +# The step sequence is the safeguard — sync, then reconfigure, then remove, then restart, +# then revoke, then record. Reordering breaks the guarantees above. +# +# Own Stack Restored +# Parked containers are brought back up on both sides before the run completes. +# +# State Written Both Ends +# INACTIVE is written locally and pushed to the mirror, so neither side is left believing +# a partnership is still active. +# +# Partner Blocklisted +# The mirror is added to the partnership blocklist, which rsync.sh checks and refuses on — +# stale access cannot survive the offboard. +# +# Tailscale Grace Window +# Device removal happens after state is written, not before, so the final state push +# cannot be cut off by removing its own transport. +# +# Dry Run Support +# --dry-run walks the full sequence reporting each step without executing any. +# +# ============================================================================================== # CONFIGURATION # ============================================================================================== # diff --git a/Partnership/partnership_transfer.sh b/Partnership/partnership_transfer.sh index 350ab83..1bdf41c 100755 --- a/Partnership/partnership_transfer.sh +++ b/Partnership/partnership_transfer.sh @@ -28,6 +28,31 @@ # Step 5: Write State — ACTIVE written locally and pushed to new mirror # # ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# Config Changes Hands, Data Does Not +# No containers move and no appdata is copied. A transfer rewrites who is authoritative +# and where WebUIs point; the sync direction reverses naturally on the next fallback or +# critical-sync cycle. Keeping data out of the transfer is what makes it cheap enough to +# be reversible. +# +# Owner Initiates, Always +# Only the current owner can run this. The owner holds the authoritative config, so a +# mirror-initiated transfer would be writing ownership state it does not own — and if both +# sides ran it, neither would be owner. +# +# Prove Health Before Swapping +# Both servers must pass consecutive health checks first. Handing ownership to a partner +# that is unhealthy converts a recoverable situation into an outage with the authoritative +# side on the weaker host. +# +# Roles Swap Atomically +# Owner and mirror are two ends of one relationship, not independent flags. Any window +# where both believe they are owner — or neither does — is worse than the transfer simply +# failing, so the swap is written as one transition rather than two updates. +# +# ============================================================================================== # OPERATIONAL SAFEGUARDS # ============================================================================================== # diff --git a/Partnership/share_setup.sh b/Partnership/share_setup.sh index 1170b43..9b93d6b 100755 --- a/Partnership/share_setup.sh +++ b/Partnership/share_setup.sh @@ -25,7 +25,96 @@ # Remote admin assigns disk sets from the Unraid UI after onboarding. # # ============================================================================================== -# USAGE +# OPERATIONAL MODEL +# ============================================================================================== +# +# 1. Resolve the mirror and its Tailscale IP — unresolvable aborts before any remote call +# 2. Detect the remote's appdata cache pool from its own appdata.cfg (default: cache) +# 3. For each path across the owner's daily/weekly/critical/intermediate share lists: +# a. Extract the top-level Unraid share name +# b. Remote already has a .cfg for it? → skip, never modify +# c. Otherwise use the LOCAL .cfg as a template: +# - substitute the remote's detected pool +# - clear disk include/exclude (disk layouts differ per server) +# d. Write the .cfg to remote /boot/config/shares/ and mkdir -p the share directory +# e. Sub-paths (e.g. appdata-Fallback/Critical-Data) get their subdir created after +# the top-level share exists +# +# ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# Create Only, Never Modify +# An existing remote .cfg is always left alone. The remote admin may have deliberately +# tuned a share's pool, allocation or disk set — this script has no way to tell an +# intentional setting from a stale one, so it never overwrites. +# +# Disk Layout Is Not Portable +# Include/exclude lists are cleared rather than copied, because the two servers have +# different disks. Copying the owner's disk set onto a mirror with a different array +# would produce a share pointing at disks that do not exist. +# +# Detect the Pool, Do Not Assume It +# The remote's cache pool name is read from its own appdata.cfg rather than hardcoded or +# copied from local. Pool names differ per server and a wrong one silently lands appdata +# on the array. +# +# Media Shares Land Array-Only +# Shares with shareUseCache=no are created without a pool assignment. Disk sets are the +# remote admin's call from the Unraid UI after onboarding. +# +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# Root Enforcement +# Reads $SSH_KEY from /root/.ssh and the local /boot/config/shares/*.cfg, and writes share +# configs onto the mirror as root. +# +# Lock Acquisition +# acquire_lock prevents concurrent runs. Two instances could both observe a share as +# missing and race to create it. +# +# Host Detection +# detect_hosts() resolves REMOTE_SERVER_NAME — the mirror this script targets. +# +# Mirror Resolution Guard +# Aborts if REMOTE_SERVER_NAME is unset or its Tailscale IP cannot be resolved, before any +# remote command is attempted. +# +# Existing Share Protection +# A remote .cfg that already exists is never touched — see Create Only above. +# +# SSH Timeouts and BatchMode +# Every remote call uses ConnectTimeout and BatchMode=yes, so an unreachable or +# password-prompting mirror fails fast instead of hanging the onboarding run. +# +# Pool Fallback +# An undetectable remote pool defaults to "cache" rather than writing an empty pool name +# into the share config. +# +# Dry Run Support +# --dry-run reports every share it would create and writes nothing. +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# host*.conf (aliased by detect_hosts()) +# +# HOST*_DAILY_SYNC_SHARES / _WEEKLY_ / _CRITICAL_ / _INTERMEDIATE_ +# The share lists this script reads. Any path appearing in one of these on the owner +# is a share the mirror is expected to have. +# +# SSH_KEY +# Key used for every remote call. Written by ssh_setup.sh. +# +# master.conf +# +# HOST* — hostnames, used to resolve the mirror via detect_hosts() +# +# ============================================================================================== +# RUNTIME MODES # ============================================================================================== # # share_setup.sh @@ -40,9 +129,21 @@ set -uo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/../load_config.sh" -detect_hosts parse_args "$@" +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== +# Reads $SSH_KEY from /root/.ssh, reads local /boot/config/shares/*.cfg, and writes share +# configs onto the mirror as root. +[[ "$EUID" -ne 0 ]] && { error "Must be run as root"; exit 1; } + +# Writes share .cfg files to the mirror. Two concurrent runs could both see a share as +# missing and race to create it. +acquire_lock + +detect_hosts + # ── Resolve mirror ──────────────────────────────────────────────────────────────────────────── [[ -z "${REMOTE_SERVER_NAME:-}" ]] && { error "Cannot determine mirror hostname — check HOST* in master.conf"; exit 1; } diff --git a/Partnership/ssh_setup.sh b/Partnership/ssh_setup.sh index bcdd32d..881d551 100755 --- a/Partnership/ssh_setup.sh +++ b/Partnership/ssh_setup.sh @@ -2,40 +2,137 @@ # ============================================================================================== # ============================= SSH Setup ====================================================== # ============================================================================================== +# +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── # Generates the SSH keypair for rsync automation and installs it on the remote server. +# Every cross-host operation in the ecosystem — rsync, conf sync, fallback container +# control, the upgrade webhook — authenticates with this key. If it is missing or broken, +# the mesh silently degrades to single-host. +# # Key named after this server: hostname lowercased, unraid- prefix stripped. -# unRAID-Gmer4Lfe → gmer4lfe_rsync_automation +# unRAID-Gmer4Lfe → gmer4lfe_rsync_automation # unRAID-Jayred365 → jayred365_rsync_automation -# Idempotent — skips generation if key already exists (use --force to regenerate). -# Updates host*.conf with key path on success. # -# ── MODES ───────────────────────────────────────────────────────────────────────────────────── -# (default) — generate key if missing, copy to remote, update conf -# --force — regenerate key even if it exists, re-copy to remote -# --validate — test SSH auth to remote, track strikes, notify at limit -# --status — show key state, fingerprint, remote connectivity -# --dry-run — preview without creating, copying, or updating conf +# Idempotent — skips generation if the key already exists (--force to regenerate). +# Updates host*.conf with the key path on success. # -# ── STRIKE SYSTEM (--validate) ──────────────────────────────────────────────────────────────── -# Called during partnership --check cycles to detect broken SSH auth. -# Tracks consecutive SSH auth failures — not network unreachability. -# Remote Tailscale IP unreachable = network issue → not counted as SSH strike. -# Remote reachable but SSH auth fails = key issue → strike incremented. -# Strikes reset automatically after SSH_STRIKE_RESET_HRS of clean connectivity. -# At SSH_MAX_STRIKES: notify + return exit 2 (caller can escalate). -# State: DATA_DIR/ssh_strikes_{REMOTE_SERVER_NAME}.db +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== # -# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── -# SSH_MAX_STRIKES — consecutive failures before notifying (default 5) -# SSH_STRIKE_RESET_HRS — hours since last failure before counter resets (default 24) +# Setup (default): +# 1. Key exists? → skip generation unless --force +# 2. Generate keypair, named from this host +# 3. Copy the public key to the remote's authorized_keys +# 4. Verify authentication actually works before claiming success +# 5. Write the key path into host*.conf +# +# Validate (--validate), called during partnership --check cycles: +# Remote unreachable on Tailscale → network problem, NOT counted as a strike +# Remote reachable but SSH auth fails → key problem, strike incremented +# Clean connectivity for SSH_STRIKE_RESET_HRS → strikes reset automatically +# At SSH_MAX_STRIKES → notify and return exit 2 so the caller can escalate +# +# State: DATA_DIR/ssh_strikes_{REMOTE_SERVER_NAME}.db +# +# ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# Distinguish Unreachable From Unauthorised +# The strike system counts SSH auth failures only. A partner that is simply offline is a +# network condition, and counting it would fire a key-rotation alarm every time the remote +# reboots. Only "I can reach you but you will not let me in" is a key problem. +# +# Idempotent by Default, Destructive Only on Request +# A bare run never replaces an existing key. Regenerating invalidates every authorized_keys +# entry the old key was in — including on hosts this script is not talking to right now — +# so it requires --force explicitly. +# +# Verify Before Recording +# The key path is written into host*.conf only after authentication has been proven to +# work. Recording a key that does not authenticate would leave every downstream script +# pointed at a credential that silently fails. +# +# Strikes Reset on Recovery +# Counters clear themselves after a period of clean connectivity, so a transient outage +# does not accumulate toward an alarm across unrelated weeks. +# +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# Root Enforcement +# Reads and writes /root/.ssh and installs keys on the remote as root. +# +# Lock Acquisition +# acquire_lock prevents concurrent runs. Two instances generating or copying keys at once +# could leave authorized_keys holding a key whose private half was already replaced. +# +# Host Detection +# detect_hosts() resolves MY_ID / REMOTE_ID for key naming and remote targeting. +# +# Existing Key Protection +# Generation is skipped when a key is present. Overwriting requires --force. +# +# Auth Verified Before Conf Write +# host*.conf is updated only after a successful authentication test. +# +# Network-vs-Auth Discrimination +# Unreachable remotes never increment the strike counter — see Design Principles. +# +# Strike Ceiling +# SSH_MAX_STRIKES bounds how long a genuinely broken key goes unreported, and exit 2 lets +# the caller decide whether that is escalation-worthy. +# +# Local-Only Escape Hatch +# --local-only generates the key without touching the remote, for onboarding a partner +# that is not reachable yet. +# +# Dry Run Support +# --dry-run previews generation, copy and conf update without performing any. +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master.conf +# +# SSH_MAX_STRIKES +# Consecutive SSH auth failures before notifying (default: 5) +# +# SSH_STRIKE_RESET_HRS +# Hours of clean connectivity before the strike counter resets (default: 24) +# +# host*.conf +# +# HOST*_SSH_KEY +# Written by this script on success. Read by rsync.sh, conf_sync.sh, fallback.sh and +# upgrade_webhook_handler.sh — every cross-host operation depends on it. +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# ssh_setup.sh +# Initial setup — generate if missing, copy to remote, update conf. Idempotent. +# +# ssh_setup.sh --force +# Regenerate the key even if one exists, and re-copy to the remote. +# +# ssh_setup.sh --validate +# Health check with strike tracking. Exit 2 at the strike limit. +# +# ssh_setup.sh --status +# Show key state, fingerprint, and remote connectivity. Then exit. +# +# ssh_setup.sh --local-only +# Generate the key locally and skip the remote copy. +# +# ssh_setup.sh --dry-run / --log +# Supported by every mode above. # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# Partnership/ssh_setup.sh — initial setup (idempotent) -# Partnership/ssh_setup.sh --force — regenerate + re-copy -# Partnership/ssh_setup.sh --validate — health check + strike tracking -# Partnership/ssh_setup.sh --status — show key and connectivity state -# Partnership/ssh_setup.sh --local-only — generate key locally, skip remote copy -# Any mode supports --dry-run and --log # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/Plugin/build.sh b/Plugin/build.sh index 538e9ed..08e8a2f 100755 --- a/Plugin/build.sh +++ b/Plugin/build.sh @@ -26,6 +26,73 @@ # rebuild cron when the array mounts. Scripts stay on appdata (git clone). # # ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== +# +# 1. Resolve version — first argument, else today's date as YYYY.MM.DD +# 2. Verify makepkg exists (Slackware tooling — this only runs on unRAID) +# 3. Stage Plugin/unraid/ into the install layout the package expects +# 4. makepkg the stage into Plugin/dist/--noarch-1.txz +# 5. sha256sum the result and write the .sha256 alongside it +# 6. Rewrite and in varaverk.plg so the .plg +# always points at the package just built +# +# ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# Release Path, Not the Dev Path +# Day-to-day work uses plugin_setup.sh, which symlinks the source directory so edits are +# live immediately. This produces the artifact unRAID reinstalls from flash on every boot. +# Keeping the two separate is what allows fast iteration without shipping half-finished +# files into a package. +# +# The .plg Always Matches the Package +# Version and checksum are written into varaverk.plg in the same run that produces the +# .txz. A .plg pointing at a checksum it was not built against fails to install with a +# mismatch error, so the two are never updated independently. +# +# Date Version by Default +# An omitted version becomes today's date. Builds are therefore always ordered and always +# unique without anyone maintaining a counter. +# +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# No Root — Deliberate +# Writes only into Plugin/dist/ and rewrites varaverk.plg inside the repo. It installs +# nothing and touches nothing under /usr or /boot, so it needs no privilege. Requiring +# root would mean building release artifacts as root for no reason. +# +# makepkg Presence Check +# Aborts immediately if makepkg is missing, with a note that this runs on unRAID. The +# Slackware toolchain is not present on a normal dev machine and a partial build would be +# worse than a clear refusal. +# +# Checksum Written From the Real Artifact +# The sha256 is computed from the .txz that was just produced, never assumed or carried +# forward, so the .plg cannot advertise a checksum for a different build. +# +# Staged Build +# The package is assembled from a staging directory rather than from the live source +# tree, so an in-progress edit cannot end up inside a release artifact. +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# None — this is a build tool, not a runtime script. It reads no conf and calls no +# load_config.sh. +# +# Inputs are positional and derived: +# $1 version string (default: today, YYYY.MM.DD) +# Plugin/unraid/ source tree that gets packaged +# Plugin/varaverk.plg rewritten in place with the new version and checksum +# +# Output: Plugin/dist/--noarch-1.txz and its .sha256 +# +# ============================================================================================== # RUNTIME MODES # ============================================================================================== # diff --git a/Plugin/plugin_setup.sh b/Plugin/plugin_setup.sh index 90a46d6..b1e799c 100755 --- a/Plugin/plugin_setup.sh +++ b/Plugin/plugin_setup.sh @@ -14,9 +14,52 @@ # immediately without re-running this script. # # ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== +# +# 1. Detect the OS from /etc/*-version markers +# 2. Resolve Plugin// as the source — missing means abort before any change +# 3. Inspect the web server's plugin location: +# symlink already → remove it, it will be recreated +# real directory → refuse, this is an existing install to clean up by hand +# absent → proceed +# 4. Create the symlink +# +# After this, every edit under Plugin// is live immediately — that is the whole point, +# and why this is the development path rather than build.sh's packaged one. +# +# ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# Symlink, Not Copy +# The web server points at the repo rather than holding its own copy, so there is exactly +# one source of truth for the plugin's PHP. A copy would drift the moment anyone edited +# either side, and the drift would only surface as a UI behaving unlike the code. +# +# Run Once, By Hand +# Never scheduled. It performs a one-time structural change to where the web server looks; +# putting that on a timer would mean an unattended job could recreate a symlink over a +# deliberate manual install. +# +# Refuse, Do Not Replace +# A real directory at the target is left alone and the script exits. That directory is a +# genuine installation, and silently deleting it to make room for a symlink would discard +# an install this script did not create. +# +# OS Detected, Not Assumed +# The source directory comes from the detected OS, so the same script works unchanged on a +# second platform once Plugin// exists — matching the adapter's approach. +# +# ============================================================================================== # OPERATIONAL SAFEGUARDS # ============================================================================================== # +# No Root — Deliberate +# Run by hand during install, when the operator already has whatever privilege the web +# server's plugin path requires. The guards below are structural rather than privilege +# based: the dangerous outcome here is clobbering an existing install, not lacking rights. +# # Idempotent Symlink # Removes any existing symlink at the target before recreating it. Safe to # re-run after a repo move without leaving stale paths. diff --git a/Plugin/unraid/Partnership/containers.sh b/Plugin/unraid/Partnership/containers.sh index 6dd11fc..f18a077 100755 --- a/Plugin/unraid/Partnership/containers.sh +++ b/Plugin/unraid/Partnership/containers.sh @@ -23,6 +23,82 @@ # - Deployed stack tracking via _STACK_DEPLOYED / _STACK_FAILED counters # # ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# Platform Adapter, Same Contract as adapter.sh +# Container deployment is Unraid-specific — CA XML templates, dockerMan paths, the +# templates-user directory. Confining it here means the partnership scripts contain no +# Unraid knowledge and a second platform is a new Partnership/containers.sh, not edits +# scattered through onboard and offboard. +# +# Deploy From the Template, Not a Copy +# Containers are created from the CA XML the operator already maintains, so a partnership +# deployment produces the same container the Unraid UI would. Hand-built docker run lines +# would drift from the template the moment anyone edited it in the UI. +# +# GPU Detection Cached Per Session +# The remote GPU type is probed once and reused. Onboarding deploys several containers and +# each would otherwise repeat the same SSH round-trip to learn an answer that cannot change +# mid-run. +# +# Count Outcomes, Do Not Abort +# Failures increment _STACK_FAILED rather than exiting. A stack deployment that fails on +# one container should report which one and continue — the caller owns whether a partial +# stack is acceptable, and it is the only side with the context to decide. +# +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# No Root, No Lock, No detect_hosts — Deliberate +# Sourced by partnership_onboard.sh and partnership_offboard.sh, both of which already +# enforce root and hold their own strict locks. Re-checking here would be redundant, and +# taking a lock would deadlock against the caller's. Do not add them. +# +# Caller Scope Is the Contract +# Functions read MIRROR, MIRROR_IP, MIRROR_SSH_KEY, SSH_TIMEOUT, DRY_RUN and SCRIPTS_ROOT +# from the caller. That coupling is deliberate — it keeps one definition of who the mirror +# is — but it means these functions are only valid inside the partnership scripts and +# cannot be sourced standalone. +# +# DRY_RUN Honoured Throughout +# Every deploy and cleanup path checks the caller's DRY_RUN, so a dry-run onboard makes no +# remote container changes. +# +# SSH Timeouts on Every Remote Call +# All remote operations use the caller's SSH_TIMEOUT — an unreachable mirror cannot hang +# an onboard partway through a stack deployment. +# +# Template Existence Checked +# A missing CA XML is counted as a failure for that container rather than producing a +# container built from nothing. +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# No config vars of its own. Inputs come from the calling script's scope (see above). +# +# Platform paths it owns: +# +# /boot/config/plugins/dockerMan/templates-user +# Unraid CA template directory. Source of every container definition deployed here. +# +# The container lists themselves live in host*.conf as HOST*_PARTNERSHIP_AUTH_STACK and +# HOST*_PARTNERSHIP_ARR_STACK — read by the partnership scripts, passed in as arguments. +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# None — sourced, never executed: +# +# source "$SCRIPTS_ROOT/Plugin/$PLATFORM/Partnership/containers.sh" +# +# No argument parsing and no flags. Dry-run behaviour comes from the caller's DRY_RUN. +# +# ============================================================================================== TEMPLATES_DIR="/boot/config/plugins/dockerMan/templates-user" diff --git a/Plugin/unraid/System_Essentials/mover_stop.sh b/Plugin/unraid/System_Essentials/mover_stop.sh index fd58d92..dcecbd0 100755 --- a/Plugin/unraid/System_Essentials/mover_stop.sh +++ b/Plugin/unraid/System_Essentials/mover_stop.sh @@ -29,6 +29,28 @@ # used as a last resort and may leave a file split across cache and array. # # ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# Warn the Room First +# A wall message goes out before the mover is signalled. The mover moves other people's +# data between cache and array; anyone watching a transfer deserves to know why it stopped. +# +# Graceful First, Forced Last +# SIGTERM with a configurable window, then SIGKILL only if it is ignored. The mover is +# mid-file-move by definition — giving it the chance to finish the current file and exit +# cleanly is the difference between a stopped transfer and a half-moved file. +# +# Not Running Is Success +# An absent mover exits 0. Callers use this as a precondition ("ensure the mover is not +# running"), not as a command that must find something to kill — treating "already stopped" +# as failure would abort every reboot on a quiet system. +# +# Stop, Never Start +# This script has no counterpart that restarts the mover. Unraid's own schedule owns when +# the mover runs; this only ever removes it from the picture for a window. +# +# ============================================================================================== # OPERATIONAL SAFEGUARDS # ============================================================================================== # diff --git a/Plugin/unraid/System_Essentials/php_fpm_max_children.sh b/Plugin/unraid/System_Essentials/php_fpm_max_children.sh index 0b2070f..c8863da 100755 --- a/Plugin/unraid/System_Essentials/php_fpm_max_children.sh +++ b/Plugin/unraid/System_Essentials/php_fpm_max_children.sh @@ -41,6 +41,19 @@ # 7. Read back config to confirm value applied # # ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== +# +# 1. Read the current pm.max_children from the PHP-FPM pool config +# 2. Already at or above the target → exit silently, no write, no restart +# 3. Otherwise rewrite the value and restart PHP-FPM via the adapter +# 4. Verify PHP-FPM came back up +# +# Runs at array start, before the WebGUI sees real load. The setting does not survive an +# unRAID update — the OS replaces the pool config — which is why this reapplies every boot +# rather than being a one-time install step. +# +# ============================================================================================== # OPERATIONAL SAFEGUARDS # ============================================================================================== # diff --git a/Plugin/unraid/System_Essentials/unraid_api_key_renew.sh b/Plugin/unraid/System_Essentials/unraid_api_key_renew.sh index f30871e..e4b0452 100755 --- a/Plugin/unraid/System_Essentials/unraid_api_key_renew.sh +++ b/Plugin/unraid/System_Essentials/unraid_api_key_renew.sh @@ -28,6 +28,19 @@ # without a separate sync step. # # ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== +# +# 1. Check whether a Varaverk key already exists in the unraid-api registry +# 2. Create or overwrite it — the registry is ephemeral, so re-registering is the norm +# 3. Write the resulting key into this host's conf, replacing any previous value +# 4. Report whether the key was created, refreshed, or unchanged +# +# Runs at array start. The registry does not survive OS updates or an unraid-api restart, +# which is why this re-registers unconditionally rather than only when the key is missing — +# a key present in the conf but absent from the registry is the exact failure it repairs. +# +# ============================================================================================== # OPERATIONAL SAFEGUARDS # ============================================================================================== # @@ -37,6 +50,22 @@ # dry-run mode — shows what would happen without touching anything # # ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# host*.conf +# +# HOST*_UNRAID_API_KEY +# Written by this script every array start. Read by the plugin's PHP for enhanced +# monitoring. Treated as output, not input — an existing value is always replaced, +# because a conf value that no longer matches the registry is precisely the broken +# state this exists to fix. +# +# Platform-owned: +# +# The unraid-api service registry — ephemeral, cleared by OS updates and service restarts. +# +# ============================================================================================== # RUNTIME MODES # ============================================================================================== # @@ -56,6 +85,10 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/../../../load_config.sh" parse_args "$@" + +# Rewrites the API key into host*.conf and registers it with the unraid-api service. +[[ "$EUID" -ne 0 ]] && { error "Must be run as root"; exit 1; } + acquire_lock detect_hosts diff --git a/Plugin/unraid/System_Essentials/user_scripts_stop.sh b/Plugin/unraid/System_Essentials/user_scripts_stop.sh index 174c41a..925b413 100755 --- a/Plugin/unraid/System_Essentials/user_scripts_stop.sh +++ b/Plugin/unraid/System_Essentials/user_scripts_stop.sh @@ -32,6 +32,25 @@ # tree mid-execution. # # ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# Name the Scripts, Not Just the PIDs +# Output resolves each process to the script it is running. "Stopping 4 processes" tells +# an operator nothing they can act on; "stopping mover_tuning, preclear" tells them +# exactly what they are about to lose and whether they should wait. +# +# Targeted, Never Blanket +# Only processes spawned by the User Scripts plugin are matched. A broad pattern would +# catch unrelated shells and Varaverk's own scripts — including, during a reboot sequence, +# the very script doing the stopping. +# +# Shutdown Step and Manual Tool, Same Behaviour +# Called automatically by server_reboot.sh and usable by hand on a stuck script. It takes +# no "reboot mode" — the correct action is identical either way, and a mode flag would be +# a second code path that only ever runs unattended. +# +# ============================================================================================== # OPERATIONAL SAFEGUARDS # ============================================================================================== # @@ -51,6 +70,21 @@ # No processes running = log() only, no visible output. # # ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# No config vars. Targets are discovered from running processes rather than from a list, +# because the User Scripts plugin owns what exists and any list here would go stale. +# +# Platform-owned: +# +# The unRAID User Scripts plugin's script directory and the processes it spawns. Matching +# is scoped to those — see Targeted, Never Blanket above for why that matters during a +# reboot sequence. +# +# Called by server_reboot.sh as a shutdown step; also safe to run by hand. +# +# ============================================================================================== # RUNTIME MODES # ============================================================================================== # diff --git a/Plugin/unraid/Tools/api_cache_writer.sh b/Plugin/unraid/Tools/api_cache_writer.sh index 86e39f5..c115c5f 100755 --- a/Plugin/unraid/Tools/api_cache_writer.sh +++ b/Plugin/unraid/Tools/api_cache_writer.sh @@ -13,6 +13,58 @@ # RAM-speed reads and auto-cleared on reboot. # # ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== +# +# A one-line shim: exec php on api_cache_writer.php in the same directory. +# +# All logic lives in the PHP, because the payload builders (vv_monitor_*, vv_arrs_*) are PHP +# functions shared with the API endpoints. Reimplementing them in bash would mean two +# implementations of the same payload drifting apart. +# +# ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# A Shim, Not a Program +# This file exists only because the scheduler runs shell scripts and the work is PHP. It +# deliberately contains no logic — anything added here would be logic the API endpoints +# do not share, which is exactly the drift it exists to prevent. +# +# Same Builders as the Live API +# The cache is written by the same functions that serve a live request, so a cached +# response and a ?live=1 response cannot disagree in shape. +# +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# No Root, No Lock — Deliberate +# Writes only to /tmp/vv_cache as whatever user the scheduler runs as, and a stale cache +# is self-correcting on the next minute's run. There is no privileged operation to gate +# and no state worth locking: a torn cache file is replaced within 60 seconds, and every +# reader already falls back to a live call when the cache is missing or unparseable. +# +# Failure Is Non-Fatal by Design +# If the PHP fails, the cache simply is not refreshed. Pages fall back to live API calls — +# slower, but correct. This script must never be able to take the UI down. +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# None. Cache location and payload contents are owned by api_cache_writer.php and +# include/config.php (VV_CACHE_DIR). Nothing is configurable from this file. +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# api_cache_writer.sh +# Refresh the monitor and arrs caches once. No flags — the PHP takes no arguments and +# there is nothing to preview, since the only effect is replacing a regenerable cache. +# +# ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" php "$SCRIPT_DIR/api_cache_writer.php" diff --git a/Plugin/unraid/Tools/recreate_shares.sh b/Plugin/unraid/Tools/recreate_shares.sh index 00f11d2..5ff671a 100755 --- a/Plugin/unraid/Tools/recreate_shares.sh +++ b/Plugin/unraid/Tools/recreate_shares.sh @@ -33,6 +33,25 @@ # --delete behaviour automatically. No manual cleanup needed. # # ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# The .cfg Files Are the Source of Truth +# Directories are recreated from /boot/config/shares/*.cfg rather than from a list in +# Varaverk's conf. Those files are Unraid's own record of what a share is and which disks +# it spans — anything Varaverk maintained separately would be a second copy free to drift. +# +# Create Only, Never Delete +# Missing directories are created and existing ones left alone. This runs after a rebuild, +# when the operator's mental model of what should exist may be out of date; removing +# anything on that basis is how a recovery step becomes a data loss step. +# +# Array Must Be Started +# Refuses to run without /mnt/user mounted. Creating share directories against an +# unmounted array writes them into the underlying root filesystem, which then shadows the +# real shares once the array does mount. +# +# ============================================================================================== # OPERATIONAL SAFEGUARDS # ============================================================================================== # @@ -58,6 +77,22 @@ # platform_require_cmd confirms the notify script is present before use. # # ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# No Varaverk config vars. Everything is read from Unraid's own share definitions: +# +# /boot/config/shares/*.cfg +# One file per share. shareInclude names the disks the share spans; the directory is +# created on each of them. A share with no shareInclude spans all array disks. +# +# /mnt/user +# Must be mounted — see Array Must Be Started above. +# +# Deliberately not driven by HOST*_*_SYNC_SHARES: this recreates every share the server +# knows about, not only the ones Varaverk syncs. +# +# ============================================================================================== # RUNTIME MODES # ============================================================================================== # diff --git a/Plugin/unraid/Tools/remote_arr_cache_writer.sh b/Plugin/unraid/Tools/remote_arr_cache_writer.sh index 892d290..3598b60 100755 --- a/Plugin/unraid/Tools/remote_arr_cache_writer.sh +++ b/Plugin/unraid/Tools/remote_arr_cache_writer.sh @@ -28,6 +28,22 @@ # for the full 2-hour cycle. Keeps the cache fresh when a user requests it. # # ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== +# +# Per partner host: +# 1. Resolve its Tailscale IP — unresolvable skips that host, not the run +# 2. SSH across and call vv_arrs_local_node() on its own PHP stack +# 3. Write the JSON to /tmp/vv_cache/arrs_remote_.json locally +# +# The remote builds its own payload rather than this host querying the remote's arr APIs +# directly — the partner already has working local URLs and keys for its own arrs, so no +# cross-host credentials or path mapping are involved. +# +# Runs every 2 hours. The arrs page reads these files for an instant first paint and falls +# back to live calls when a file is missing or stale. +# +# ============================================================================================== # OPERATIONAL SAFEGUARDS # ============================================================================================== # @@ -37,6 +53,24 @@ # /tmp/vv_cache/ — auto-created if missing; cleared on reboot (intentional) # # ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# host*.conf +# +# SSH_KEY +# Used to reach each partner. Written by ssh_setup.sh. +# +# HOST* — partner hostnames, resolved to Tailscale IPs at runtime +# +# Cache output: /tmp/vv_cache/arrs_remote_.json +# tmpfs, cleared on reboot. Regenerable by definition — losing it costs one slow page +# load, never correctness, which is why nothing here retries hard on failure. +# +# No arr credentials are read locally. Each partner uses its own SONARR_*/RADARR_*/LIDARR_* +# values on its own side, so this host never holds keys for a remote's arrs. +# +# ============================================================================================== # RUNTIME MODES # ============================================================================================== # @@ -51,6 +85,10 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/../../../load_config.sh" parse_args "$@" + +# Reads $SSH_KEY from /root/.ssh to reach partner hosts. +[[ "$EUID" -ne 0 ]] && { error "Must be run as root"; exit 1; } + acquire_lock detect_hosts diff --git a/Plugin/unraid/Tools/storage_migrate.sh b/Plugin/unraid/Tools/storage_migrate.sh index 5521d55..299070c 100755 --- a/Plugin/unraid/Tools/storage_migrate.sh +++ b/Plugin/unraid/Tools/storage_migrate.sh @@ -24,6 +24,21 @@ # varaverk.cron rebuilt via PHP (job paths regenerated for new SCRIPTS_DIR) # # ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== +# +# 1. Determine current mode from varaverk.cfg SCRIPTS_DIR, and the requested target mode +# Already in the target mode → exit cleanly, nothing to do +# 2. rsync -a --delete SRC → DST, then carry .git across so history survives the move +# 3. Rewrite the pointers, in this order: +# varaverk.cfg SCRIPTS_DIR +# master.conf TARGET_DIR +# host*.conf HOST*_STORAGE_MODE_INTERNAL +# 4. Rebuild varaverk.cron via PHP so every job path points at the new SCRIPTS_DIR +# 5. Flash mode only: sync Plugin/ back to /boot so the webUI keeps serving current PHP +# 6. Remove the old location once the new one is confirmed in place +# +# ============================================================================================== # DESIGN PRINCIPLES # ============================================================================================== # @@ -47,6 +62,31 @@ # --to= required — refuses to run without an explicit target mode # # ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# This script WRITES these rather than reading them — they are the migration's output: +# +# varaverk.cfg +# SCRIPTS_DIR the authoritative install path. Everything else in the +# ecosystem derives from it, which is why it is written +# first and the cron is rebuilt from it afterwards. +# +# master.conf +# TARGET_DIR kept in step with SCRIPTS_DIR +# +# host*.conf +# HOST*_STORAGE_MODE_INTERNAL true = /boot/config/plugins/varaverk +# false = /mnt/user/appdata/Varaverk +# +# Load-bearing: STATE_DIR, DATA_DIR, PERSISTENT_CONF_CACHE and the orchestrator job paths are +# all built from SCRIPTS_DIR. Changing storage mode moves every one of them, which is why the +# cron is regenerated rather than edited. +# +# In flash mode the array must be started before Varaverk can function at all — appdata is +# not mounted before that. +# +# ============================================================================================== # RUNTIME MODES # ============================================================================================== # @@ -68,6 +108,11 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/../../../load_config.sh" parse_args "$@" + +# Relocates the entire Varaverk installation with rsync --delete and rm -rf, and rewrites +# varaverk.cfg, master.conf and host*.conf. Everything here needs root. +[[ "$EUID" -ne 0 ]] && { error "Must be run as root"; exit 1; } + acquire_lock detect_hosts diff --git a/Plugin/unraid/Watchdogs/System/webgui_watchdog.sh b/Plugin/unraid/Watchdogs/System/webgui_watchdog.sh index bcba840..74a2e34 100755 --- a/Plugin/unraid/Watchdogs/System/webgui_watchdog.sh +++ b/Plugin/unraid/Watchdogs/System/webgui_watchdog.sh @@ -36,6 +36,29 @@ # All three failed → notify, manual intervention needed → exit 1. # # ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# Lightest Fix First +# Escalation runs cheapest-to-heaviest, and stops the moment the WebGUI answers. Most +# hangs clear with a php-fpm restart; going straight to the heavy remedy would take the +# whole management interface down for a fault a service reload would have fixed. +# +# The WebGUI Is Not the Server +# An unresponsive WebGUI does not mean an unhealthy machine. Docker, the array and every +# share keep working while emhttp is wedged, so nothing here reboots or touches storage — +# the remedy stays scoped to the web stack. +# +# Verify After Every Step +# Responsiveness is re-tested between escalation steps rather than assuming a restart +# worked. Without that the script would walk the full ladder every time, restarting +# services that were already fixed one step earlier. +# +# Silent When Healthy +# Runs every cycle via system_watchdog.sh and prints nothing on a working WebGUI. A +# per-minute "all good" line would bury the one cycle that mattered. +# +# ============================================================================================== # OPERATIONAL SAFEGUARDS # ============================================================================================== # diff --git a/Plugin/unraid/adapter.sh b/Plugin/unraid/adapter.sh index 76bf4c9..66d1f86 100755 --- a/Plugin/unraid/adapter.sh +++ b/Plugin/unraid/adapter.sh @@ -49,6 +49,84 @@ # platform_push_setup_state — push wizard setup state to WebGUI PHP # # ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# One Place Knows the OS +# Every Unraid-specific path, rc.d script and config location lives here. Scripts never +# branch on OS and never hardcode /etc/rc.d, /boot/config or dynamix paths. That is what +# makes a second platform a single new file rather than a hundred edits. +# +# Never Exit, Always Return +# No function here calls exit. A platform capability being absent is information the +# caller needs, not a decision the adapter gets to make — a watchdog may want to skip a +# check where an installer wants to abort, and only they know which. +# +# Report, Do Not Remediate +# The adapter answers questions and performs the single action asked of it. It does not +# retry, escalate, notify or heal. Every one of those policies belongs to the caller, and +# burying them here would make identical calls behave differently per platform. +# +# Stdout Is the Return Channel +# Value-producing functions write to stdout and are captured with $(). Status is carried +# by the exit code. Keeping those separate is what lets callers use them in conditionals +# without parsing output. +# +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# No Root, No Lock, No detect_hosts — Deliberate +# This is a sourced library, loaded by load_config.sh whenever PLATFORM=unraid. It runs +# inside the caller's process and holds no state of its own. A root check here would fire +# for every script in the ecosystem including read-only ones, and a lock would be taken +# on every source. The executable scripts own those gates. Do not "fix" this to match them. +# +# Executable Validation Before Use +# platform_require_cmd() and the rc.d helpers verify a target exists and is executable +# before invoking it, so a missing platform binary returns a clean failure rather than a +# command-not-found in the middle of a caller's flow. +# +# Notification Is Best-Effort +# platform_send_os_notification() returns 1 if the dynamix notify script is absent rather +# than failing the caller. A missing notifier must never turn a successful operation into +# a reported failure. +# +# Graceful Degradation on Unknown Services +# _platform_rc_script() falls through to /etc/rc.d/rc. for any service it does not +# explicitly map, so a new service works without an adapter change — and still fails +# cleanly if that path does not exist. +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# None of its own. The adapter is selected by PLATFORM in varaverk.cfg, which load_config.sh +# uses to source Plugin/$PLATFORM/adapter.sh. +# +# It reads platform-owned files rather than Varaverk config: +# +# /boot/config/plugins/dynamix/dynamix.cfg disk temperature thresholds +# /boot/config/plugins/varaverk/varaverk.cfg SCRIPTS_DIR probe +# /boot/config/plugins/dockerMan/templates-user container rebuild templates +# /etc/rc.d/rc.* service control +# +# Those paths are Unraid's, not Varaverk's — which is exactly why they are confined to this +# file. STATE_DIR, DATA_DIR and friends belong to master.conf and are not the adapter's +# concern. +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# None — sourced, never executed: +# +# source "$LOAD_CONFIG_DIR/Plugin/$PLATFORM/adapter.sh" +# +# No argument parsing, no --dry-run, no --status. Callers that need a dry run implement it +# around the platform_*() call, since only they know which of their actions are destructive. +# +# ============================================================================================== # ────────────────────────────────────────────────────────────────────────────────────────────── # Internal: map a logical service name → its rc.d script path diff --git a/Plugin/unraid/user_script_plug-in.sh b/Plugin/unraid/user_script_plug-in.sh index 2e1b3e2..73f51c3 100755 --- a/Plugin/unraid/user_script_plug-in.sh +++ b/Plugin/unraid/user_script_plug-in.sh @@ -144,6 +144,82 @@ # bash /boot/config/plugins/varaverk/Orchestrators/array_started.sh +# ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# Schedule Orchestrators, Not Scripts +# The catalog below lists every script in the ecosystem, but only a handful of +# orchestrators are meant to be scheduled. The orchestrators own ordering, dependencies, +# pass/fail tracking and one notification per window — scheduling their children directly +# bypasses all of that and produces overlapping runs the locks then have to fight. +# +# Everything Listed, Most Commented +# Individual scripts are documented here so they can be run standalone for a specific +# task, not so they can be scheduled. The commented-out default is the safe state: paste +# the file, uncomment exactly one block, set the schedule. +# +# One Block Per User Script Entry +# Each entry runs one thing. Combining blocks defeats the per-entry schedule and makes a +# failure in the first silently skip the rest. +# +# Documentation That Cannot Drift Silently +# Schedules here are stated alongside what the script does, so a mismatch with +# varaverk.cron is visible when read. Cross-check against the cron before trusting a +# cadence quoted in a comment — the cron is authoritative. +# +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# Inert by Default +# Every command in this file is commented out. Pasted as-is it does nothing — the +# operator must deliberately uncomment a block. That is the safeguard: there is no state +# in which this file runs something unintended. +# +# No Root, No Lock, No detect_hosts — Deliberate +# This is a reference template, never executed as a file. Each uncommented block invokes +# a real script that enforces its own root check and takes its own lock. Adding them here +# would gate a file that is only ever read. +# +# Full Paths Throughout +# Every example uses an absolute path. User Scripts entries run with an unpredictable +# working directory, and a relative path would resolve differently depending on how the +# plugin happened to invoke it. +# +# Background Flag Documented Per Entry +# Continuous scripts are marked Background: YES and single-pass ones NO. Getting this +# wrong either blocks the array-start sequence on a script that never exits, or detaches +# one whose exit code the sequence needed. +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# None — this file configures nothing and reads nothing. It is documentation shaped like a +# shell script so it can be pasted into a User Scripts entry. +# +# What the scheduled orchestrators actually read: +# +# master.conf ARRAY_START_SCRIPTS, ARRAY_STOP_SCRIPTS, WATCHDOG_ORCHESTRATOR_SCRIPTS, +# CRITICAL_/INTERMEDIATE_/DAILY_/WEEKLY_/MONTHLY_MAINTENANCE_SCRIPTS +# +# Add or remove a script by editing those lists — not by adding another User Scripts entry. +# That is the whole point of the orchestrator model. +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# Not executed. Paste into a User Scripts entry, uncomment ONE block, set its schedule. +# +# The blocks below are grouped by area (Fallback, Rsync, Docker Essentials, Unraid +# Essentials, Media, Transcodes, Monitors, Partnership). Each carries its own recommended +# schedule and Background flag. +# +# ============================================================================================== + + # ── ARRAY STOPPING ──────────────────────────────────────────────────────────────────────────── # Schedule: At Stopping of Array # Background: YES diff --git a/Watchdogs/resource_watchdog.sh b/Watchdogs/resource_watchdog.sh index a11f5d0..0282d9f 100755 --- a/Watchdogs/resource_watchdog.sh +++ b/Watchdogs/resource_watchdog.sh @@ -102,7 +102,7 @@ # Timeout Protection # All docker commands wrapped in a 15 second timeout. Pressure response runs # during a degraded system, which is exactly when the daemon is most likely -# to be slow — a hang here would stall the whole watchdog chain every minute. +# to be slow — a hang here would stall the whole watchdog chain for that cycle. # # Downloader Availability Guards # SABnzbd and qBittorrent throttling no-ops when the service is disabled or diff --git a/common.sh b/common.sh index ae44845..ec412f3 100755 --- a/common.sh +++ b/common.sh @@ -2,9 +2,103 @@ # ============================================================================================== # ================================= COMMON LIBRARY ============================================= # ============================================================================================== +# +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── # Shared functions used by every script in the ecosystem. # Sourced automatically by load_config.sh — do not source directly. # +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== +# +# A pure function library. It defines and returns; it starts nothing on its own. +# +# Load order matters and is owned by load_config.sh: +# varaverk.cfg → master.conf → host*.conf → common.sh → Plugin/$PLATFORM/adapter.sh +# +# common.sh is sourced AFTER the conf files because detect_hosts() and the health checks +# need HOST* and the thresholds to already exist. It is sourced BEFORE the adapter so the +# adapter can rely on log()/warn()/error() being defined. +# +# Note it defines detect_hosts() but never calls it. MY_ID and REMOTE_ID stay unset until a +# script calls it itself — anything building HOST*-prefixed variable names must do so after +# that call, not before. +# +# ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# Define, Never Act +# Sourcing this file changes no system state: no containers touched, no files written, no +# locks taken, no host detected. Every script in the ecosystem sources it, including +# read-only reporting ones, so anything that acted here would act everywhere. +# +# One Implementation of Each Shared Behaviour +# Locking, host detection, notification, retry and timeout wrapping live here once. When +# a safeguard needs strengthening it is strengthened for every caller at the same moment — +# which is why individual scripts call acquire_lock() rather than rolling their own flock. +# +# Platform-Agnostic +# No OS-specific paths or commands. Anything Unraid-specific belongs in the adapter, which +# is why this file calls platform_*() rather than touching /etc/rc.d or /boot directly. +# +# Callers Own Policy +# Helpers return status; they do not decide what failure means. retry_docker() reports +# that a command failed after N attempts — whether that aborts a run, skips an item, or +# raises a notification is the caller's call, and only the caller has the context. +# +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# No Root, No Lock, No detect_hosts on Load — Deliberate +# This is a sourced library and must stay inert. A root check here would fire for every +# script including ones that legitimately do not need it; a lock would be taken on every +# source; calling detect_hosts() automatically would exit the caller on an unknown +# hostname before it had a chance to handle that itself. Do not add them. +# +# Safe Defaults on Every Threshold +# Helpers apply :- defaults (DOCKER_TIMEOUT, RESTART_VERIFY_WAIT, RETRY_COUNT and friends) +# so a conf missing a key degrades to a sane value rather than an empty string that would +# silently disable a timeout or a retry ceiling. +# +# Timeout Wrapping Provided Centrally +# docker_cmd() and the remote helpers wrap calls in timeout, so no caller has to remember +# to. A hung daemon or unreachable partner cannot stall a scheduled window. +# +# Locking Provided Centrally +# acquire_lock() implements strict, wait and continuous modes in one place, so every +# script gets identical semantics and a fix applies everywhere at once. +# +# Host Identity Is Exact-Match Only +# detect_hosts() matches hostname exactly, with a single explicit NetBIOS-truncation +# fallback that requires an unambiguous candidate. No fuzzy or similarity matching — a +# wrong host identity would alias the wrong credentials and paths into every script. +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# Defines no config of its own. It consumes what load_config.sh has already sourced from +# master.conf and host*.conf, and applies defaults where a key may be absent. +# +# Consumed broadly: DOCKER_TIMEOUT, RESTART_VERIFY_WAIT, RETRY_COUNT, SLEEP, SSH_KEY, +# SSH_TIMEOUT, STATE_DIR, DATA_DIR, LOCK_DIR, NOTIFY_UNRAID, HOST*, and the RSYNC_*/ +# PARTNERSHIP_* gates. +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# None — sourced, never executed: +# +# source "$SCRIPT_DIR/../load_config.sh" # which sources this file +# +# It parses no arguments of its own. parse_args() is defined here for callers to invoke, +# and --dry-run/--status/--log are honoured by the calling script, not by this file. +# +# ============================================================================================== # ── WHAT THIS FILE PROVIDES ─────────────────────────────────────────────────────────────────── # Icons — consistent visual language across all script output # Output helpers — info, warn, error, success, log, notify diff --git a/load_config.sh b/load_config.sh index 5dfc6b2..e8ef7c8 100755 --- a/load_config.sh +++ b/load_config.sh @@ -36,6 +36,65 @@ # This loader sources whatever conf files ARE present — sparse checkout handles the rest. # # ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# No Root, No Lock, No detect_hosts on Load — Deliberate +# Sourced by every script in the ecosystem, including read-only ones. A root check here +# would gate all of them, a lock would be taken on every source, and auto-calling +# detect_hosts() would exit the caller on an unknown hostname before it could handle that +# itself. The executable scripts own those gates. Do not add them here. +# +# Fatal on Missing common.sh +# Aborts loudly if common.sh is absent. Every downstream script assumes log(), error(), +# acquire_lock() and detect_hosts() exist; continuing without them would produce +# command-not-found errors scattered through unrelated scripts instead of one clear cause. +# +# Adapter Load Is Optional +# A missing Plugin/$PLATFORM/adapter.sh is tolerated so the ecosystem can run before the +# plugin directory exists — bootstrap and early install paths depend on that. +# +# Load Order Is Enforced, Not Incidental +# confs before common.sh before adapter. common.sh needs HOST* to already be set, and the +# adapter needs common.sh's output helpers. Reordering breaks both silently. +# +# Partner Confs Are Cache-Only +# Partner host*.conf files load from the tmpfs RAM cache, never from disk. Sparse checkout +# means this host has no partner conf in the repo, and reading a stale on-disk copy would +# resurrect credentials the partner has since rotated. +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# This file is the thing that loads configuration, so it consumes almost none itself. The one +# input it must find on its own: +# +# /boot/config/plugins/varaverk/varaverk.cfg +# SCRIPTS_DIR the authoritative install path — everything else derives from it +# PLATFORM selects which Plugin//adapter.sh gets sourced +# +# It then defines the paths the rest of the ecosystem builds on: +# +# CONF_RAM_CACHE_DIR /tmp/.cache/vv/d — tmpfs partner conf cache, cleared each reboot +# ARR_CACHE_DIR /tmp/arr_cache — tmpfs, restored from DATA_DIR on demand +# +# Everything else (STATE_DIR, DATA_DIR, thresholds, credentials) comes out of master.conf and +# host*.conf, which this file sources rather than defines. +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# None — sourced, never executed. Every script begins with: +# +# SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# source "$SCRIPT_DIR/../load_config.sh" +# +# It takes no arguments and honours no flags. parse_args() arrives via common.sh and is +# called by the script itself afterwards. +# +# ============================================================================================== # DESIGN PRINCIPLES # ============================================================================================== #