diff --git a/.gitignore b/.gitignore index 61ebe9c..23f6d63 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ Configurations/*.bak # ── Runtime state, data, logs ───────────────────────────────────────────────── data/ State_Files/ +.cache/ *.log *.lock diff --git a/Configurations/master.conf b/Configurations/master.conf index 4aca331..52de01a 100644 --- a/Configurations/master.conf +++ b/Configurations/master.conf @@ -123,6 +123,7 @@ # Requirement: ALL state files MUST use $STATE_DIR. No /tmp, no /boot/config root. DATA_DIR="${SCRIPTS_DIR}/data" STATE_DIR="${SCRIPTS_DIR}/State_Files" + PERSISTENT_CONF_CACHE="${SCRIPTS_DIR}/.cache/vv/d" # ── Version Parity ── # Controls behaviour when local and remote unRAID versions differ. @@ -295,7 +296,7 @@ # watchdog_orchestrator.sh — NOT launched here. ARRAY_START_SCRIPTS=( "Plugin/unraid/System_Essentials/unraid_api_key_renew.sh" # re-register Varaverk API key — registry is ephemeral - "System_Essentials/conf_sync.sh" # pull partner confs + push own conf into /tmp/.vv/ RAM cache + "System_Essentials/conf_sync.sh" # pull partner confs + push own conf into /tmp/.cache/vv/d/ RAM cache "System_Essentials/conf_cache_restore.sh" # load partner confs from persistent backup if conf_sync couldn't reach partner "Transcodes/ramdisk_setup.sh" # creates ramdisk + symlink before Emby starts "System_Essentials/docker_syslog_filter.sh" # suppress veth noise before logs fill diff --git a/Deployment/conf_populate.sh b/Deployment/conf_populate.sh index 0e41b11..d27d13e 100755 --- a/Deployment/conf_populate.sh +++ b/Deployment/conf_populate.sh @@ -10,7 +10,7 @@ # EMPTY fields, never overwrites existing values unless --overwrite is passed. # # After populating, pushes the updated conf to all partners via conf_sync.sh -# so they have the fresh keys in their /tmp/.vv/ cache immediately. +# so they have the fresh keys in their /tmp/.cache/vv/d/ cache immediately. # # ============================================================================================== # AUTO-DETECTED FIELDS diff --git a/Deployment/master.conf.template b/Deployment/master.conf.template index 9c62272..0bf29b0 100644 --- a/Deployment/master.conf.template +++ b/Deployment/master.conf.template @@ -120,6 +120,7 @@ # Requirement: ALL state files MUST use $STATE_DIR. No /tmp, no /boot/config root. DATA_DIR="/boot/config/plugins/varaverk/data" STATE_DIR="/boot/config/plugins/varaverk/State_Files" + PERSISTENT_CONF_CACHE="/boot/config/plugins/varaverk/.cache/vv/d" # ── Version Parity ── # Controls behaviour when local and remote unRAID versions differ. diff --git a/Docker_Essentials/README-Docker_Essentials.md b/Docker_Essentials/README-Docker_Essentials.md index d77d6c0..ffd93d1 100644 --- a/Docker_Essentials/README-Docker_Essentials.md +++ b/Docker_Essentials/README-Docker_Essentials.md @@ -139,7 +139,7 @@ happens after an update, and you want to know when it does. A container keeps appearing in a broken state. You SSH in and see it's stopped. You don't know if the watchdog tried to restart it and failed, gave up and skip-listed it, is mid-attempt right now, or hasn't noticed yet. You have to manually check the skip -list file on `/boot/config/`, check the restart history file, check the state file — +list file in `$STATE_DIR`, check the restart history file, check the state file — none of which have obvious formats. The fix: `watchdog_skip_list_manager.sh` (in `Tools/`). One command to see exactly what's diff --git a/Fallback/Manual-Fallback.md b/Fallback/Manual-Fallback.md index 74c2a08..f9b929e 100644 --- a/Fallback/Manual-Fallback.md +++ b/Fallback/Manual-Fallback.md @@ -37,11 +37,11 @@ up required before handback sequence starts. **(default: 3)** --- ``` -FALLBACK_STATE_FILE=/boot/config/fallback_state.db +FALLBACK_STATE_FILE="$STATE_DIR/fallback_state.db" ``` -Path to the persistent state file. Lives on `/boot/` intentionally — survives reboots. -If the server was in FALLBACK state when it rebooted, it resumes FALLBACK on restart -rather than assuming everything is normal. +Path to the persistent state file. In `$STATE_DIR` — survives reboots whether storage +mode is internal (boot device) or appdata (array). If the server was in FALLBACK state +when it rebooted, it resumes FALLBACK on restart rather than assuming everything is normal. --- @@ -172,7 +172,7 @@ daily_sync_maintenance.sh uses, in the opposite direction. No separate TIER4 lis FALLBACK_ENABLED=true FALLBACK_CHECK_INTERVAL=30 FALLBACK_HANDBACK_STRIKES=3 -FALLBACK_STATE_FILE=/boot/config/fallback_state.db +FALLBACK_STATE_FILE="$STATE_DIR/fallback_state.db" FALLBACK_RSYNC_ENABLED=true EXTERNAL_IP=8.8.8.8 FALLBACK_TEST_BLOCK_WAIT=60 @@ -293,7 +293,10 @@ FALLBACK_HOST2_WRITEBACK_TIER1=( ## ━━━ STATE FILE REFERENCE ━━━ -Location: `/boot/config/fallback_state.db` (survives reboots) +Location: `$STATE_DIR/fallback_state.db` (survives reboots — boot device or appdata) + +> In a shell where load_config.sh is not sourced, use the full path: +> `/boot/config/plugins/varaverk/State_Files/fallback_state.db` (internal storage mode) ``` state=NORMAL # NORMAL | FALLBACK | NO_INTERNET | DARK @@ -304,8 +307,8 @@ tier3_started=false # whether Tier 3 containers started tier4_started=false # whether Tier 4 containers started ``` -View state: `cat /boot/config/fallback_state.db` -Check state: `fallback.sh --status` +View state: `fallback.sh --status` (preferred — parsed output) +Raw file: `cat "$STATE_DIR/fallback_state.db"` (requires STATE_DIR set, or use full path) The file is managed exclusively by fallback.sh. Do not edit it while fallback.sh is running — the next cycle will overwrite your changes. Use the Manual State Reset @@ -384,7 +387,7 @@ servers must be running it continuously for mutual coverage. pgrep -f "fallback.sh" # Check the state file -cat /boot/config/fallback_state.db +cat "$STATE_DIR/fallback_state.db" ``` Start via User Scripts plugin on both servers. @@ -405,7 +408,7 @@ in smart mode — a non-NORMAL state at digest time needs attention. ```bash fallback.sh --status # full state snapshot -cat /boot/config/fallback_state.db # raw state file +cat "$STATE_DIR/fallback_state.db" # raw state file ``` --- @@ -426,7 +429,7 @@ fallback_test.sh # Step 3 — check state after test completes fallback.sh --status -cat /boot/config/fallback_state.db +cat "$STATE_DIR/fallback_state.db" ``` If the test doesn't complete cleanly, the state file may be left in FALLBACK. The @@ -459,11 +462,14 @@ ping -c 5 [remote-tailscale-ip] **Stop fallback.sh first (via User Scripts Abort), then reset:** ```bash +# Set STATE_DIR (or source load_config.sh to get it from the environment) +source /boot/config/plugins/varaverk/load_config.sh + # View current state -cat /boot/config/fallback_state.db +cat "$STATE_DIR/fallback_state.db" # Write a clean NORMAL state -cat > /boot/config/fallback_state.db << 'EOF' +cat > "$STATE_DIR/fallback_state.db" << 'EOF' state=NORMAL fallback_start=0 handback_strikes=0 @@ -473,7 +479,7 @@ tier4_started=false EOF # Verify the write -cat /boot/config/fallback_state.db +cat "$STATE_DIR/fallback_state.db" ``` Restart fallback.sh via User Scripts plugin. It will resume from NORMAL on its next cycle. @@ -510,7 +516,7 @@ Can this server reach the remote Tailscale IP? What does fallback.sh report? → fallback.sh --status - → cat /boot/config/fallback_state.db + → cat "$STATE_DIR/fallback_state.db" ``` ### Handback Not Completing @@ -559,7 +565,7 @@ If it has happened: 3. Understand the state before resetting fallback.sh --status - cat /boot/config/fallback_state.db + cat "$STATE_DIR/fallback_state.db" 4. Perform Manual State Reset above on the server in a bad state diff --git a/Fallback/fallback.sh b/Fallback/fallback.sh index 9b90231..244f4d6 100755 --- a/Fallback/fallback.sh +++ b/Fallback/fallback.sh @@ -130,11 +130,13 @@ # STATE FILES # ============================================================================================== # -# FALLBACK_STATE_FILE — /boot/config/fallback_state.db (survives reboots) +# FALLBACK_STATE_FILE — $STATE_DIR/fallback_state.db (survives reboots) # Keys: state, fallback_start, handback_strikes, tier2_started, # tier3_started, tier4_started, partnership_suspended, partner_lost_at. -# Lives on /boot/ intentionally — if the server was in FALLBACK when it -# rebooted, it resumes FALLBACK on restart. +# Survives reboots — $STATE_DIR is on the boot device (internal) or appdata +# (flash). Either way the array is up before this script runs, so the file +# is always accessible. If the server was in FALLBACK when it rebooted, +# it resumes FALLBACK on restart. # # ============================================================================================== # CONFIGURATION @@ -185,7 +187,7 @@ # Consecutive remote-up checks required before handback begins. (default: 3) # # FALLBACK_STATE_FILE -# State file path — /boot/config/fallback_state.db — survives reboots. +# State file path — $STATE_DIR/fallback_state.db — survives reboots. # # FALLBACK_RSYNC_ENABLED # Gate for writeback rsync jobs during handback. (default: true) @@ -302,7 +304,7 @@ log "$ICON_GEAR Timeouts: docker=${DOCKER_TIMEOUT}s ssh=${SSH_TIMEOUT}s verify-w # ============================================================================================== # ── STATE FILE HELPERS ──────────────────────────────────────────────────────────────────────── # ============================================================================================== -# State file on /boot/config — survives reboots. +# State file at $FALLBACK_STATE_FILE ($STATE_DIR/fallback_state.db) — survives reboots. # Format: key=value one per line. # Keys: state, fallback_start, handback_strikes, tier2_started, tier3_started, tier4_started diff --git a/Manual.md b/Manual.md index b448a5e..ed5d231 100644 --- a/Manual.md +++ b/Manual.md @@ -409,7 +409,7 @@ pgrep -a -f fallback.sh Check current state: ```bash -cat /boot/config/fallback_state.db +cat "$STATE_DIR/fallback_state.db" # state=NORMAL — both servers up ``` @@ -510,7 +510,7 @@ bash Orchestrators/daily_sync_maintenance.sh --dry-run --log ### ── Check fallback state ──────────────────────────────────────────────────── ```bash -cat /boot/config/fallback_state.db +cat "$STATE_DIR/fallback_state.db" # Expected: # state=NORMAL # fallback_start=0 @@ -600,7 +600,7 @@ The remote server is unreachable on the first check. Common causes: ```bash # Confirm which state fallback is in -cat /boot/config/fallback_state.db +cat "$STATE_DIR/fallback_state.db" # If stuck in FALLBACK after remote comes back: reset state bash Tools/fallback_state_reset.sh ``` diff --git a/Media/Manual-Media.md b/Media/Manual-Media.md index d2d2095..4602040 100644 --- a/Media/Manual-Media.md +++ b/Media/Manual-Media.md @@ -132,8 +132,8 @@ LIDARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan LIDARR_VERSION_MAJOR=3 # expected Lidarr major version (API safety check) LIDARR_EXTENSIONS=("flac" "mp3" "m4a" "wav" "aac" "ogg" "opus" "wma") LIDARR_PROTECTED_PATTERNS=("*.jpg" "*.jpeg" "*.png" "*.nfo" "*.lrc") -LIDARR_TRACKED_COUNT_FILE=/boot/config/lidarr_tracked_count # persistent baseline -ARR_CLEANUP_STATS=/boot/config/arr_cleanup_stats.db # read by coffee report +LIDARR_TRACKED_COUNT_FILE="$DATA_DIR/lidarr_tracked.count" # persistent baseline +ARR_CLEANUP_STATS="$DATA_DIR/arr_cleanup_stats.db" # read by coffee report ``` --- @@ -171,7 +171,7 @@ RADARR_PROTECTED_PATTERNS=("*.jpg" "*.jpeg" "*.png" "*.nfo" "*.srt" "*.sub" "*.a ```bash ARR_SYNC_ENABLED=true -ARR_SYNC_BLOCKLIST=/boot/config/arr_sync_blocklist.tsv # tombstone file +ARR_SYNC_BLOCKLIST="$DATA_DIR/arr_sync_blocklist.tsv" # tombstone file ARR_SYNC_CONNECT_TIMEOUT=10 # SSH connect timeout in seconds ARR_SYNC_API_TIMEOUT=60 # curl API call timeout in seconds DOCKER_APPDATA_BASE=/mnt/user/appdata @@ -189,7 +189,7 @@ ARR_IMPORT_RECOVERY_AGE=6 # hours — items newer than this are skipped SONARR_VERSION_MAJOR=4 RADARR_VERSION_MAJOR=6 LIDARR_VERSION_MAJOR=3 -ARR_RECOVERY_STATS=/boot/config/arr_recovery_stats.db # read by coffee report +ARR_RECOVERY_STATS="$DATA_DIR/arr_recovery_stats.db" # read by coffee report ``` --- @@ -274,6 +274,47 @@ Requires `HOST*_TMDB_API_KEY` in `host*.conf`. --- +### Play State Sync + +```bash +PLAY_STATE_SYNC_ENABLED=true # toggle entire sync +PLAY_STATE_SYNC_LOOKBACK_DAYS=30 # history window for played items +``` + +Emby and Jellyfin servers configured per-host: + +```bash +# host*.conf +HOST1_EMBY_URL="http://192.168.50.2:8096" +HOST1_EMBY_API_KEY="..." +HOST1_JELLYFIN_URL="" # empty = skip Jellyfin on this host +HOST1_JELLYFIN_API_KEY="" +``` + +--- + +### Upgrade Webhook + +```bash +# master.conf +WEBHOOK_PORT=9456 # 0 = disable listener +WEBHOOK_SECRET="" # auto-generated on first start if empty +WEBHOOK_UPGRADE_RSYNC_ENABLED=true # push upgraded file to remote nodes +``` + +```bash +# host*.conf +HOST1_WEBHOOK_LAN_IP="192.168.50.2" # IP arrs call for webhook delivery +``` + +When `WEBHOOK_PORT=0`, `start_webhook_listener.sh` exits cleanly and no listener starts. +When `WEBHOOK_SECRET` is empty, a 32-byte hex secret is generated on first start and +written back to `master.conf`. Run `Tools/webhook_setup.sh` to register the URL in arrs. + +Webhook log: `/var/log/varaverk/upgrade_webhook.log` + +--- + ### Orchestrator Job Order ```bash diff --git a/Media/README-Media.md b/Media/README-Media.md index 3740da6..3f701b9 100644 --- a/Media/README-Media.md +++ b/Media/README-Media.md @@ -4,8 +4,9 @@ Library health, consistency, sync, and behavior-driven discovery for a multi-ser stack. Correct permissions so arrs can manage files. Junk removal so orphan detection isn't confused by scene debris. Library sync so every node tracks the same content. Orphan cleanup against live arr APIs so deleted content actually leaves disk. Emby -notified automatically after every deletion. Weekly discovery adds new music, movies, -and TV shows based on what you actually play — no manual browsing required. +notified automatically after every deletion. Watch state synced across Emby/Jellyfin +every 30 minutes. Quality upgrades propagate to all nodes immediately via webhook. +Weekly discovery adds new music, movies, and TV shows based on what you actually play. > **These scripts permanently delete files.** The arr cleanup scripts are protected by > multiple safety layers that must all pass before anything is touched — but dry runs and @@ -91,6 +92,19 @@ free overnight recovery. `lidarr_missing_art.sh` — fetch missing album and artist artwork from fanart.tv and fallback sources. Never overwrites existing files. +**Play State Sync** +`play_state_sync.sh` — syncs watched/played state and resume positions across all +configured Emby and Jellyfin servers. Newest timestamp wins. Runs every 30 minutes +via critical_sync_maintenance.sh. + +**Upgrade Propagation** +`start_webhook_listener.sh` — Node.js HTTP server that receives Sonarr/Radarr/Lidarr +OnUpgrade webhooks. Continuous; started at array start. Writes to +`/var/log/varaverk/upgrade_webhook.log`. +`upgrade_webhook_handler.sh` — triggered by the webhook listener. Pushes the upgraded +item folder to every remote node immediately, then triggers an arr library rescan on +each remote so the upgraded file is accepted without triggering a redundant quality search. + **Discovery** `playback_aware_lidarr_discovery.sh` — behavior-driven music discovery. Scores your Emby play history, runs Last.fm getSimilar on top artists, adds the best matches to @@ -138,6 +152,24 @@ playback_aware_radarr_discovery.sh — score watch history → TMDB recommenda playback_aware_sonarr_discovery.sh — score episode history → TMDB TV recommendations → add to Sonarr ``` +**Every 30 min via `critical_sync_maintenance.sh` (CRITICAL_MAINTENANCE_SCRIPTS):** + +``` +play_state_sync.sh — sync watched/resume state across Emby + Jellyfin +``` + +**Continuous (started by `array_started.sh`):** + +``` +start_webhook_listener.sh — Node.js webhook server; dispatch upgrade_webhook_handler.sh +``` + +**On every arr upgrade (triggered by webhook):** + +``` +upgrade_webhook_handler.sh — push upgraded folder to all remote nodes + trigger arr rescan +``` + **Ad-hoc or separate schedule:** ``` @@ -189,6 +221,9 @@ in `HOST*_MEDIA_PERMISSION_SHARES` and `HOST*_MEDIA_CLEAN_FOLDERS` in host*.conf | `lidarr_missing_art.sh` | Fetch missing album and artist artwork | Ad-hoc or separate schedule | | `radarr_tmdb_removed.sh` | Remove movies dropped from TMDb | Ad-hoc or weekly | | `sonarr_tvdb_removed.sh` | Remove series dropped from TVDB | Ad-hoc or weekly | +| `play_state_sync.sh` | Sync watched/played state + resume positions across Emby + Jellyfin | Every 30 min via `critical_sync_maintenance.sh` | +| `start_webhook_listener.sh` | Node.js webhook server — receive arr OnUpgrade and dispatch handler | Continuous (started by `array_started.sh`) | +| `upgrade_webhook_handler.sh` | Push upgraded item folder to remote nodes + trigger arr rescan | On each arr upgrade (via webhook) | | `playback_aware_lidarr_discovery.sh` | Behavior-driven music discovery — Emby plays → Last.fm similar → Lidarr | Weekly via `weekly_sync_maintenance.sh` | | `playback_aware_radarr_discovery.sh` | Behavior-driven movie discovery — Emby watches → TMDB recommendations → Radarr | Weekly via `weekly_sync_maintenance.sh` | | `playback_aware_sonarr_discovery.sh` | Behavior-driven TV discovery — Emby episodes → TMDB TV recommendations → Sonarr | Weekly via `weekly_sync_maintenance.sh` | @@ -228,6 +263,17 @@ Weekly discovery (WEEKLY_MAINTENANCE_SCRIPTS): │ └── each discovery script fires arr search immediately after successful add +Every 30 min (critical_sync_maintenance.sh CRITICAL_MAINTENANCE_SCRIPTS): + play_state_sync.sh ─── newest timestamp wins → watched/resume state synced + across all configured Emby + Jellyfin servers + +Continuous (started by array_started.sh): + start_webhook_listener.sh ── Node.js HTTP server listens on WEBHOOK_PORT + │ arr OnUpgrade fires webhook → POST to http://HOST_LAN_IP:WEBHOOK_PORT/webhook?key=SECRET + └── upgrade_webhook_handler.sh + ├── rsync upgraded folder → all remote nodes immediately + └── trigger arr library rescan on each remote (accept new file, no quality search) + Ad-hoc enrichment: lidarr_missing_art.sh ─────── discovers missing artwork → fetches from fanart.tv radarr_tmdb_removed.sh ────── status="deleted" → remove from Radarr + add exclusion diff --git a/Media/start_webhook_listener.sh b/Media/start_webhook_listener.sh index 7a421d3..b1294e8 100755 --- a/Media/start_webhook_listener.sh +++ b/Media/start_webhook_listener.sh @@ -5,7 +5,7 @@ # # PURPOSE # ───────────────────────────────────────────────────────────────────────────── -# Starts a standalone PHP HTTP server that receives Sonarr/Radarr/Lidarr +# Starts a standalone Node.js HTTP server that receives Sonarr/Radarr/Lidarr # OnUpgrade webhooks and dispatches upgrade_webhook_handler.sh. # # Runs outside Unraid nginx — no session auth required. The shared secret in @@ -13,9 +13,12 @@ # # http://:/webhook?key= # -# Runs as a continuous script started by array_started.sh. Execs php -S which +# Runs as a continuous script started by array_started.sh. Execs node which # replaces this process — the PID stays the same for array_started.sh's check. # +# Uses Node.js instead of php -S: php -S on Unraid PHP 8.4 silently drops +# POST request bodies, making webhook payloads arrive empty. +# # If WEBHOOK_SECRET is empty in master.conf: generates and saves one, then starts. # If WEBHOOK_PORT is 0: exits cleanly (disables the listener). # diff --git a/Monitors/Manual-Monitors.md b/Monitors/Manual-Monitors.md index dfbced0..6223b70 100644 --- a/Monitors/Manual-Monitors.md +++ b/Monitors/Manual-Monitors.md @@ -151,14 +151,14 @@ BACKUP_VERIFY_MIN_SIZE="1M" # skip files smaller than this ```bash # master.conf -BANDWIDTH_LOG="/boot/config/bandwidth_history.db" # survives reboots +BANDWIDTH_LOG="$DATA_DIR/bandwidth_history.db" # survives reboots BANDWIDTH_LOG_RETENTION=90 # days — file stays bounded, never grows unbounded BANDWIDTH_WARN_GB=50 # flag transfers or daily totals exceeding this ``` -**Why `/boot/config/`**: The log needs to survive reboots to build a useful history. -`/boot/config/` is on the USB flash drive, which survives reboots and is backed up -by unRAID's flash backup. The log is bounded by `BANDWIDTH_LOG_RETENTION` so it never +**Why `$DATA_DIR`**: The log needs to survive reboots to build a useful history. +`$DATA_DIR` (`$SCRIPTS_DIR/data/`) is on the boot device (internal) or appdata (flash), +both of which survive reboots. The log is bounded by `BANDWIDTH_LOG_RETENTION` so it never grows unbounded. **`BANDWIDTH_WARN_GB`**: Set to a value that represents "unexpectedly large" for your @@ -401,7 +401,7 @@ BACKUP_VERIFY_SAMPLE=10 BACKUP_VERIFY_MIN_SIZE="1M" # bandwidth_monitor.sh -BANDWIDTH_LOG="/boot/config/bandwidth_history.db" +BANDWIDTH_LOG="$DATA_DIR/bandwidth_history.db" BANDWIDTH_LOG_RETENTION=90 BANDWIDTH_WARN_GB=50 diff --git a/Orchestrators/README-Orchestrators.md b/Orchestrators/README-Orchestrators.md index f3f6e37..489b171 100644 --- a/Orchestrators/README-Orchestrators.md +++ b/Orchestrators/README-Orchestrators.md @@ -849,7 +849,7 @@ long tests — that should only run on stable systems that have been up for at l Two gates must both pass before any job runs: 1. Server uptime ≥ `MONTHLY_UPTIME_THRESHOLD_DAYS` -2. Last run ≥ `MONTHLY_RUN_INTERVAL_DAYS` ago (state file on `/boot/config/` — survives reboots) +2. Last run ≥ `MONTHLY_RUN_INTERVAL_DAYS` ago (state file in `$STATE_DIR` — survives reboots) If either gate fails, the script exits 0 with no output. This is expected — it runs daily and most days are no-ops. @@ -866,7 +866,7 @@ MONTHLY_MAINTENANCE_SCRIPTS=( ) MONTHLY_UPTIME_THRESHOLD_DAYS=30 MONTHLY_RUN_INTERVAL_DAYS=30 -MONTHLY_LAST_RUN_FILE="/boot/config/monthly_maintenance_last_run.db" +MONTHLY_LAST_RUN_FILE="$STATE_DIR/monthly_maintenance_last_run.db" ``` Scripts are commented out by default — uncomment what applies to your hardware. diff --git a/Orchestrators/intermediate_sync_maintenance.sh b/Orchestrators/intermediate_sync_maintenance.sh index 5b76a86..8936923 100755 --- a/Orchestrators/intermediate_sync_maintenance.sh +++ b/Orchestrators/intermediate_sync_maintenance.sh @@ -6,7 +6,7 @@ # Schedule: 0 */4 * * * (every 4 hours) # # ── EXECUTION ORDER ─────────────────────────────────────────────────────────────────────────── -# 1. conf_sync.sh --pull-only — refresh partner conf cache in RAM (/tmp/.vv/) +# 1. conf_sync.sh --pull-only — refresh partner conf cache in RAM (/tmp/.cache/vv/d/) # 2. arr_sync.sh — sync Lidarr/Sonarr/Radarr libraries across all nodes # 3. Rsync window (optional) — INTERMEDIATE_SYNC_SHARES, if any configured # 4. INTERMEDIATE_MAINTENANCE_SCRIPTS — artwork fetch and any future 4-hour jobs diff --git a/Partnership/Manual-Partnership.md b/Partnership/Manual-Partnership.md index 7a6aa51..7612e17 100644 --- a/Partnership/Manual-Partnership.md +++ b/Partnership/Manual-Partnership.md @@ -574,7 +574,7 @@ Partnership/partnership_manager.sh --status # shows both sides via SSH ```bash # Check the offline counter: -cat /boot/config/partnership_offline_days.db +cat "$STATE_DIR/partnership_offline_days.db" # Extended Tailscale outage may have incremented the counter. # Check Tailscale peer visibility: @@ -591,9 +591,9 @@ Partnership/partnership_manager.sh --onboard ## ━━━ STATE FILES ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ ```bash -/boot/config/partnership_HOST1.db # HOST1 writes only -/boot/config/partnership_HOST2.db # HOST2 writes only -/boot/config/partnership_blocklist.db # hostname|timestamp|reason +$STATE_DIR/partnership_HOST1.db # HOST1 writes only +$STATE_DIR/partnership_HOST2.db # HOST2 writes only +$STATE_DIR/partnership_blocklist.db # hostname|timestamp|reason # Example state file: state=ACTIVE diff --git a/Partnership/partnership_manager.sh b/Partnership/partnership_manager.sh index e7bc8e1..f0022bf 100755 --- a/Partnership/partnership_manager.sh +++ b/Partnership/partnership_manager.sh @@ -93,11 +93,11 @@ # STATE FILES # ============================================================================================== # -# /boot/config/partnership_HOST1.db — HOST1 writes, HOST2 reads via SSH -# /boot/config/partnership_HOST2.db — HOST2 writes, HOST1 reads via SSH -# /boot/config/partnership_blocklist.db — hostname|timestamp|reason, persists until cleared +# $STATE_DIR/partnership_.db — each host writes its own, partner reads via SSH +# $STATE_DIR/partnership_blocklist.db — hostname|timestamp|reason, persists until cleared +# $STATE_DIR/partnership_offline_days.db — cumulative offline day counter # -# On /boot/config — survives reboots, available before array starts, minimal flash wear. +# All in STATE_DIR — survives reboots (on /boot in internal mode, appdata in flash mode). # # ============================================================================================== # CONFIGURATION diff --git a/Plugin/unraid/adapter.sh b/Plugin/unraid/adapter.sh index 64b25b8..8e0fc15 100755 --- a/Plugin/unraid/adapter.sh +++ b/Plugin/unraid/adapter.sh @@ -274,7 +274,7 @@ platform_get_templates_dir() { # Writes the path to the persistent Varaverk setup/wizard state database. # ────────────────────────────────────────────────────────────────────────────────────────────── platform_setup_db_path() { - echo "/boot/config/varaverk_setup.db" + echo "${STATE_DIR}/varaverk_setup.db" } # ────────────────────────────────────────────────────────────────────────────────────────────── diff --git a/Plugin/unraid/api/snapshot.php b/Plugin/unraid/api/snapshot.php index 7ca81c1..072b7ea 100644 --- a/Plugin/unraid/api/snapshot.php +++ b/Plugin/unraid/api/snapshot.php @@ -13,7 +13,7 @@ $ramUsedMb = $ramTotalMb - $res['ram_free_mb']; $ramPct = $ramTotalMb > 0 ? (int)round($ramUsedMb / $ramTotalMb * 100) : 0; // Fallback state (fast file read, no exec) -$fbRaw = @file_get_contents('/tmp/fallback_state.db') ?: ''; +$fbRaw = @file_get_contents(STATE_DIR . '/fallback_state.db') ?: ''; $fbData = vv_parse_kv_db($fbRaw); $fallbackState = $fbData['state'] ?? 'UNKNOWN'; diff --git a/Plugin/unraid/include/config.php b/Plugin/unraid/include/config.php index 493db3b..7d76126 100644 --- a/Plugin/unraid/include/config.php +++ b/Plugin/unraid/include/config.php @@ -34,8 +34,7 @@ function vv_setup_state_write(array $data): void { } // Push the setup state file to all remote hosts via scp. -// Unlike master.conf push, this does NOT require the plugin to be installed on the remote — -// it only needs SSH to be reachable, and pushes to /boot/config/ (always available). +// Reads the remote's varaverk.cfg to find their actual SCRIPTS_DIR (handles appdata mode). function vv_push_setup_state(): void { if (!file_exists(VV_SETUP_STATE_FILE)) return; $myHostId = vv_detect_host(); @@ -56,9 +55,20 @@ function vv_push_setup_state(): void { if (!$ip) continue; $sshBase = 'ssh -i ' . escapeshellarg($sshKey) . ' -o ConnectTimeout=10 -o StrictHostKeyChecking=no root@' . $ip; - // Ensure the target dir exists (it always should on Unraid, but be safe) - shell_exec($sshBase . ' "mkdir -p /boot/config" 2>/dev/null'); - $dest = escapeshellarg('root@' . $ip . ':/boot/config/varaverk_setup.db'); + + // Get remote SCRIPTS_DIR from varaverk.cfg — handles appdata mode on remote. + // Falls back to the default install path if varaverk.cfg is absent (pre-install). + $cfgRaw = trim(shell_exec($sshBase . ' "cat /boot/config/plugins/varaverk/varaverk.cfg 2>/dev/null"') ?: ''); + $remoteSD = '/boot/config/plugins/varaverk'; + foreach (explode("\n", $cfgRaw) as $line) { + if (str_starts_with(trim($line), 'SCRIPTS_DIR=')) { + $remoteSD = trim(substr(trim($line), strlen('SCRIPTS_DIR=')), '"\''); + break; + } + } + $remoteStatePath = $remoteSD . '/State_Files/varaverk_setup.db'; + shell_exec($sshBase . ' "mkdir -p ' . escapeshellarg(dirname($remoteStatePath)) . '" 2>/dev/null'); + $dest = escapeshellarg('root@' . $ip . ':' . $remoteStatePath); exec('scp -i ' . escapeshellarg($sshKey) . ' -o ConnectTimeout=10 -o StrictHostKeyChecking=no' . ' ' . escapeshellarg(VV_SETUP_STATE_FILE) . ' ' . $dest . ' 2>&1'); @@ -391,6 +401,15 @@ function vv_auto_create_api_key(string $hostId, string $confFile): array { return ['ok' => true, 'key_preview' => $key ? substr($key, 0, 8) . '...' . substr($key, -4) : 'registered']; } +// Build a bash command that reads a state file from the REMOTE host's State_Files/. +// Reads the remote's varaverk.cfg to resolve their SCRIPTS_DIR (may differ from ours +// when the remote is in appdata mode). Falls back to the internal plugin path. +function vv_remote_state_cmd(string $filename): string { + $fn = basename($filename); + return 'sd=$(grep -m1 SCRIPTS_DIR= /boot/config/plugins/varaverk/varaverk.cfg 2>/dev/null' + . ' | cut -d\'"\' -f2); cat "${sd:-/boot/config/plugins/varaverk}/State_Files/' . $fn . '" 2>/dev/null'; +} + // Local LAN IP via routing table — static-cached per request. // Previously duplicated in include/docker_folders.php and inline in include/docker.php. function vv_local_ip(): string { diff --git a/Plugin/unraid/include/fallback.php b/Plugin/unraid/include/fallback.php index bdc0957..f1e99b3 100644 --- a/Plugin/unraid/include/fallback.php +++ b/Plugin/unraid/include/fallback.php @@ -18,12 +18,14 @@ function vv_fb_scalar(string $raw, string $varname): string { function vv_fb_parse_state(string $text): array { $out = [ - 'state' => 'UNKNOWN', - 'fallback_start' => 0, - 'handback_strikes' => 0, - 'tier2_started' => false, - 'tier3_started' => false, - 'tier4_started' => false, + 'state' => 'UNKNOWN', + 'fallback_start' => 0, + 'handback_strikes' => 0, + 'tier2_started' => false, + 'tier3_started' => false, + 'tier4_started' => false, + 'partnership_suspended' => false, + 'partner_lost_at' => 0, ]; foreach (explode("\n", $text) as $line) { $line = trim($line); @@ -31,24 +33,26 @@ function vv_fb_parse_state(string $text): array { [$k, $v] = array_pad(explode('=', $line, 2), 2, ''); $k = trim($k); $v = trim($v, '"\''); switch ($k) { - case 'state': $out['state'] = $v; break; - case 'fallback_start': $out['fallback_start'] = (int)$v; break; - case 'handback_strikes': $out['handback_strikes'] = (int)$v; break; - case 'tier2_started': $out['tier2_started'] = $v === 'true'; break; - case 'tier3_started': $out['tier3_started'] = $v === 'true'; break; - case 'tier4_started': $out['tier4_started'] = $v === 'true'; break; + case 'state': $out['state'] = $v; break; + case 'fallback_start': $out['fallback_start'] = (int)$v; break; + case 'handback_strikes': $out['handback_strikes'] = (int)$v; break; + case 'tier2_started': $out['tier2_started'] = $v === 'true'; break; + case 'tier3_started': $out['tier3_started'] = $v === 'true'; break; + case 'tier4_started': $out['tier4_started'] = $v === 'true'; break; + case 'partnership_suspended': $out['partnership_suspended'] = $v === 'true'; break; + case 'partner_lost_at': $out['partner_lost_at'] = (int)$v; break; } } return $out; } function vv_fb_local_state(): array { - $path = '/boot/config/fallback_state.db'; + $path = STATE_DIR . '/fallback_state.db'; return vv_fb_parse_state(file_exists($path) ? file_get_contents($path) : ''); } function vv_fb_remote_state(string $ip, string $sshKey): array { - $out = vv_pt_ssh($ip, $sshKey, 'cat /boot/config/fallback_state.db 2>/dev/null'); + $out = vv_pt_ssh($ip, $sshKey, vv_remote_state_cmd('fallback_state.db')); return vv_fb_parse_state($out); } @@ -94,8 +98,13 @@ function vv_fb_all(): array { $currentHost = vv_detect_host(); $hosts = vv_fb_known_hosts(); $tsPeers = vv_pt_ts_peers(); - $handbackReq = (int)(vv_fb_scalar(vv_read_conf_raw('master.conf'), 'FALLBACK_HANDBACK_STRIKES') ?: 3); - $fbEnabled = vv_fb_scalar(vv_read_conf_raw('master.conf'), 'FALLBACK_ENABLED') === 'true'; + $masterRaw = vv_read_conf_raw('master.conf'); + $handbackReq = (int)(vv_fb_scalar($masterRaw, 'FALLBACK_HANDBACK_STRIKES') ?: 3); + $fbEnabled = vv_fb_scalar($masterRaw, 'FALLBACK_ENABLED') === 'true'; + $ptEnabled = vv_fb_scalar($masterRaw, 'PARTNERSHIP_ENABLED') === 'true'; + $rsyncEnabled = vv_fb_scalar($masterRaw, 'FALLBACK_RSYNC_ENABLED') !== 'false'; + $checkInterval = (int)(vv_fb_scalar($masterRaw, 'FALLBACK_CHECK_INTERVAL') ?: 30); + $suspendAfter = (int)(vv_fb_scalar($masterRaw, 'FALLBACK_PARTNERSHIP_SUSPEND_AFTER') ?: 120); // Read all host conf raws upfront $raws = []; @@ -162,9 +171,13 @@ function vv_fb_all(): array { } return [ - 'ts' => time(), - 'fb_enabled' => $fbEnabled, - 'handback_req' => $handbackReq, - 'nodes' => $nodes, + 'ts' => time(), + 'fb_enabled' => $fbEnabled, + 'partnership_enabled' => $ptEnabled, + 'fb_rsync_enabled' => $rsyncEnabled, + 'handback_req' => $handbackReq, + 'check_interval' => $checkInterval, + 'suspend_after' => $suspendAfter, + 'nodes' => $nodes, ]; } diff --git a/Plugin/unraid/include/monitor.php b/Plugin/unraid/include/monitor.php index 079f094..df649b8 100644 --- a/Plugin/unraid/include/monitor.php +++ b/Plugin/unraid/include/monitor.php @@ -58,7 +58,7 @@ function vv_fallback_state(): array { $reqStrikes = (int)($vars['FALLBACK_HANDBACK_STRIKES'] ?? 3); $suspendAfter = (int)($vars['FALLBACK_PARTNERSHIP_SUSPEND_AFTER'] ?? 120); - $stateFile = '/tmp/fallback_state.db'; + $stateFile = STATE_DIR . '/fallback_state.db'; if (!file_exists($stateFile)) { return ['state' => 'UNKNOWN', 'enabled' => $enabled, 'check_interval' => $interval, 'handback_strikes' => 0, 'handback_strikes_required' => $reqStrikes, @@ -164,7 +164,7 @@ function vv_watchdog_summary(): array { usort($restarts, fn($a, $b) => $b['ts'] - $a['ts']); // Reboots (12 h) - $rebootRaw = @file_get_contents('/boot/config/system_watchdog_reboots.db') ?: ''; + $rebootRaw = @file_get_contents(STATE_DIR . '/system_watchdog_reboots.db') ?: ''; $rbootCutoff = time() - 43200; $reboots = 0; foreach (explode("\n", trim($rebootRaw)) as $line) { diff --git a/Plugin/unraid/include/partnership.php b/Plugin/unraid/include/partnership.php index 7e8b811..edf3b76 100644 --- a/Plugin/unraid/include/partnership.php +++ b/Plugin/unraid/include/partnership.php @@ -10,7 +10,7 @@ require_once __DIR__ . '/common.php'; // vv_system_info(), vv_docker_containers( function vv_pt_config(): array { $v = vv_conf_vars(); $offlineDays = null; - $odFile = '/boot/config/partnership_offline_days.db'; + $odFile = STATE_DIR . '/partnership_offline_days.db'; if (file_exists($odFile)) { $raw = trim(@file_get_contents($odFile) ?: ''); if (is_numeric($raw)) $offlineDays = (int)$raw; @@ -209,12 +209,11 @@ function vv_pt_nodes(): array { // Fallback state $fbState = 'UNKNOWN'; - $fbPath = '/boot/config/fallback_state.db'; if ($isMe) { - $fb = vv_pt_read_db($fbPath); + $fb = vv_pt_read_db(STATE_DIR . '/fallback_state.db'); $fbState = $fb['state'] ?? 'UNKNOWN'; } elseif ($ts['online'] && $ts['ip'] && $mySshKey) { - $out = vv_pt_ssh($ts['ip'], $mySshKey, 'cat /boot/config/fallback_state.db 2>/dev/null'); + $out = vv_pt_ssh($ts['ip'], $mySshKey, vv_remote_state_cmd('fallback_state.db')); if ($out) { $fb = []; foreach (explode("\n", $out) as $line) { @@ -226,7 +225,7 @@ function vv_pt_nodes(): array { } // Partnership DB — local only (each server writes its own) - $dbPath = "/boot/config/partnership_{$hostname}.db"; + $dbPath = STATE_DIR . "/partnership_{$hostname}.db"; $ptDb = vv_pt_read_db($dbPath); // System info diff --git a/Plugin/unraid/include/watchdog.php b/Plugin/unraid/include/watchdog.php index 2a8a31d..0b43c9c 100644 --- a/Plugin/unraid/include/watchdog.php +++ b/Plugin/unraid/include/watchdog.php @@ -127,14 +127,14 @@ function vv_wd_parse_network_state(string $raw): array { // ── Local state files ───────────────────────────────────────────────────────── function vv_wd_local_states(string $restartLogPath): array { - $rwRaw = @file_get_contents('/tmp/resource_watchdog_state.db') ?: ''; - $dockRaw = @file_get_contents('/tmp/container_watchdog_state.db') ?: ''; - $skipRaw = @file_get_contents('/boot/config/system_watchdog_failed.db') ?: ''; - $sysRaw = @file_get_contents('/tmp/system_watchdog_state.db') ?: ''; - $rebootRaw = @file_get_contents('/boot/config/system_watchdog_reboots.db')?: ''; - $restartRaw= @file_get_contents($restartLogPath) ?: ''; - $storRaw = @file_get_contents('/tmp/storage_watchdog_state.db') ?: ''; - $netWdRaw = @file_get_contents('/tmp/network_watchdog_state.db') ?: ''; + $rwRaw = @file_get_contents(STATE_DIR . '/resource_watchdog_state.db') ?: ''; + $dockRaw = @file_get_contents(STATE_DIR . '/container_watchdog_state.db') ?: ''; + $skipRaw = @file_get_contents(STATE_DIR . '/docker_watchdog_failed.db') ?: ''; + $sysRaw = @file_get_contents(STATE_DIR . '/system_watchdog_state.db') ?: ''; + $rebootRaw = @file_get_contents(STATE_DIR . '/system_watchdog_reboots.db') ?: ''; + $restartRaw= @file_get_contents($restartLogPath) ?: ''; + $storRaw = @file_get_contents(STATE_DIR . '/storage_watchdog_state.db') ?: ''; + $netWdRaw = @file_get_contents(STATE_DIR . '/network_watchdog_state.db') ?: ''; $rw = vv_wd_parse_kv($rwRaw); $dock = vv_wd_parse_kv($dockRaw); diff --git a/Plugin/unraid/pages/fallback.php b/Plugin/unraid/pages/fallback.php index 4a8c77a..fc080b2 100644 --- a/Plugin/unraid/pages/fallback.php +++ b/Plugin/unraid/pages/fallback.php @@ -1,4 +1,5 @@ + +
+
Controls
+ +
+
+
Partnership
+
Master gate — disabling stops all cross-server operations
+
+
+
+ +
+ +
+
+
Fallback
+
Mutual container failover between nodes
+
+
+
+ +
+ +
+
+
Rsync on handback
+
Writeback rsync when the covered host recovers and containers return
+
+
+
+
+ +
- FallBack + Status
-
+
Loading…
+ +
+
Settings
+ +
+ Check interval +
+ + seconds +
+
+ +
+ Handback strikes +
+ + consecutive +
+
+ +
+ Partnership suspend after +
+ + minutes +
+
+ +
+ + +
+
+