diff --git a/Deployment/master.conf.template b/Deployment/master.conf.template index 53a3356..e835264 100644 --- a/Deployment/master.conf.template +++ b/Deployment/master.conf.template @@ -115,12 +115,37 @@ # scripts, state, and data are all available before the array mounts. # Both directories are created automatically if they don't exist. # -# DATA_DIR — historical logs, statistics, discovery histories, blocklists -# STATE_DIR — runtime state files for all scripts (watchdogs, fallback, transcode, etc.) -# Requirement: ALL state files MUST use $STATE_DIR. No /tmp, no /boot/config root. +# DATA_DIR is the one on-disk root. Everything Varaverk persists lives under it, in a +# subdirectory named for what the files are. Move DATA_DIR and the whole tree follows. +# +# It used to be two roots plus two strays: DATA_DIR beside State_Files/ as siblings, with the +# conf-cache backup off in SCRIPTS_DIR/.cache/vv/d and the arr cache backups loose in DATA_DIR's +# root. Nothing was wrong with any one of those decisions; together they meant no single place +# answered "what does Varaverk keep on disk". State is data — it is the data that happens to +# describe right now — so it belongs under the same root as the rest. +# +# db/ — statistics, histories, counters, blocklists. Things that accumulate. +# state/ — runtime state for every script: watchdogs, fallback, transcode, setup. +# Requirement: ALL state files MUST use $STATE_DIR. No /tmp, no repo root. +# ai/ — the retrieval index, operator memory, token ledger, filed bugs, saved chats. +# cache/ — persistent backups of the tmpfs caches, and ONLY those. A file belongs here when +# losing it costs a re-fetch and nothing else; anything that is a source of truth +# belongs in db/ or state/. +# logs/ — retained log output. Live logging still goes to LOG_DIR (/var/log/varaverk). +# +# STATE_DIR keeps its name and changes only its value, which is why this restructure did not +# touch the 15 conf entries, 18 shell paths and 23 PHP paths that build on it. +# +# The tmpfs caches are NOT here and must not be moved here — see VV_CACHE_ROOT below. These are +# on flash; those are read every second by the WebGUI and rewritten by the hundred megabytes. DATA_DIR="/boot/config/plugins/varaverk/data" - STATE_DIR="/boot/config/plugins/varaverk/State_Files" - PERSISTENT_CONF_CACHE="/boot/config/plugins/varaverk/.cache/vv/d" + DB_DIR="${DATA_DIR}/db" + STATE_DIR="${DATA_DIR}/state" + AI_DATA_DIR="${DATA_DIR}/ai" + CACHE_BACKUP_DIR="${DATA_DIR}/cache" + LOG_ARCHIVE_DIR="${DATA_DIR}/logs" + PERSISTENT_CONF_CACHE="${CACHE_BACKUP_DIR}/conf" + ARR_CACHE_BACKUP_DIR="${CACHE_BACKUP_DIR}/arr" # ── Cache Roots ── # Everything Varaverk keeps in RAM, under one root, defined once. @@ -177,7 +202,7 @@ # Runs before rsync — all nodes agree on tracked library before files are transferred. # Remote API keys are read live from each node's config.xml via SSH — never stored here. ARR_SYNC_ENABLED=true - ARR_SYNC_BLOCKLIST="${DATA_DIR}/arr_sync_blocklist.tsv" + ARR_SYNC_BLOCKLIST="${DB_DIR}/arr_sync_blocklist.tsv" ARR_SYNC_CONNECT_TIMEOUT=10 # seconds — SSH connect timeout per node ARR_SYNC_API_TIMEOUT=60 # seconds — curl timeout for library fetches DOCKER_APPDATA_BASE="/mnt/user/appdata" @@ -505,8 +530,8 @@ # than restarting them a second time. A file older than DOCKER_UPDATE_REBUILT_STALE_HOURS is # treated as untrustworthy (docker_update.sh likely didn't run, or didn't run recently) — deleted, # and every container in that tier restarts normally, same as if the file never existed. - DOCKER_UPDATE_REBUILT_DAILY_FILE="$DATA_DIR/docker_update_rebuilt_daily.list" - DOCKER_UPDATE_REBUILT_WEEKLY_FILE="$DATA_DIR/docker_update_rebuilt_weekly.list" + DOCKER_UPDATE_REBUILT_DAILY_FILE="${DB_DIR}/docker_update_rebuilt_daily.list" + DOCKER_UPDATE_REBUILT_WEEKLY_FILE="${DB_DIR}/docker_update_rebuilt_weekly.list" DOCKER_UPDATE_REBUILT_STALE_HOURS=12 # Shares synced during the weekly maintenance window — defined per host in host*.conf. @@ -861,7 +886,7 @@ # Restart loop protection — prevents watchdog from endlessly restarting a broken container WATCHDOG_CONTAINER_RESTART_LIMIT=3 WATCHDOG_CONTAINER_RESTART_WINDOW=1 # rolling window in hours - WATCHDOG_CONTAINER_RESTART_LOG="$DATA_DIR/container_restart_history.db" + WATCHDOG_CONTAINER_RESTART_LOG="${DB_DIR}/container_restart_history.db" # Notification batching — one summary per cycle instead of one ping per event WATCHDOG_BATCH_NOTIFY=true @@ -937,7 +962,7 @@ # Read by sunday_morning_coffee_report.sh for weekly peak/avg/warning summary. INOTIFY_WARN_PCT=80 # warn if inotify instances exceed this % of limit PHP_FPM_WARN_PCT=80 # warn if php-fpm workers exceed this % of max_children - TUNING_MONITOR_LOG="$DATA_DIR/system_tuning_history.db" + TUNING_MONITOR_LOG="${DB_DIR}/system_tuning_history.db" TUNING_LOG_RETENTION=30 # days before old entries are purged # ━━━ Reboot ━━━ @@ -1111,14 +1136,14 @@ LIDARR_MAX_DELETE_GB=5 # require --i-know-what-im-doing if deletion exceeds this LIDARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run # protects against API returning partial data on a bad day - LIDARR_TRACKED_COUNT_FILE="$DATA_DIR/lidarr_tracked.count" + LIDARR_TRACKED_COUNT_FILE="${DB_DIR}/lidarr_tracked.count" LIDARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan # Lidarr tracked-data cache — shared by lidarr_cleanup.sh, lidarr_duplicate_artist_cleanup.sh, # lidarr_missing_art.sh, lidarr_release_fixer.sh, and arr_cache_prefill.sh. See # lidarr_get_tracked_data() in common.sh for the fresh/stale/rescan-active branching logic. - LIDARR_CACHE_FILE="$DATA_DIR/lidarr_tracked_cache.json" - LIDARR_RESCAN_DURATION_DB="$DATA_DIR/lidarr_rescan_duration.db" + LIDARR_CACHE_FILE="${ARR_CACHE_BACKUP_DIR}/lidarr_tracked_cache.json" + LIDARR_RESCAN_DURATION_DB="${DB_DIR}/lidarr_rescan_duration.db" LIDARR_CACHE_MAX_AGE_DAYS=1 # force a live refresh (or rescan-aware wait) past this age ARR_PREFILL_WAIT_MINUTES=10 # array-start prefill: how long to retry reaching each arr LIDARR_EXTENSIONS=("flac" "mp3" "m4a" "wav" "aac" "ogg" "opus" "wma") @@ -1140,7 +1165,7 @@ LIDARR_ART_RETRIES=2 # download retry attempts per image LIDARR_ART_SLEEP_BETWEEN=0.2 # seconds between fanart.tv API calls LIDARR_ART_RECHECK_DAYS=30 # days before re-querying art that upstream didn't have - LIDARR_ART_MISS_CACHE="${DATA_DIR}/lidarr_art_miss_cache.tsv" # negative cache — art upstream has never had + LIDARR_ART_MISS_CACHE="${DB_DIR}/lidarr_art_miss_cache.tsv" # negative cache — art upstream has never had # HOST*_FANART_API_KEY / HOST*_LASTFM_API_KEY — set in host*.conf # Lidarr discovery settings (playback_aware_lidarr_discovery.sh) @@ -1150,7 +1175,7 @@ LIDARR_DISCOVERY_USER_CAP_PCT=35 # max % any single user can contribute to play score (prevents one listener dominating) LIDARR_DISCOVERY_MAX_ADDS=5 # max artists to add per run — quality over bulk LIDARR_DISCOVERY_REJECT_COOLDOWN=30 # days before re-evaluating a rejected artist - LIDARR_DISCOVERY_HISTORY="$DATA_DIR/lidarr_discovery_history.db" + LIDARR_DISCOVERY_HISTORY="${DB_DIR}/lidarr_discovery_history.db" # Sonarr discovery settings (playback_aware_sonarr_discovery.sh) SONARR_DISCOVERY_THRESHOLD=52 # score to accept candidate (0-100) @@ -1162,7 +1187,7 @@ SONARR_DISCOVERY_REJECT_COOLDOWN=60 # days before re-evaluating a rejected show SONARR_DISCOVERY_USER_EPISODE_CAP=8 # max episodes any one user contributes to seed volume score SONARR_DISCOVERY_MONITOR_MODE="all" # Sonarr monitor mode on add: all | future | first | latest | none - SONARR_DISCOVERY_HISTORY="$DATA_DIR/sonarr_discovery_history.db" + SONARR_DISCOVERY_HISTORY="${DB_DIR}/sonarr_discovery_history.db" # Radarr discovery shared settings RADARR_DISCOVERY_THRESHOLD=52 # score to accept candidate (0-100) — lower than Lidarr since diverse seeds rarely overlap @@ -1173,7 +1198,7 @@ RADARR_DISCOVERY_MIN_RATING=60 # min TMDB vote_average × 10 (60 = 6.0/10) RADARR_DISCOVERY_REJECT_COOLDOWN=60 # days before re-evaluating a rejected movie RADARR_DISCOVERY_SEED_LIBRARIES=("Movies") # Emby libraries to draw seed movies from - RADARR_DISCOVERY_HISTORY="$DATA_DIR/radarr_discovery_history.db" + RADARR_DISCOVERY_HISTORY="${DB_DIR}/radarr_discovery_history.db" # Emby → arr sync library allowlists # Only these Emby library names will be considered by the sync tools. @@ -1186,14 +1211,14 @@ SONARR_MAX_DELETE_GB=10 # require --i-know-what-im-doing if deletion exceeds this SONARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run # protects against API returning partial data on a bad day - SONARR_TRACKED_COUNT_FILE="$DATA_DIR/sonarr_tracked.count" + SONARR_TRACKED_COUNT_FILE="${DB_DIR}/sonarr_tracked.count" SONARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan SONARR_MOVE_POLL_TIMEOUT=3600 # seconds to wait for a single async MoveSeries command to # reach "completed" — generous because a large series can sit # queued behind other moves already in progress, not just its # own copy time - CORRUPTION_SCAN_STATE_FILE="${DATA_DIR}/corruption_scan_state.tsv" # clean-file skip-cache - CORRUPTION_SCAN_STRIKES_FILE="${DATA_DIR}/corruption_scan_strikes.tsv" # consecutive corrupt-detection counts, keyed by host path + CORRUPTION_SCAN_STATE_FILE="${DB_DIR}/corruption_scan_state.tsv" # clean-file skip-cache + CORRUPTION_SCAN_STRIKES_FILE="${DB_DIR}/corruption_scan_strikes.tsv" # consecutive corrupt-detection counts, keyed by host path CORRUPTION_SCAN_STRIKE_LIMIT=2 # consecutive corrupt detections (across separate scan runs) # required before --remediate deletes+re-searches — guards # against a one-off ffprobe hiccup (mid-write file, NFS blip) @@ -1224,7 +1249,7 @@ RADARR_MAX_DELETE_GB=30 # require --i-know-what-im-doing if deletion exceeds this RADARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run # protects against API returning partial data on a bad day - RADARR_TRACKED_COUNT_FILE="$DATA_DIR/radarr_tracked.count" + RADARR_TRACKED_COUNT_FILE="${DB_DIR}/radarr_tracked.count" RADARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan RADARR_MOVE_POLL_TIMEOUT=3600 # seconds to wait for a single async MoveMovie command to # reach "completed" — mirrors SONARR_MOVE_POLL_TIMEOUT @@ -1378,7 +1403,7 @@ # Daily statistics log — read by weekly_health_digest.sh for transcode summary. TRANSCODE_STATE_FILE="$STATE_DIR/transcode_state.db" - TRANSCODE_DAILY_LOG="$DATA_DIR/transcode_daily.db" + TRANSCODE_DAILY_LOG="${DB_DIR}/transcode_daily.db" TRANSCODE_LOG_RETENTION=90 # days before old entries purged TRANSCODE_CHECK_EMBY=true @@ -1423,7 +1448,7 @@ # ━━━ ZFS Memory Snapshot ━━━ # Weekly ZFS pool health and memory diagnostic report — informational only. - ZFS_REPORT_LOG="/var/log/zfs-weekly-health.log" + ZFS_REPORT_LOG="${LOG_ARCHIVE_DIR}/zfs-weekly-health.log" ZFS_REPORT_ARC_WARN_PCT=90 # warn if ARC using more than this % of its max ZFS_REPORT_ARC_FREE_WARN_GB=10 # warn if ARC headroom (max - current) drops below this GB ZFS_REPORT_AVAIL_WARN_GB=20 # warn if less than this GB available on ZFS pool @@ -1434,15 +1459,15 @@ # ━━━ Bandwidth Monitor ━━━ # Called automatically by rsync.sh after each sync — one bounded write per run. # Tracks transfer size, duration and profile per sync for weekly summary reporting. - BANDWIDTH_LOG="$DATA_DIR/bandwidth_history.db" + BANDWIDTH_LOG="${DB_DIR}/bandwidth_history.db" BANDWIDTH_LOG_RETENTION=90 # days before old entries purged BANDWIDTH_WARN_GB=50 # flag syncs larger than this in weekly report # Stats files — written by cleanup and recovery scripts, read by coffee report. # All in DATA_DIR — array always running when these are written. - ARR_CLEANUP_STATS="$DATA_DIR/arr_cleanup_stats.db" # lidarr/sonarr/radarr orphan stats - ARR_RECOVERY_STATS="$DATA_DIR/arr_recovery_stats.db" # blocklist + re-search stats - ARR_RECOVERY_FAILURE_COUNTS="$DATA_DIR/arr_recovery_failure_counts.db" # per-item chronic-failure tracking + ARR_CLEANUP_STATS="${DB_DIR}/arr_cleanup_stats.db" # lidarr/sonarr/radarr orphan stats + ARR_RECOVERY_STATS="${DB_DIR}/arr_recovery_stats.db" # blocklist + re-search stats + ARR_RECOVERY_FAILURE_COUNTS="${DB_DIR}/arr_recovery_failure_counts.db" # per-item chronic-failure tracking # ━━━ Health Digest ━━━ # Aggregated system health summary — reads existing state files, no new writes. @@ -1658,7 +1683,7 @@ # gitignored, which is what makes it structurally impossible for a credential to reach the # index: the files holding them were never in the repo. Do not "improve" this to a filesystem # walk — an embedded secret cannot be rotated out of a vector. - AI_INDEX_DB="$DATA_DIR/ai_index.db" + AI_INDEX_DB="${AI_DATA_DIR}/ai_index.db" AI_INDEX_BATCH=32 # chunks per embed request # A pull is the only thing that changes tracked files on a server, so it is the only moment the # index can go stale — and staleness is invisible in the answers, which keep citing the old @@ -1680,7 +1705,7 @@ # The character cap is a context budget, not a style guide. At 16384 the retrieved passages, # the model's reasoning and the conversation history are already competing; memory takes its # share off the top of every single turn, so keep it short and factual. - AI_MEMORY_FILE="$DATA_DIR/ai_memory.md" + AI_MEMORY_FILE="${AI_DATA_DIR}/ai_memory.md" AI_MEMORY_MAX_CHARS=4000 # ~1000 tokens — truncated with a notice if exceeded # ━━━ AI Stored Conversations ━━━ @@ -1711,7 +1736,7 @@ # # Retention is by row count rather than age: pruning is considered only when the file passes a # size threshold, so an ordinary turn costs one stat() and an append. - AI_TOKEN_DB="$DATA_DIR/ai_token_history.db" + AI_TOKEN_DB="${AI_DATA_DIR}/ai_token_history.db" AI_TOKEN_RETAIN_ROWS=20000 # oldest rows dropped past this — years of ordinary use # AI/ai_token_sync.sh pulls each partner's ledger into the tmpfs cache the tab reads, so the diff --git a/Deployment/migrate_data_layout.sh b/Deployment/migrate_data_layout.sh new file mode 100755 index 0000000..758dc15 --- /dev/null +++ b/Deployment/migrate_data_layout.sh @@ -0,0 +1,243 @@ +#!/bin/bash +# ============================================================================================== +# ============================== DATA LAYOUT MIGRATION ========================================= +# ============================================================================================== +# +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# One-time move of everything Varaverk persists into a single rooted tree under DATA_DIR. +# +# State_Files/ → data/state/ +# data/*.db|.count|.tsv|.list → data/db/ +# data/ai_* → data/ai/ +# data/*_tracked_cache.json → data/cache/arr/ +# data/*.log → data/logs/ +# SCRIPTS_DIR/.cache/vv/d/ → data/cache/conf/ +# +# ============================================================================================== +# WHY THIS EXISTS SEPARATELY FROM conf_upgrade +# ============================================================================================== +# +# conf_upgrade adds keys the template has and the installation does not; it never rewrites a +# value the operator already has, which is exactly the behaviour you want from it and exactly +# why it cannot perform this migration. The paths being moved are existing keys — STATE_DIR, +# BANDWIDTH_LOG, AI_INDEX_DB and two dozen more — so their values would keep pointing at the old +# layout forever while the new directory variables sat beside them unused. +# +# So this rewrites those values, then moves the files to match. Both halves, or neither: a conf +# pointing at a directory the data is not in is worse than not having started. +# +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# Idempotent +# Every step tests before acting. A second run reports "already migrated" and changes nothing, +# which matters because the natural instinct after a partial failure is to run it again. +# +# Moves, never copies-and-deletes +# mv within one filesystem is atomic per file, so a reader either sees the file at the old +# path or the new one — never a half-written copy at both. Nothing is deleted; if a file +# cannot be moved it is reported and left exactly where it is. +# +# Conf is backed up before it is rewritten +# master.conf.bak-, next to the original, same convention conf_upgrade uses. +# +# Refuses to run while the orchestrators might be writing +# A watchdog that sourced conf before the rewrite and writes state after the move would put a +# file back at the old path. The window is seconds and the damage is one stale file, but the +# check costs nothing and the failure is silent otherwise. +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# migrate_data_layout.sh --dry-run Show what would move. Changes nothing. Do this first. +# migrate_data_layout.sh Perform the migration. +# migrate_data_layout.sh --force Skip the running-orchestrator check. +# +# ============================================================================================== + +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +CONF="$ROOT/Configurations/master.conf" + +DRY_RUN=false +FORCE=false +for a in "$@"; do + case "$a" in + --dry-run) DRY_RUN=true ;; + --force) FORCE=true ;; + *) echo "Unknown argument: $a" >&2; exit 2 ;; + esac +done + +[[ -f "$CONF" ]] || { echo "[FATAL] master.conf not found at $CONF" >&2; exit 1; } + +# Resolve the roots the same way load_config.sh will after this runs. +SCRIPTS_DIR="$ROOT" +DATA_DIR="$(grep -m1 -E '^\s*DATA_DIR=' "$CONF" | cut -d'"' -f2)" +DATA_DIR="${DATA_DIR//\$\{SCRIPTS_DIR\}/$SCRIPTS_DIR}" +DATA_DIR="${DATA_DIR:-$ROOT/data}" + +OLD_STATE="$ROOT/State_Files" +OLD_CONFCACHE="$ROOT/.cache/vv/d" + +DB_DIR="$DATA_DIR/db" +STATE_DIR="$DATA_DIR/state" +AI_DATA_DIR="$DATA_DIR/ai" +CACHE_BACKUP_DIR="$DATA_DIR/cache" +ARR_CACHE_BACKUP_DIR="$CACHE_BACKUP_DIR/arr" +CONF_CACHE_BACKUP_DIR="$CACHE_BACKUP_DIR/conf" +LOG_ARCHIVE_DIR="$DATA_DIR/logs" + +moved=0; skipped=0; failed=0 + +say() { printf ' %s\n' "$*"; } +step() { printf '\n━━━ %s ━━━\n' "$*"; } + +# ── Guard: orchestrators mid-run ────────────────────────────────────────────── +if [[ "$FORCE" == false && "$DRY_RUN" == false ]]; then + running=$(pgrep -fa 'Orchestrators/|Watchdogs/' 2>/dev/null | grep -v "$$" | grep -v migrate_data_layout || true) + if [[ -n "$running" ]]; then + echo "[ABORT] Orchestrator or watchdog is running — it may rewrite state mid-move:" >&2 + echo "$running" >&2 + echo "Wait for it to finish, or re-run with --force if you are sure." >&2 + exit 1 + fi +fi + +# ── Move one path ───────────────────────────────────────────────────────────── +move() { + local src="$1" dstdir="$2" base + base="$(basename "$src")" + [[ -e "$src" ]] || return 0 + if [[ -e "$dstdir/$base" ]]; then + say "skip $base — already at ${dstdir#$DATA_DIR/}/" + ((skipped++)); return 0 + fi + if [[ "$DRY_RUN" == true ]]; then + say "would $base → ${dstdir#$DATA_DIR/}/" + ((moved++)); return 0 + fi + mkdir -p "$dstdir" 2>/dev/null + if mv "$src" "$dstdir/$base" 2>/dev/null; then + say "moved $base → ${dstdir#$DATA_DIR/}/" + ((moved++)) + else + say "FAILED $base — left in place" + ((failed++)) + fi +} + +# ── 1. Rewrite the conf values conf_upgrade cannot ──────────────────────────── +step "Step 1: master.conf path values" +if grep -q 'STATE_DIR="\${DATA_DIR}/state"' "$CONF"; then + say "already migrated — no conf changes needed" +else + if [[ "$DRY_RUN" == false ]]; then + cp "$CONF" "${CONF}.bak-$(date +%Y%m%d%H%M%S)" + sed -i -E \ + -e 's|^(\s*STATE_DIR=)".*"|\1"${DATA_DIR}/state"|' \ + -e 's|^(\s*PERSISTENT_CONF_CACHE=)".*"|\1"${CACHE_BACKUP_DIR}/conf"|' \ + "$CONF" + for v in ARR_SYNC_BLOCKLIST DOCKER_UPDATE_REBUILT_DAILY_FILE DOCKER_UPDATE_REBUILT_WEEKLY_FILE \ + WATCHDOG_CONTAINER_RESTART_LOG TUNING_MONITOR_LOG LIDARR_TRACKED_COUNT_FILE \ + LIDARR_RESCAN_DURATION_DB LIDARR_ART_MISS_CACHE LIDARR_DISCOVERY_HISTORY \ + SONARR_DISCOVERY_HISTORY RADARR_DISCOVERY_HISTORY SONARR_TRACKED_COUNT_FILE \ + CORRUPTION_SCAN_STATE_FILE CORRUPTION_SCAN_STRIKES_FILE RADARR_TRACKED_COUNT_FILE \ + TRANSCODE_DAILY_LOG BANDWIDTH_LOG ARR_CLEANUP_STATS ARR_RECOVERY_STATS \ + ARR_RECOVERY_FAILURE_COUNTS; do + sed -i -E "s|^(\s*${v}=\")\\\$\{?DATA_DIR\}?/|\1\${DB_DIR}/|" "$CONF" + done + for v in AI_INDEX_DB AI_MEMORY_FILE AI_TOKEN_DB; do + sed -i -E "s|^(\s*${v}=\")\\\$\{?DATA_DIR\}?/|\1\${AI_DATA_DIR}/|" "$CONF" + done + sed -i -E 's|^(\s*LIDARR_CACHE_FILE=")\$\{?DATA_DIR\}?/|\1${ARR_CACHE_BACKUP_DIR}/|' "$CONF" + sed -i -E 's|^(\s*ZFS_REPORT_LOG=")\$\{?DATA_DIR\}?/|\1${LOG_ARCHIVE_DIR}/|' "$CONF" + say "rewritten — backup kept beside it" + else + say "would rewrite STATE_DIR, PERSISTENT_CONF_CACHE and 25 file paths" + fi +fi + +# ── 2. Build the tree ───────────────────────────────────────────────────────── +step "Step 2: directory tree" +for d in "$DB_DIR" "$STATE_DIR" "$AI_DATA_DIR" "$ARR_CACHE_BACKUP_DIR" "$LOG_ARCHIVE_DIR"; do + if [[ -d "$d" ]]; then say "exists ${d#$DATA_DIR/}" + elif [[ "$DRY_RUN" == true ]]; then say "would create ${d#$DATA_DIR/}" + else mkdir -p "$d" && say "created ${d#$DATA_DIR/}" + fi +done +# The conf cache carries partner credentials and keeps its restrictive mode. +if [[ "$DRY_RUN" == false ]]; then + mkdir -p "$CONF_CACHE_BACKUP_DIR" && chmod 700 "$CONF_CACHE_BACKUP_DIR" + say "created cache/conf (0700)" +fi + +# ── 3. State files ──────────────────────────────────────────────────────────── +step "Step 3: State_Files → data/state" +if [[ -d "$OLD_STATE" ]]; then + shopt -s nullglob dotglob + for f in "$OLD_STATE"/*; do move "$f" "$STATE_DIR"; done + shopt -u nullglob dotglob + if [[ "$DRY_RUN" == false && -d "$OLD_STATE" ]]; then + rmdir "$OLD_STATE" 2>/dev/null && say "removed empty State_Files/" \ + || say "State_Files/ not empty — left in place, inspect it" + fi +else + say "no State_Files/ — nothing to do" +fi + +# ── 4. Sort the data root ───────────────────────────────────────────────────── +step "Step 4: sort data/ into subfolders" +classify() { + local f="$1" base; base="$(basename "$f")" + case "$base" in + ai_*) move "$f" "$AI_DATA_DIR" ;; + *_tracked_cache.json) move "$f" "$ARR_CACHE_BACKUP_DIR" ;; + *.log) move "$f" "$LOG_ARCHIVE_DIR" ;; + *.db|*.count|*.tsv|*.list|*.json) move "$f" "$DB_DIR" ;; + *) say "leave $base — unclassified, left in data/" ;; + esac +} + +# Two passes, sidecars first. A SQLite database is three files, and the -wal holds committed +# transactions that have not been checkpointed into the .db yet. Move the .db first and any +# process that opens it during the gap sees a database with no write-ahead log, creates a fresh +# one at the old path, and everything still in the old -wal is lost when it is moved over the +# top. Sidecars ahead of their base closes that ordering: the worst case becomes a database +# opened without its log still sitting beside it, which SQLite handles. +shopt -s nullglob +for f in "$DATA_DIR"/*-wal "$DATA_DIR"/*-shm; do + [[ -d "$f" ]] && continue + classify "$f" +done +for f in "$DATA_DIR"/*; do + [[ -d "$f" ]] && continue # subfolders are the destinations + case "$(basename "$f")" in *-wal|*-shm) continue ;; esac + classify "$f" +done +shopt -u nullglob + +# ── 5. Conf cache backup ────────────────────────────────────────────────────── +step "Step 5: conf cache backup" +if [[ -d "$OLD_CONFCACHE" ]]; then + shopt -s nullglob dotglob + for f in "$OLD_CONFCACHE"/*; do move "$f" "$CONF_CACHE_BACKUP_DIR"; done + shopt -u nullglob dotglob + [[ "$DRY_RUN" == false ]] && rmdir "$OLD_CONFCACHE" "$ROOT/.cache/vv" "$ROOT/.cache" 2>/dev/null +else + say "no $OLD_CONFCACHE — nothing to do" +fi + +# ── Summary ─────────────────────────────────────────────────────────────────── +printf '\n━━━━━ SUMMARY ━━━━━\n' +printf ' %-10s %s\n' "moved:" "$moved" +printf ' %-10s %s\n' "skipped:" "$skipped" +printf ' %-10s %s\n' "failed:" "$failed" +[[ "$DRY_RUN" == true ]] && printf '\n DRY RUN — nothing was changed.\n' +[[ "$failed" -gt 0 ]] && exit 1 +exit 0 diff --git a/Plugin/unraid/Tools/storage_migrate.sh b/Plugin/unraid/Tools/storage_migrate.sh index 299070c..8e783be 100755 --- a/Plugin/unraid/Tools/storage_migrate.sh +++ b/Plugin/unraid/Tools/storage_migrate.sh @@ -283,10 +283,10 @@ if [[ "$DRY_RUN" == false ]]; then if [[ -f "$NEW_MASTER" ]]; then sed -i "s|^\(\s*TARGET_DIR\s*=\s*\)\"[^\"]*\"|\1\"$DST\"|" "$NEW_MASTER" sed -i "s|^\(\s*DATA_DIR\s*=\s*\)\"[^\"]*\"|\1\"$DST/data\"|" "$NEW_MASTER" - sed -i "s|^\(\s*STATE_DIR\s*=\s*\)\"[^\"]*\"|\1\"$DST/State_Files\"|" "$NEW_MASTER" + sed -i "s|^\(\s*STATE_DIR\s*=\s*\)\"[^\"]*\"|\1\"${DST}/data/state\"|" "$NEW_MASTER" echo " TARGET_DIR → $DST ✅" echo " DATA_DIR → $DST/data ✅" - echo " STATE_DIR → $DST/State_Files ✅" + echo " STATE_DIR → $DST/data/state ✅" else error "master.conf not found at $NEW_MASTER" exit 1 @@ -326,7 +326,7 @@ if [[ "$DRY_RUN" == false ]]; then define('SCRIPTS_DIR', \$_c['SCRIPTS_DIR'] ?? '/boot/config/plugins/varaverk'); define('CONF_DIR', SCRIPTS_DIR . '/Configurations'); define('DATA_DIR', SCRIPTS_DIR . '/data'); - define('STATE_DIR', SCRIPTS_DIR . '/State_Files'); + define('STATE_DIR', SCRIPTS_DIR . '/data/state'); define('LOG_DIR', '/var/log/varaverk'); define('VV_SETUP_STATE_FILE', STATE_DIR . '/varaverk_setup.db'); require_once '/usr/local/emhttp/plugins/varaverk/include/confform.php'; diff --git a/Plugin/unraid/api/rsync.php b/Plugin/unraid/api/rsync.php index 34f94e6..9c56291 100644 --- a/Plugin/unraid/api/rsync.php +++ b/Plugin/unraid/api/rsync.php @@ -138,7 +138,7 @@ $base['windows']['fallback'] = ($vars['FALLBACK_RSYNC_ENABLED'] ?? 'true') !== $base['windows']['monthly'] = ($vars['MONTHLY_RSYNC_ENABLED'] ?? 'true') !== 'false'; // Bandwidth history — last 30 days -$bwLog = DATA_DIR . '/bandwidth_history.db'; +$bwLog = DB_DIR . '/bandwidth_history.db'; $warnGb = (float)($vars['BANDWIDTH_WARN_GB'] ?? 50); $cutoff = date('Y-m-d', strtotime('-30 days')); $history = []; diff --git a/Plugin/unraid/include/ai.php b/Plugin/unraid/include/ai.php index 6a42688..3aa32bc 100644 --- a/Plugin/unraid/include/ai.php +++ b/Plugin/unraid/include/ai.php @@ -115,7 +115,7 @@ function vv_ai_config(): array { 'url' => rtrim(trim($vars["{$host}_OLLAMA_URL"] ?? ''), '/'), 'model' => trim($vars["{$host}_OLLAMA_MODEL"] ?? ''), 'embed_model' => trim($vars["{$host}_OLLAMA_EMBED_MODEL"] ?? 'nomic-embed-text'), - 'db' => trim($vars['AI_INDEX_DB'] ?? '') ?: DATA_DIR . '/ai_index.db', + 'db' => trim($vars['AI_INDEX_DB'] ?? '') ?: AI_DATA_DIR . '/ai_index.db', 'k' => (int)($vars['AI_SEARCH_K'] ?? 8), 'per_file' => (int)($vars['AI_SEARCH_PER_FILE'] ?? 3), 'timeout' => (int)($vars['AI_REQUEST_TIMEOUT'] ?? 240), @@ -552,7 +552,7 @@ function vv_ai_memory_path(): string { $vars = vv_conf_vars(); $p = trim($vars['AI_MEMORY_FILE'] ?? ''); $p = str_replace(['$DATA_DIR', '${DATA_DIR}'], DATA_DIR, $p); - return $p !== '' ? $p : DATA_DIR . '/ai_memory.md'; + return $p !== '' ? $p : AI_DATA_DIR . '/ai_memory.md'; } function vv_ai_memory_max(): int { @@ -624,7 +624,7 @@ function vv_ai_memory_write(string $text): array { function vv_ai_token_db(): string { $p = str_replace(['$DATA_DIR', '${DATA_DIR}'], DATA_DIR, trim(vv_conf_vars()['AI_TOKEN_DB'] ?? '')); - return $p !== '' ? $p : DATA_DIR . '/ai_token_history.db'; + return $p !== '' ? $p : AI_DATA_DIR . '/ai_token_history.db'; } function vv_ai_token_retain(): int { @@ -966,7 +966,7 @@ function vv_ai_scope_ok(string $scope): bool { // // Under data/ and therefore gitignored: these quote this installation's logs. function vv_ai_bugs_dir(): string { - $d = DATA_DIR . '/ai_bugs'; + $d = AI_DATA_DIR . '/ai_bugs'; if (!is_dir($d)) @mkdir($d, 0755, true); return $d; } @@ -1094,7 +1094,7 @@ function vv_ai_bug_set_open(string $id, bool $open): bool { // Markdown rather than a delimited .db because the useful part is prose — a fix is a sentence, // not a field — and it stays hand-editable when a note turns out to be wrong. function vv_ai_incidents_path(): string { - return DATA_DIR . '/ai_incidents.md'; + return AI_DATA_DIR . '/ai_incidents.md'; } // One entry, appended. The symptom is captured from what was being asked; the fix is written by @@ -1159,7 +1159,7 @@ function vv_ai_incidents_for(string $scope, int $max = 4): array { // runs often enough to be trusted with it, and an unbounded directory here would quietly grow // for as long as the operator keeps talking to the assistant. function vv_ai_chats_dir(): string { - $d = DATA_DIR . '/ai_chats'; + $d = AI_DATA_DIR . '/ai_chats'; if (!is_dir($d)) @mkdir($d, 0755, true); return $d; } diff --git a/Plugin/unraid/include/arrs.php b/Plugin/unraid/include/arrs.php index 9ed8b33..771772b 100644 --- a/Plugin/unraid/include/arrs.php +++ b/Plugin/unraid/include/arrs.php @@ -212,7 +212,7 @@ function vv_arr_cleanup_stats(string $type): array { // Fallback: daily aggregate db — date|arr|orphan_count|orphan_bytes|junk_count|junk_bytes|recent_count|tracked_count if ($out['last_run'] === null) { - $dbFile = DATA_DIR . '/arr_cleanup_stats.db'; + $dbFile = DB_DIR . '/arr_cleanup_stats.db'; if (file_exists($dbFile)) { $last = null; foreach (file($dbFile, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES) as $line) { @@ -255,7 +255,7 @@ function vv_arr_discovery_stats(string $type): array { // Fallback: per-title history db — status|id|date[|title] if ($out['last_run'] === null) { - $dbFile = DATA_DIR . '/' . $type . '_discovery_history.db'; + $dbFile = DB_DIR . '/' . $type . '_discovery_history.db'; if (file_exists($dbFile)) { $lastDate = null; $added = 0; foreach (file($dbFile, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES) as $line) { @@ -326,7 +326,7 @@ function vv_arr_recovery_stats(): array { // Fallback: daily aggregate db — date|time|count|bytes if ($out['last_run'] === null) { - $dbFile = DATA_DIR . '/arr_recovery_stats.db'; + $dbFile = DB_DIR . '/arr_recovery_stats.db'; if (file_exists($dbFile)) { $lines = file($dbFile, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES); $last = $lines ? end($lines) : null; diff --git a/Plugin/unraid/include/common.php b/Plugin/unraid/include/common.php index 39c91f8..3e26338 100644 --- a/Plugin/unraid/include/common.php +++ b/Plugin/unraid/include/common.php @@ -64,6 +64,23 @@ require_once __DIR__ . '/config.php'; require_once __DIR__ . '/unraid_api.php'; +// Timestamp out of the container restart log, whose second field is a formatted local date +// ("2026-08-02 13:15:07") rather than an epoch — docker_watchdog.sh writes it that way because +// its own rolling-window trim compares the strings lexically in awk. +// +// Lives here rather than in watchdog.php because include/monitor.php parses the same file for +// the dashboard card and does not include watchdog.php. Both readers previously cast the field +// with (int), which stops at the first non-digit and returned 2026 for every line ever written — +// below any cutoff, so both restart lists were permanently empty and looked exactly like +// "nothing has restarted". +// +// Accepts an epoch too, so that changing the writer later does not require changing the readers. +function vv_wd_restart_ts(string $raw): int { + $raw = trim($raw); + if (ctype_digit($raw)) return (int)$raw; + return (int)(strtotime($raw) ?: 0); +} + function vv_system_info(): array { // ── Shared local reads (always needed regardless of API) ────────────────── $ident = @parse_ini_file('/boot/config/ident.cfg') ?: []; diff --git a/Plugin/unraid/include/config.php b/Plugin/unraid/include/config.php index 1481422..8bdafe1 100644 --- a/Plugin/unraid/include/config.php +++ b/Plugin/unraid/include/config.php @@ -103,8 +103,20 @@ $_vv_cfg = @parse_ini_file(PLUGIN_CFG) ?: []; define('SCRIPTS_DIR', $_vv_cfg['SCRIPTS_DIR'] ?? '/boot/config/plugins/varaverk'); define('CONF_DIR', SCRIPTS_DIR . '/Configurations'); define('DEPLOY_DIR', SCRIPTS_DIR . '/Deployment'); -define('DATA_DIR', SCRIPTS_DIR . '/data'); -define('STATE_DIR', SCRIPTS_DIR . '/State_Files'); +// DATA_DIR is the one on-disk root; everything Varaverk persists lives under it in a +// subdirectory named for what the files are. Mirrors the same block in master.conf, which is +// where the shell layer reads them from — these are derived from SCRIPTS_DIR rather than parsed +// so that a conf that has not upgraded yet still resolves, and so this file keeps working when +// master.conf is missing entirely (setup, first boot, a botched pull). +// +// STATE_DIR moved from SCRIPTS_DIR/State_Files to DATA_DIR/state and kept its name, which is why +// the 23 call sites in this layer that build on it needed no edits at all. +define('DATA_DIR', SCRIPTS_DIR . '/data'); +define('DB_DIR', DATA_DIR . '/db'); +define('STATE_DIR', DATA_DIR . '/state'); +define('AI_DATA_DIR', DATA_DIR . '/ai'); +define('CACHE_BACKUP_DIR', DATA_DIR . '/cache'); +define('LOG_ARCHIVE_DIR', DATA_DIR . '/logs'); define('LOG_DIR', '/var/log/varaverk'); // User-authored custom scripts (scheduler page "+ Create Script") — kept outside the git // repo entirely, alongside the User Scripts plugin's own storage. Any *.sh file placed @@ -204,12 +216,21 @@ function vv_push_setup_state(): void { break; } } - $remoteStatePath = $remoteSD . '/State_Files/varaverk_setup.db'; - shell_exec($sshBase . ' "mkdir -p ' . escapeshellarg(dirname($remoteStatePath)) . '" 2>/dev/null'); - $dest = escapeshellarg('root@' . $ip . ':' . $remoteStatePath); - exec('scp -i ' . escapeshellarg($sshKey) - . ' -o ConnectTimeout=10 -o StrictHostKeyChecking=no' - . ' ' . escapeshellarg(VV_SETUP_STATE_FILE) . ' ' . $dest . ' 2>&1'); + // Which layout the partner uses is decided ON the partner, not assumed here. State moved + // from SCRIPTS_DIR/State_Files to DATA_DIR/state, and this file is written to whichever + // one that host will actually read — a setup state pushed to the directory the partner + // does not read is worse than not pushing it, because the push reports success. + // + // The order matters: prefer the new path, fall back to the old ONLY if it already exists. + // A partner that has neither is a fresh install on current code, which reads the new one. + // Piped over ssh rather than scp'd so the resolution and the write are the same call — + // scp needs the path decided here, which is the thing that cannot be known here. + $remoteResolve = 'sf="' . $remoteSD . '/data/state"; ' + . '[ -d "$sf" ] || { [ -d "' . $remoteSD . '/State_Files" ] ' + . '&& sf="' . $remoteSD . '/State_Files"; }; ' + . 'mkdir -p "$sf" && cat > "$sf/varaverk_setup.db"'; + exec('cat ' . escapeshellarg(VV_SETUP_STATE_FILE) . ' | ' + . $sshBase . ' ' . escapeshellarg($remoteResolve) . ' 2>&1'); } } @@ -612,13 +633,22 @@ function vv_auto_create_api_key(string $hostId, string $confFile): array { return ['ok' => true, 'key_preview' => $key ? substr($key, 0, 8) . '...' . substr($key, -4) : 'registered']; } -// Build a bash command that reads a state file from the REMOTE host's State_Files/. +// Build a bash command that reads a state file from the REMOTE host's state directory. // Reads the remote's varaverk.cfg to resolve their SCRIPTS_DIR (may differ from ours // when the remote is in appdata mode). Falls back to the internal plugin path. +// +// The state directory is probed on the far side rather than assumed, because it moved: +// SCRIPTS_DIR/State_Files became DATA_DIR/state, and a partner may not have pulled that yet. +// This is the call that reads the partner's fallback_state.db, and a miss returns an empty +// string — which the callers cannot distinguish from "partner is in NORMAL state". Reading the +// wrong directory would therefore not look like an error, it would look like an answer. Probing +// also means the two hosts can be upgraded in either order. function vv_remote_state_cmd(string $filename): string { $fn = basename($filename); return 'sd=$(grep -m1 SCRIPTS_DIR= /boot/config/plugins/varaverk/varaverk.cfg 2>/dev/null' - . ' | cut -d\'"\' -f2); cat "${sd:-/boot/config/plugins/varaverk}/State_Files/' . $fn . '" 2>/dev/null'; + . ' | cut -d\'"\' -f2); sd="${sd:-/boot/config/plugins/varaverk}"; ' + . 'sf="$sd/data/state"; [ -d "$sf" ] || sf="$sd/State_Files"; ' + . 'cat "$sf/' . $fn . '" 2>/dev/null'; } // Local LAN IP via routing table — static-cached per request. diff --git a/Plugin/unraid/include/monitor.php b/Plugin/unraid/include/monitor.php index 3a05bc8..e18f8ea 100644 --- a/Plugin/unraid/include/monitor.php +++ b/Plugin/unraid/include/monitor.php @@ -200,14 +200,18 @@ function vv_watchdog_summary(): array { } // Recent restarts (24 h) - $restartLog = DATA_DIR . '/container_restart_history.db'; + $restartLog = DB_DIR . '/container_restart_history.db'; $restartRaw = @file_get_contents($restartLog) ?: ''; $cutoff = time() - 86400; $restarts = []; foreach (explode("\n", trim($restartRaw)) as $line) { if (!$line || !str_contains($line, '|')) continue; - [$name, $ts] = explode('|', $line, 2); - if ((int)$ts >= $cutoff) $restarts[] = ['name' => trim($name), 'ts' => (int)$ts]; + // The second field is a formatted local date, not an epoch — see + // vv_wd_restart_ts() in watchdog.php for why, and for what casting it with (int) + // silently did to this list for as long as it has existed. + [$name, $raw] = explode('|', $line, 2); + $ts = vv_wd_restart_ts(trim($raw)); + if ($ts >= $cutoff) $restarts[] = ['name' => trim($name), 'ts' => $ts]; } usort($restarts, fn($a, $b) => $b['ts'] - $a['ts']); @@ -422,7 +426,7 @@ function vv_rsync_status(): array { } // Profile activity — last 7 days, aggregated per profile - $bwLog = DATA_DIR . '/bandwidth_history.db'; + $bwLog = DB_DIR . '/bandwidth_history.db'; $cutoff7 = date('Y-m-d', strtotime('-7 days')); $profiles = []; if (file_exists($bwLog)) { diff --git a/Plugin/unraid/include/watchdog.php b/Plugin/unraid/include/watchdog.php index 4bd41b4..ad7cef2 100644 --- a/Plugin/unraid/include/watchdog.php +++ b/Plugin/unraid/include/watchdog.php @@ -95,22 +95,36 @@ function vv_wd_parse_kv(string $text): array { return $out; } -// Restart log: "container|timestamp" one per line +// Restart log: "container|2026-08-02 13:15:07" one per line. +// +// The second field is a formatted local timestamp, not an epoch — docker_watchdog.sh writes it +// with date '+%Y-%m-%d %H:%M:%S' because its own rolling-window trim compares the strings +// lexically in awk, which is correct for that format. This function used to cast it with (int), +// which stops at the first non-digit and yielded 2026 for every line ever written. 2026 is below +// any plausible cutoff, so every entry was discarded and both restart panels — the Monitor card +// and the Watchdog page — were permanently empty. Not visibly broken: an empty list reads +// exactly like "nothing has restarted", which is the answer you least want to be wrong about +// during a restart loop. +// +// Parsed, not reformatted. Changing what the watchdog writes would strand every existing entry +// and every awk comparison in Tools/watchdog_skip_list_manager.sh. function vv_wd_parse_restart_log(string $text, int $windowSeconds = 86400): array { - $now = time(); - $cutoff = $now - $windowSeconds; + $cutoff = time() - $windowSeconds; $entries = []; foreach (explode("\n", trim($text)) as $line) { $line = trim($line); if (!$line || !str_contains($line, '|')) continue; - [$name, $ts] = explode('|', $line, 2); - $ts = (int)$ts; + [$name, $raw] = explode('|', $line, 2); + $ts = vv_wd_restart_ts(trim($raw)); if ($ts >= $cutoff) $entries[] = ['name' => trim($name), 'ts' => $ts]; } usort($entries, fn($a, $b) => $b['ts'] - $a['ts']); return $entries; } +// vv_wd_restart_ts() lives in common.php — include/monitor.php parses the same file and does not +// include this one, so a helper defined here would be a fatal on the dashboard. + // Skip list: one container name per line function vv_wd_parse_skiplist(string $text): array { return array_values(array_filter(array_map('trim', explode("\n", $text)))); @@ -239,28 +253,38 @@ function vv_wd_remote_data(string $ip, string $sshKey, string $restartLogPath): // /proc/meminfo is passed as a raw section (not awk-parsed) to avoid quoting // fragility — escapeshellarg() single-quotes the whole command so awk \$2 // inside double-quotes is unreliable across Unraid builds. - // State files live under the REMOTE's own SCRIPTS_DIR/State_Files (may differ - // from ours in flash mode) — resolve it once, same idiom as vv_remote_state_cmd(). + // State files live under the REMOTE's own SCRIPTS_DIR (may differ from ours in flash mode), + // so it is resolved on the far side — same idiom as vv_remote_state_cmd(). + // + // The layout is probed rather than assumed. State moved from SCRIPTS_DIR/State_Files to + // DATA_DIR/state, and the histories from DATA_DIR's root into DATA_DIR/db, but a partner is + // not guaranteed to have pulled that yet — and this is the call that reports whether the + // partner's watchdogs are healthy. Guessing wrong returns empty strings for every state + // file, which reads as "partner has no strikes" rather than as an error. Probing costs one + // directory test and makes the answer correct in both directions, which also means the two + // hosts can be upgraded in either order. $restartLogName = basename($restartLogPath); $cmd = 'sd=$(grep -m1 SCRIPTS_DIR= /boot/config/plugins/varaverk/varaverk.cfg 2>/dev/null' . ' | cut -d\'"\' -f2); sd="${sd:-/boot/config/plugins/varaverk}"; ' + . 'sf="$sd/data/state"; [ -d "$sf" ] || sf="$sd/State_Files"; ' + . 'db="$sd/data/db"; [ -d "$db" ] || db="$sd/data"; ' . "printf 'UPTIME:%s\nLOAD:%s\nCORES:%s\nDAEMON:%s\nOOM:%s\nBASELINECOUNT:%s\nBASELINEAGE:%s\n---MEMINFO---\n%s\n---RW---\n%s\n---DOCK---\n%s\n---SKIP---\n%s\n---SYS---\n%s\n---REBOOT---\n%s\n---RESTART---\n%s\n---STORAGE---\n%s\n---NETWORK---\n%s\n' " . '"$(cat /proc/uptime|cut -d\" \" -f1)" ' . '"$(cat /proc/loadavg|cut -d\" \" -f1)" ' . '"$(nproc)" ' . '"$(docker info >/dev/null 2>&1 && echo ok || echo err)" ' - . '"$(cat "$sd/State_Files/system_watchdog_oom.db" 2>/dev/null||echo 0)" ' - . '"$(wc -l < "$sd/State_Files/watchdog_appdata_growth.db" 2>/dev/null||echo 0)" ' - . '"$(stat -c %Y "$sd/State_Files/watchdog_appdata_growth.db" 2>/dev/null||echo 0)" ' + . '"$(cat "$sf/system_watchdog_oom.db" 2>/dev/null||echo 0)" ' + . '"$(wc -l < "$sf/watchdog_appdata_growth.db" 2>/dev/null||echo 0)" ' + . '"$(stat -c %Y "$sf/watchdog_appdata_growth.db" 2>/dev/null||echo 0)" ' . '"$(cat /proc/meminfo 2>/dev/null)" ' - . '"$(cat "$sd/State_Files/resource_watchdog_state.db" 2>/dev/null)" ' - . '"$(cat "$sd/State_Files/container_watchdog_state.db" 2>/dev/null)" ' - . '"$(cat "$sd/State_Files/docker_watchdog_failed.db" 2>/dev/null)" ' - . '"$(cat "$sd/State_Files/system_watchdog_state.db" 2>/dev/null)" ' - . '"$(cat "$sd/State_Files/system_watchdog_reboots.db" 2>/dev/null)" ' - . '"$(cat "$sd/data/' . $restartLogName . '" 2>/dev/null)" ' - . '"$(cat "$sd/State_Files/storage_watchdog_state.db" 2>/dev/null)" ' - . '"$(cat "$sd/State_Files/network_watchdog_state.db" 2>/dev/null)"'; + . '"$(cat "$sf/resource_watchdog_state.db" 2>/dev/null)" ' + . '"$(cat "$sf/container_watchdog_state.db" 2>/dev/null)" ' + . '"$(cat "$sf/docker_watchdog_failed.db" 2>/dev/null)" ' + . '"$(cat "$sf/system_watchdog_state.db" 2>/dev/null)" ' + . '"$(cat "$sf/system_watchdog_reboots.db" 2>/dev/null)" ' + . '"$(cat "$db/' . $restartLogName . '" 2>/dev/null)" ' + . '"$(cat "$sf/storage_watchdog_state.db" 2>/dev/null)" ' + . '"$(cat "$sf/network_watchdog_state.db" 2>/dev/null)"'; $out = vv_pt_ssh($ip, $sshKey, $cmd, 8); if (!$out) return null; @@ -366,7 +390,7 @@ function vv_wd_all(): array { $currentHost = vv_detect_host(); $tsPeers = vv_pt_ts_peers(); $masterRaw = vv_read_conf_raw('master.conf'); - $restartLog = DATA_DIR . '/container_restart_history.db'; + $restartLog = DB_DIR . '/container_restart_history.db'; // Config thresholds from master.conf $cfg = [ diff --git a/common.sh b/common.sh index ec412f3..8fb734b 100755 --- a/common.sh +++ b/common.sh @@ -2505,7 +2505,7 @@ arr_cache_file() { echo "${ARR_CACHE_DIR}/${1}_tracked_cache.json"; } # every write, so it's never more than one write-cycle stale (2026-07-17). Exists purely so a # reboot doesn't leave the tmpfs cache genuinely empty until the next live fetch completes — # not a source of truth in its own right, just what tmpfs gets restored from when missing. -arr_cache_backup_file() { echo "${DATA_DIR}/${1}_tracked_cache.json"; } +arr_cache_backup_file() { echo "${ARR_CACHE_BACKUP_DIR}/${1}_tracked_cache.json"; } arr_rescan_duration_db() { echo "${DATA_DIR}/${1}_rescan_duration.db"; }