FailedProxy/Varaverk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Put everything Varaverk persists under one root, state included

Browse Source
This commit is contained in:
Gmer4Lfe
2026-08-08 23:26:46 -04:00
parent d5c36db531
commit f1603349cc
11 changed files with 420 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Deployment/master.conf.template
+55 -30
View File
@@ -115,12 +115,37 @@
# scripts, state, and data are all available before the array mounts.
# Both directories are created automatically if they don't exist.
#
# DATA_DIR — historical logs, statistics, discovery histories, blocklists
# STATE_DIR — runtime state files for all scripts (watchdogs, fallback, transcode, etc.)
# Requirement: ALL state files MUST use $STATE_DIR. No /tmp, no /boot/config root.
# DATA_DIR is the one on-disk root. Everything Varaverk persists lives under it, in a
# subdirectory named for what the files are. Move DATA_DIR and the whole tree follows.
#
# It used to be two roots plus two strays: DATA_DIR beside State_Files/ as siblings, with the
# conf-cache backup off in SCRIPTS_DIR/.cache/vv/d and the arr cache backups loose in DATA_DIR's
# root. Nothing was wrong with any one of those decisions; together they meant no single place
# answered "what does Varaverk keep on disk". State is data — it is the data that happens to
# describe right now — so it belongs under the same root as the rest.
#
# db/ — statistics, histories, counters, blocklists. Things that accumulate.
# state/ — runtime state for every script: watchdogs, fallback, transcode, setup.
# Requirement: ALL state files MUST use $STATE_DIR. No /tmp, no repo root.
# ai/ — the retrieval index, operator memory, token ledger, filed bugs, saved chats.
# cache/ — persistent backups of the tmpfs caches, and ONLY those. A file belongs here when
# losing it costs a re-fetch and nothing else; anything that is a source of truth
# belongs in db/ or state/.
# logs/ — retained log output. Live logging still goes to LOG_DIR (/var/log/varaverk).
#
# STATE_DIR keeps its name and changes only its value, which is why this restructure did not
# touch the 15 conf entries, 18 shell paths and 23 PHP paths that build on it.
#
# The tmpfs caches are NOT here and must not be moved here — see VV_CACHE_ROOT below. These are
# on flash; those are read every second by the WebGUI and rewritten by the hundred megabytes.
DATA_DIR="/boot/config/plugins/varaverk/data"
STATE_DIR="/boot/config/plugins/varaverk/State_Files"
PERSISTENT_CONF_CACHE="/boot/config/plugins/varaverk/.cache/vv/d"
DB_DIR="${DATA_DIR}/db"
STATE_DIR="${DATA_DIR}/state"
AI_DATA_DIR="${DATA_DIR}/ai"
CACHE_BACKUP_DIR="${DATA_DIR}/cache"
LOG_ARCHIVE_DIR="${DATA_DIR}/logs"
PERSISTENT_CONF_CACHE="${CACHE_BACKUP_DIR}/conf"
ARR_CACHE_BACKUP_DIR="${CACHE_BACKUP_DIR}/arr"
# ── Cache Roots ──
# Everything Varaverk keeps in RAM, under one root, defined once.
@@ -177,7 +202,7 @@
# Runs before rsync — all nodes agree on tracked library before files are transferred.
# Remote API keys are read live from each node's config.xml via SSH — never stored here.
ARR_SYNC_ENABLED=true
ARR_SYNC_BLOCKLIST="${DATA_DIR}/arr_sync_blocklist.tsv"
ARR_SYNC_BLOCKLIST="${DB_DIR}/arr_sync_blocklist.tsv"
ARR_SYNC_CONNECT_TIMEOUT=10 # seconds — SSH connect timeout per node
ARR_SYNC_API_TIMEOUT=60 # seconds — curl timeout for library fetches
DOCKER_APPDATA_BASE="/mnt/user/appdata"
@@ -505,8 +530,8 @@
# than restarting them a second time. A file older than DOCKER_UPDATE_REBUILT_STALE_HOURS is
# treated as untrustworthy (docker_update.sh likely didn't run, or didn't run recently) — deleted,
# and every container in that tier restarts normally, same as if the file never existed.
DOCKER_UPDATE_REBUILT_DAILY_FILE="$DATA_DIR/docker_update_rebuilt_daily.list"
DOCKER_UPDATE_REBUILT_WEEKLY_FILE="$DATA_DIR/docker_update_rebuilt_weekly.list"
DOCKER_UPDATE_REBUILT_DAILY_FILE="${DB_DIR}/docker_update_rebuilt_daily.list"
DOCKER_UPDATE_REBUILT_WEEKLY_FILE="${DB_DIR}/docker_update_rebuilt_weekly.list"
DOCKER_UPDATE_REBUILT_STALE_HOURS=12
# Shares synced during the weekly maintenance window — defined per host in host*.conf.
@@ -861,7 +886,7 @@
# Restart loop protection — prevents watchdog from endlessly restarting a broken container
WATCHDOG_CONTAINER_RESTART_LIMIT=3
WATCHDOG_CONTAINER_RESTART_WINDOW=1 # rolling window in hours
WATCHDOG_CONTAINER_RESTART_LOG="$DATA_DIR/container_restart_history.db"
WATCHDOG_CONTAINER_RESTART_LOG="${DB_DIR}/container_restart_history.db"
# Notification batching — one summary per cycle instead of one ping per event
WATCHDOG_BATCH_NOTIFY=true
@@ -937,7 +962,7 @@
# Read by sunday_morning_coffee_report.sh for weekly peak/avg/warning summary.
INOTIFY_WARN_PCT=80 # warn if inotify instances exceed this % of limit
PHP_FPM_WARN_PCT=80 # warn if php-fpm workers exceed this % of max_children
TUNING_MONITOR_LOG="$DATA_DIR/system_tuning_history.db"
TUNING_MONITOR_LOG="${DB_DIR}/system_tuning_history.db"
TUNING_LOG_RETENTION=30 # days before old entries are purged
# ━━━ Reboot ━━━
@@ -1111,14 +1136,14 @@
LIDARR_MAX_DELETE_GB=5 # require --i-know-what-im-doing if deletion exceeds this
LIDARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run
# protects against API returning partial data on a bad day
LIDARR_TRACKED_COUNT_FILE="$DATA_DIR/lidarr_tracked.count"
LIDARR_TRACKED_COUNT_FILE="${DB_DIR}/lidarr_tracked.count"
LIDARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan
# Lidarr tracked-data cache — shared by lidarr_cleanup.sh, lidarr_duplicate_artist_cleanup.sh,
# lidarr_missing_art.sh, lidarr_release_fixer.sh, and arr_cache_prefill.sh. See
# lidarr_get_tracked_data() in common.sh for the fresh/stale/rescan-active branching logic.
LIDARR_CACHE_FILE="$DATA_DIR/lidarr_tracked_cache.json"
LIDARR_RESCAN_DURATION_DB="$DATA_DIR/lidarr_rescan_duration.db"
LIDARR_CACHE_FILE="${ARR_CACHE_BACKUP_DIR}/lidarr_tracked_cache.json"
LIDARR_RESCAN_DURATION_DB="${DB_DIR}/lidarr_rescan_duration.db"
LIDARR_CACHE_MAX_AGE_DAYS=1 # force a live refresh (or rescan-aware wait) past this age
ARR_PREFILL_WAIT_MINUTES=10 # array-start prefill: how long to retry reaching each arr
LIDARR_EXTENSIONS=("flac" "mp3" "m4a" "wav" "aac" "ogg" "opus" "wma")
@@ -1140,7 +1165,7 @@
LIDARR_ART_RETRIES=2 # download retry attempts per image
LIDARR_ART_SLEEP_BETWEEN=0.2 # seconds between fanart.tv API calls
LIDARR_ART_RECHECK_DAYS=30 # days before re-querying art that upstream didn't have
LIDARR_ART_MISS_CACHE="${DATA_DIR}/lidarr_art_miss_cache.tsv" # negative cache — art upstream has never had
LIDARR_ART_MISS_CACHE="${DB_DIR}/lidarr_art_miss_cache.tsv" # negative cache — art upstream has never had
# HOST*_FANART_API_KEY / HOST*_LASTFM_API_KEY — set in host*.conf
# Lidarr discovery settings (playback_aware_lidarr_discovery.sh)
@@ -1150,7 +1175,7 @@
LIDARR_DISCOVERY_USER_CAP_PCT=35 # max % any single user can contribute to play score (prevents one listener dominating)
LIDARR_DISCOVERY_MAX_ADDS=5 # max artists to add per run — quality over bulk
LIDARR_DISCOVERY_REJECT_COOLDOWN=30 # days before re-evaluating a rejected artist
LIDARR_DISCOVERY_HISTORY="$DATA_DIR/lidarr_discovery_history.db"
LIDARR_DISCOVERY_HISTORY="${DB_DIR}/lidarr_discovery_history.db"
# Sonarr discovery settings (playback_aware_sonarr_discovery.sh)
SONARR_DISCOVERY_THRESHOLD=52 # score to accept candidate (0-100)
@@ -1162,7 +1187,7 @@
SONARR_DISCOVERY_REJECT_COOLDOWN=60 # days before re-evaluating a rejected show
SONARR_DISCOVERY_USER_EPISODE_CAP=8 # max episodes any one user contributes to seed volume score
SONARR_DISCOVERY_MONITOR_MODE="all" # Sonarr monitor mode on add: all | future | first | latest | none
SONARR_DISCOVERY_HISTORY="$DATA_DIR/sonarr_discovery_history.db"
SONARR_DISCOVERY_HISTORY="${DB_DIR}/sonarr_discovery_history.db"
# Radarr discovery shared settings
RADARR_DISCOVERY_THRESHOLD=52 # score to accept candidate (0-100) — lower than Lidarr since diverse seeds rarely overlap
@@ -1173,7 +1198,7 @@
RADARR_DISCOVERY_MIN_RATING=60 # min TMDB vote_average × 10 (60 = 6.0/10)
RADARR_DISCOVERY_REJECT_COOLDOWN=60 # days before re-evaluating a rejected movie
RADARR_DISCOVERY_SEED_LIBRARIES=("Movies") # Emby libraries to draw seed movies from
RADARR_DISCOVERY_HISTORY="$DATA_DIR/radarr_discovery_history.db"
RADARR_DISCOVERY_HISTORY="${DB_DIR}/radarr_discovery_history.db"
# Emby → arr sync library allowlists
# Only these Emby library names will be considered by the sync tools.
@@ -1186,14 +1211,14 @@
SONARR_MAX_DELETE_GB=10 # require --i-know-what-im-doing if deletion exceeds this
SONARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run
# protects against API returning partial data on a bad day
SONARR_TRACKED_COUNT_FILE="$DATA_DIR/sonarr_tracked.count"
SONARR_TRACKED_COUNT_FILE="${DB_DIR}/sonarr_tracked.count"
SONARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan
SONARR_MOVE_POLL_TIMEOUT=3600 # seconds to wait for a single async MoveSeries command to
# reach "completed" — generous because a large series can sit
# queued behind other moves already in progress, not just its
# own copy time
CORRUPTION_SCAN_STATE_FILE="${DATA_DIR}/corruption_scan_state.tsv" # clean-file skip-cache
CORRUPTION_SCAN_STRIKES_FILE="${DATA_DIR}/corruption_scan_strikes.tsv" # consecutive corrupt-detection counts, keyed by host path
CORRUPTION_SCAN_STATE_FILE="${DB_DIR}/corruption_scan_state.tsv" # clean-file skip-cache
CORRUPTION_SCAN_STRIKES_FILE="${DB_DIR}/corruption_scan_strikes.tsv" # consecutive corrupt-detection counts, keyed by host path
CORRUPTION_SCAN_STRIKE_LIMIT=2 # consecutive corrupt detections (across separate scan runs)
# required before --remediate deletes+re-searches — guards
# against a one-off ffprobe hiccup (mid-write file, NFS blip)
@@ -1224,7 +1249,7 @@
RADARR_MAX_DELETE_GB=30 # require --i-know-what-im-doing if deletion exceeds this
RADARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run
# protects against API returning partial data on a bad day
RADARR_TRACKED_COUNT_FILE="$DATA_DIR/radarr_tracked.count"
RADARR_TRACKED_COUNT_FILE="${DB_DIR}/radarr_tracked.count"
RADARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan
RADARR_MOVE_POLL_TIMEOUT=3600 # seconds to wait for a single async MoveMovie command to
# reach "completed" — mirrors SONARR_MOVE_POLL_TIMEOUT
@@ -1378,7 +1403,7 @@
# Daily statistics log — read by weekly_health_digest.sh for transcode summary.
TRANSCODE_STATE_FILE="$STATE_DIR/transcode_state.db"
TRANSCODE_DAILY_LOG="$DATA_DIR/transcode_daily.db"
TRANSCODE_DAILY_LOG="${DB_DIR}/transcode_daily.db"
TRANSCODE_LOG_RETENTION=90 # days before old entries purged
TRANSCODE_CHECK_EMBY=true
@@ -1423,7 +1448,7 @@
# ━━━ ZFS Memory Snapshot ━━━
# Weekly ZFS pool health and memory diagnostic report — informational only.
ZFS_REPORT_LOG="/var/log/zfs-weekly-health.log"
ZFS_REPORT_LOG="${LOG_ARCHIVE_DIR}/zfs-weekly-health.log"
ZFS_REPORT_ARC_WARN_PCT=90 # warn if ARC using more than this % of its max
ZFS_REPORT_ARC_FREE_WARN_GB=10 # warn if ARC headroom (max - current) drops below this GB
ZFS_REPORT_AVAIL_WARN_GB=20 # warn if less than this GB available on ZFS pool
@@ -1434,15 +1459,15 @@
# ━━━ Bandwidth Monitor ━━━
# Called automatically by rsync.sh after each sync — one bounded write per run.
# Tracks transfer size, duration and profile per sync for weekly summary reporting.
BANDWIDTH_LOG="$DATA_DIR/bandwidth_history.db"
BANDWIDTH_LOG="${DB_DIR}/bandwidth_history.db"
BANDWIDTH_LOG_RETENTION=90 # days before old entries purged
BANDWIDTH_WARN_GB=50 # flag syncs larger than this in weekly report
# Stats files — written by cleanup and recovery scripts, read by coffee report.
# All in DATA_DIR — array always running when these are written.
ARR_CLEANUP_STATS="$DATA_DIR/arr_cleanup_stats.db" # lidarr/sonarr/radarr orphan stats
ARR_RECOVERY_STATS="$DATA_DIR/arr_recovery_stats.db" # blocklist + re-search stats
ARR_RECOVERY_FAILURE_COUNTS="$DATA_DIR/arr_recovery_failure_counts.db" # per-item chronic-failure tracking
ARR_CLEANUP_STATS="${DB_DIR}/arr_cleanup_stats.db" # lidarr/sonarr/radarr orphan stats
ARR_RECOVERY_STATS="${DB_DIR}/arr_recovery_stats.db" # blocklist + re-search stats
ARR_RECOVERY_FAILURE_COUNTS="${DB_DIR}/arr_recovery_failure_counts.db" # per-item chronic-failure tracking
# ━━━ Health Digest ━━━
# Aggregated system health summary — reads existing state files, no new writes.
@@ -1658,7 +1683,7 @@
# gitignored, which is what makes it structurally impossible for a credential to reach the
# index: the files holding them were never in the repo. Do not "improve" this to a filesystem
# walk — an embedded secret cannot be rotated out of a vector.
AI_INDEX_DB="$DATA_DIR/ai_index.db"
AI_INDEX_DB="${AI_DATA_DIR}/ai_index.db"
AI_INDEX_BATCH=32 # chunks per embed request
# A pull is the only thing that changes tracked files on a server, so it is the only moment the
# index can go stale — and staleness is invisible in the answers, which keep citing the old
@@ -1680,7 +1705,7 @@
# The character cap is a context budget, not a style guide. At 16384 the retrieved passages,
# the model's reasoning and the conversation history are already competing; memory takes its
# share off the top of every single turn, so keep it short and factual.
AI_MEMORY_FILE="$DATA_DIR/ai_memory.md"
AI_MEMORY_FILE="${AI_DATA_DIR}/ai_memory.md"
AI_MEMORY_MAX_CHARS=4000 # ~1000 tokens — truncated with a notice if exceeded
# ━━━ AI Stored Conversations ━━━
@@ -1711,7 +1736,7 @@
#
# Retention is by row count rather than age: pruning is considered only when the file passes a
# size threshold, so an ordinary turn costs one stat() and an append.
AI_TOKEN_DB="$DATA_DIR/ai_token_history.db"
AI_TOKEN_DB="${AI_DATA_DIR}/ai_token_history.db"
AI_TOKEN_RETAIN_ROWS=20000 # oldest rows dropped past this — years of ordinary use
# AI/ai_token_sync.sh pulls each partner's ledger into the tmpfs cache the tab reads, so the
Deployment/migrate_data_layout.sh
Executable
+243
View File
@@ -0,0 +1,243 @@
#!/bin/bash
# ==============================================================================================
# ============================== DATA LAYOUT MIGRATION =========================================
# ==============================================================================================
#
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# One-time move of everything Varaverk persists into a single rooted tree under DATA_DIR.
#
# State_Files/ → data/state/
# data/*.db|.count|.tsv|.list → data/db/
# data/ai_* → data/ai/
# data/*_tracked_cache.json → data/cache/arr/
# data/*.log → data/logs/
# SCRIPTS_DIR/.cache/vv/d/ → data/cache/conf/
#
# ==============================================================================================
# WHY THIS EXISTS SEPARATELY FROM conf_upgrade
# ==============================================================================================
#
# conf_upgrade adds keys the template has and the installation does not; it never rewrites a
# value the operator already has, which is exactly the behaviour you want from it and exactly
# why it cannot perform this migration. The paths being moved are existing keys — STATE_DIR,
# BANDWIDTH_LOG, AI_INDEX_DB and two dozen more — so their values would keep pointing at the old
# layout forever while the new directory variables sat beside them unused.
#
# So this rewrites those values, then moves the files to match. Both halves, or neither: a conf
# pointing at a directory the data is not in is worse than not having started.
#
# ==============================================================================================
# OPERATIONAL SAFEGUARDS
# ==============================================================================================
#
# Idempotent
# Every step tests before acting. A second run reports "already migrated" and changes nothing,
# which matters because the natural instinct after a partial failure is to run it again.
#
# Moves, never copies-and-deletes
# mv within one filesystem is atomic per file, so a reader either sees the file at the old
# path or the new one — never a half-written copy at both. Nothing is deleted; if a file
# cannot be moved it is reported and left exactly where it is.
#
# Conf is backed up before it is rewritten
# master.conf.bak-<stamp>, next to the original, same convention conf_upgrade uses.
#
# Refuses to run while the orchestrators might be writing
# A watchdog that sourced conf before the rewrite and writes state after the move would put a
# file back at the old path. The window is seconds and the damage is one stale file, but the
# check costs nothing and the failure is silent otherwise.
#
# ==============================================================================================
# RUNTIME MODES
# ==============================================================================================
#
# migrate_data_layout.sh --dry-run Show what would move. Changes nothing. Do this first.
# migrate_data_layout.sh Perform the migration.
# migrate_data_layout.sh --force Skip the running-orchestrator check.
#
# ==============================================================================================
set -uo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
CONF="$ROOT/Configurations/master.conf"
DRY_RUN=false
FORCE=false
for a in "$@"; do
case "$a" in
--dry-run) DRY_RUN=true ;;
--force) FORCE=true ;;
*) echo "Unknown argument: $a" >&2; exit 2 ;;
esac
done
[[ -f "$CONF" ]] || { echo "[FATAL] master.conf not found at $CONF" >&2; exit 1; }
# Resolve the roots the same way load_config.sh will after this runs.
SCRIPTS_DIR="$ROOT"
DATA_DIR="$(grep -m1 -E '^\s*DATA_DIR=' "$CONF" | cut -d'"' -f2)"
DATA_DIR="${DATA_DIR//\$\{SCRIPTS_DIR\}/$SCRIPTS_DIR}"
DATA_DIR="${DATA_DIR:-$ROOT/data}"
OLD_STATE="$ROOT/State_Files"
OLD_CONFCACHE="$ROOT/.cache/vv/d"
DB_DIR="$DATA_DIR/db"
STATE_DIR="$DATA_DIR/state"
AI_DATA_DIR="$DATA_DIR/ai"
CACHE_BACKUP_DIR="$DATA_DIR/cache"
ARR_CACHE_BACKUP_DIR="$CACHE_BACKUP_DIR/arr"
CONF_CACHE_BACKUP_DIR="$CACHE_BACKUP_DIR/conf"
LOG_ARCHIVE_DIR="$DATA_DIR/logs"
moved=0; skipped=0; failed=0
say() { printf ' %s\n' "$*"; }
step() { printf '\n━━━ %s ━━━\n' "$*"; }
# ── Guard: orchestrators mid-run ──────────────────────────────────────────────
if [[ "$FORCE" == false && "$DRY_RUN" == false ]]; then
running=$(pgrep -fa 'Orchestrators/|Watchdogs/' 2>/dev/null | grep -v "$$" | grep -v migrate_data_layout || true)
if [[ -n "$running" ]]; then
echo "[ABORT] Orchestrator or watchdog is running — it may rewrite state mid-move:" >&2
echo "$running" >&2
echo "Wait for it to finish, or re-run with --force if you are sure." >&2
exit 1
fi
fi
# ── Move one path ─────────────────────────────────────────────────────────────
move() {
local src="$1" dstdir="$2" base
base="$(basename "$src")"
[[ -e "$src" ]] || return 0
if [[ -e "$dstdir/$base" ]]; then
say "skip $base — already at ${dstdir#$DATA_DIR/}/"
((skipped++)); return 0
fi
if [[ "$DRY_RUN" == true ]]; then
say "would $base → ${dstdir#$DATA_DIR/}/"
((moved++)); return 0
fi
mkdir -p "$dstdir" 2>/dev/null
if mv "$src" "$dstdir/$base" 2>/dev/null; then
say "moved $base → ${dstdir#$DATA_DIR/}/"
((moved++))
else
say "FAILED $base — left in place"
((failed++))
fi
}
# ── 1. Rewrite the conf values conf_upgrade cannot ────────────────────────────
step "Step 1: master.conf path values"
if grep -q 'STATE_DIR="\${DATA_DIR}/state"' "$CONF"; then
say "already migrated — no conf changes needed"
else
if [[ "$DRY_RUN" == false ]]; then
cp "$CONF" "${CONF}.bak-$(date +%Y%m%d%H%M%S)"
sed -i -E \
-e 's|^(\s*STATE_DIR=)".*"|\1"${DATA_DIR}/state"|' \
-e 's|^(\s*PERSISTENT_CONF_CACHE=)".*"|\1"${CACHE_BACKUP_DIR}/conf"|' \
"$CONF"
for v in ARR_SYNC_BLOCKLIST DOCKER_UPDATE_REBUILT_DAILY_FILE DOCKER_UPDATE_REBUILT_WEEKLY_FILE \
WATCHDOG_CONTAINER_RESTART_LOG TUNING_MONITOR_LOG LIDARR_TRACKED_COUNT_FILE \
LIDARR_RESCAN_DURATION_DB LIDARR_ART_MISS_CACHE LIDARR_DISCOVERY_HISTORY \
SONARR_DISCOVERY_HISTORY RADARR_DISCOVERY_HISTORY SONARR_TRACKED_COUNT_FILE \
CORRUPTION_SCAN_STATE_FILE CORRUPTION_SCAN_STRIKES_FILE RADARR_TRACKED_COUNT_FILE \
TRANSCODE_DAILY_LOG BANDWIDTH_LOG ARR_CLEANUP_STATS ARR_RECOVERY_STATS \
ARR_RECOVERY_FAILURE_COUNTS; do
sed -i -E "s|^(\s*${v}=\")\\\$\{?DATA_DIR\}?/|\1\${DB_DIR}/|" "$CONF"
done
for v in AI_INDEX_DB AI_MEMORY_FILE AI_TOKEN_DB; do
sed -i -E "s|^(\s*${v}=\")\\\$\{?DATA_DIR\}?/|\1\${AI_DATA_DIR}/|" "$CONF"
done
sed -i -E 's|^(\s*LIDARR_CACHE_FILE=")\$\{?DATA_DIR\}?/|\1${ARR_CACHE_BACKUP_DIR}/|' "$CONF"
sed -i -E 's|^(\s*ZFS_REPORT_LOG=")\$\{?DATA_DIR\}?/|\1${LOG_ARCHIVE_DIR}/|' "$CONF"
say "rewritten — backup kept beside it"
else
say "would rewrite STATE_DIR, PERSISTENT_CONF_CACHE and 25 file paths"
fi
fi
# ── 2. Build the tree ─────────────────────────────────────────────────────────
step "Step 2: directory tree"
for d in "$DB_DIR" "$STATE_DIR" "$AI_DATA_DIR" "$ARR_CACHE_BACKUP_DIR" "$LOG_ARCHIVE_DIR"; do
if [[ -d "$d" ]]; then say "exists ${d#$DATA_DIR/}"
elif [[ "$DRY_RUN" == true ]]; then say "would create ${d#$DATA_DIR/}"
else mkdir -p "$d" && say "created ${d#$DATA_DIR/}"
fi
done
# The conf cache carries partner credentials and keeps its restrictive mode.
if [[ "$DRY_RUN" == false ]]; then
mkdir -p "$CONF_CACHE_BACKUP_DIR" && chmod 700 "$CONF_CACHE_BACKUP_DIR"
say "created cache/conf (0700)"
fi
# ── 3. State files ────────────────────────────────────────────────────────────
step "Step 3: State_Files → data/state"
if [[ -d "$OLD_STATE" ]]; then
shopt -s nullglob dotglob
for f in "$OLD_STATE"/*; do move "$f" "$STATE_DIR"; done
shopt -u nullglob dotglob
if [[ "$DRY_RUN" == false && -d "$OLD_STATE" ]]; then
rmdir "$OLD_STATE" 2>/dev/null && say "removed empty State_Files/" \
|| say "State_Files/ not empty — left in place, inspect it"
fi
else
say "no State_Files/ — nothing to do"
fi
# ── 4. Sort the data root ─────────────────────────────────────────────────────
step "Step 4: sort data/ into subfolders"
classify() {
local f="$1" base; base="$(basename "$f")"
case "$base" in
ai_*) move "$f" "$AI_DATA_DIR" ;;
*_tracked_cache.json) move "$f" "$ARR_CACHE_BACKUP_DIR" ;;
*.log) move "$f" "$LOG_ARCHIVE_DIR" ;;
*.db|*.count|*.tsv|*.list|*.json) move "$f" "$DB_DIR" ;;
*) say "leave $base — unclassified, left in data/" ;;
esac
}
# Two passes, sidecars first. A SQLite database is three files, and the -wal holds committed
# transactions that have not been checkpointed into the .db yet. Move the .db first and any
# process that opens it during the gap sees a database with no write-ahead log, creates a fresh
# one at the old path, and everything still in the old -wal is lost when it is moved over the
# top. Sidecars ahead of their base closes that ordering: the worst case becomes a database
# opened without its log still sitting beside it, which SQLite handles.
shopt -s nullglob
for f in "$DATA_DIR"/*-wal "$DATA_DIR"/*-shm; do
[[ -d "$f" ]] && continue
classify "$f"
done
for f in "$DATA_DIR"/*; do
[[ -d "$f" ]] && continue # subfolders are the destinations
case "$(basename "$f")" in *-wal|*-shm) continue ;; esac
classify "$f"
done
shopt -u nullglob
# ── 5. Conf cache backup ──────────────────────────────────────────────────────
step "Step 5: conf cache backup"
if [[ -d "$OLD_CONFCACHE" ]]; then
shopt -s nullglob dotglob
for f in "$OLD_CONFCACHE"/*; do move "$f" "$CONF_CACHE_BACKUP_DIR"; done
shopt -u nullglob dotglob
[[ "$DRY_RUN" == false ]] && rmdir "$OLD_CONFCACHE" "$ROOT/.cache/vv" "$ROOT/.cache" 2>/dev/null
else
say "no $OLD_CONFCACHE — nothing to do"
fi
# ── Summary ───────────────────────────────────────────────────────────────────
printf '\n━━━━━ SUMMARY ━━━━━\n'
printf ' %-10s %s\n' "moved:" "$moved"
printf ' %-10s %s\n' "skipped:" "$skipped"
printf ' %-10s %s\n' "failed:" "$failed"
[[ "$DRY_RUN" == true ]] && printf '\n DRY RUN — nothing was changed.\n'
[[ "$failed" -gt 0 ]] && exit 1
exit 0