Put everything Varaverk persists under one root, state included
This commit is contained in:
Gmer4Lfe
Executable
+243
@@ -0,0 +1,243 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ============================== DATA LAYOUT MIGRATION =========================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# One-time move of everything Varaverk persists into a single rooted tree under DATA_DIR.
|
||||
#
|
||||
# State_Files/ → data/state/
|
||||
# data/*.db|.count|.tsv|.list → data/db/
|
||||
# data/ai_* → data/ai/
|
||||
# data/*_tracked_cache.json → data/cache/arr/
|
||||
# data/*.log → data/logs/
|
||||
# SCRIPTS_DIR/.cache/vv/d/ → data/cache/conf/
|
||||
#
|
||||
# ==============================================================================================
|
||||
# WHY THIS EXISTS SEPARATELY FROM conf_upgrade
|
||||
# ==============================================================================================
|
||||
#
|
||||
# conf_upgrade adds keys the template has and the installation does not; it never rewrites a
|
||||
# value the operator already has, which is exactly the behaviour you want from it and exactly
|
||||
# why it cannot perform this migration. The paths being moved are existing keys — STATE_DIR,
|
||||
# BANDWIDTH_LOG, AI_INDEX_DB and two dozen more — so their values would keep pointing at the old
|
||||
# layout forever while the new directory variables sat beside them unused.
|
||||
#
|
||||
# So this rewrites those values, then moves the files to match. Both halves, or neither: a conf
|
||||
# pointing at a directory the data is not in is worse than not having started.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Idempotent
|
||||
# Every step tests before acting. A second run reports "already migrated" and changes nothing,
|
||||
# which matters because the natural instinct after a partial failure is to run it again.
|
||||
#
|
||||
# Moves, never copies-and-deletes
|
||||
# mv within one filesystem is atomic per file, so a reader either sees the file at the old
|
||||
# path or the new one — never a half-written copy at both. Nothing is deleted; if a file
|
||||
# cannot be moved it is reported and left exactly where it is.
|
||||
#
|
||||
# Conf is backed up before it is rewritten
|
||||
# master.conf.bak-<stamp>, next to the original, same convention conf_upgrade uses.
|
||||
#
|
||||
# Refuses to run while the orchestrators might be writing
|
||||
# A watchdog that sourced conf before the rewrite and writes state after the move would put a
|
||||
# file back at the old path. The window is seconds and the damage is one stale file, but the
|
||||
# check costs nothing and the failure is silent otherwise.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# migrate_data_layout.sh --dry-run Show what would move. Changes nothing. Do this first.
|
||||
# migrate_data_layout.sh Perform the migration.
|
||||
# migrate_data_layout.sh --force Skip the running-orchestrator check.
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
CONF="$ROOT/Configurations/master.conf"
|
||||
|
||||
DRY_RUN=false
|
||||
FORCE=false
|
||||
for a in "$@"; do
|
||||
case "$a" in
|
||||
--dry-run) DRY_RUN=true ;;
|
||||
--force) FORCE=true ;;
|
||||
*) echo "Unknown argument: $a" >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
[[ -f "$CONF" ]] || { echo "[FATAL] master.conf not found at $CONF" >&2; exit 1; }
|
||||
|
||||
# Resolve the roots the same way load_config.sh will after this runs.
|
||||
SCRIPTS_DIR="$ROOT"
|
||||
DATA_DIR="$(grep -m1 -E '^\s*DATA_DIR=' "$CONF" | cut -d'"' -f2)"
|
||||
DATA_DIR="${DATA_DIR//\$\{SCRIPTS_DIR\}/$SCRIPTS_DIR}"
|
||||
DATA_DIR="${DATA_DIR:-$ROOT/data}"
|
||||
|
||||
OLD_STATE="$ROOT/State_Files"
|
||||
OLD_CONFCACHE="$ROOT/.cache/vv/d"
|
||||
|
||||
DB_DIR="$DATA_DIR/db"
|
||||
STATE_DIR="$DATA_DIR/state"
|
||||
AI_DATA_DIR="$DATA_DIR/ai"
|
||||
CACHE_BACKUP_DIR="$DATA_DIR/cache"
|
||||
ARR_CACHE_BACKUP_DIR="$CACHE_BACKUP_DIR/arr"
|
||||
CONF_CACHE_BACKUP_DIR="$CACHE_BACKUP_DIR/conf"
|
||||
LOG_ARCHIVE_DIR="$DATA_DIR/logs"
|
||||
|
||||
moved=0; skipped=0; failed=0
|
||||
|
||||
say() { printf ' %s\n' "$*"; }
|
||||
step() { printf '\n━━━ %s ━━━\n' "$*"; }
|
||||
|
||||
# ── Guard: orchestrators mid-run ──────────────────────────────────────────────
|
||||
if [[ "$FORCE" == false && "$DRY_RUN" == false ]]; then
|
||||
running=$(pgrep -fa 'Orchestrators/|Watchdogs/' 2>/dev/null | grep -v "$$" | grep -v migrate_data_layout || true)
|
||||
if [[ -n "$running" ]]; then
|
||||
echo "[ABORT] Orchestrator or watchdog is running — it may rewrite state mid-move:" >&2
|
||||
echo "$running" >&2
|
||||
echo "Wait for it to finish, or re-run with --force if you are sure." >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Move one path ─────────────────────────────────────────────────────────────
|
||||
move() {
|
||||
local src="$1" dstdir="$2" base
|
||||
base="$(basename "$src")"
|
||||
[[ -e "$src" ]] || return 0
|
||||
if [[ -e "$dstdir/$base" ]]; then
|
||||
say "skip $base — already at ${dstdir#$DATA_DIR/}/"
|
||||
((skipped++)); return 0
|
||||
fi
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
say "would $base → ${dstdir#$DATA_DIR/}/"
|
||||
((moved++)); return 0
|
||||
fi
|
||||
mkdir -p "$dstdir" 2>/dev/null
|
||||
if mv "$src" "$dstdir/$base" 2>/dev/null; then
|
||||
say "moved $base → ${dstdir#$DATA_DIR/}/"
|
||||
((moved++))
|
||||
else
|
||||
say "FAILED $base — left in place"
|
||||
((failed++))
|
||||
fi
|
||||
}
|
||||
|
||||
# ── 1. Rewrite the conf values conf_upgrade cannot ────────────────────────────
|
||||
step "Step 1: master.conf path values"
|
||||
if grep -q 'STATE_DIR="\${DATA_DIR}/state"' "$CONF"; then
|
||||
say "already migrated — no conf changes needed"
|
||||
else
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
cp "$CONF" "${CONF}.bak-$(date +%Y%m%d%H%M%S)"
|
||||
sed -i -E \
|
||||
-e 's|^(\s*STATE_DIR=)".*"|\1"${DATA_DIR}/state"|' \
|
||||
-e 's|^(\s*PERSISTENT_CONF_CACHE=)".*"|\1"${CACHE_BACKUP_DIR}/conf"|' \
|
||||
"$CONF"
|
||||
for v in ARR_SYNC_BLOCKLIST DOCKER_UPDATE_REBUILT_DAILY_FILE DOCKER_UPDATE_REBUILT_WEEKLY_FILE \
|
||||
WATCHDOG_CONTAINER_RESTART_LOG TUNING_MONITOR_LOG LIDARR_TRACKED_COUNT_FILE \
|
||||
LIDARR_RESCAN_DURATION_DB LIDARR_ART_MISS_CACHE LIDARR_DISCOVERY_HISTORY \
|
||||
SONARR_DISCOVERY_HISTORY RADARR_DISCOVERY_HISTORY SONARR_TRACKED_COUNT_FILE \
|
||||
CORRUPTION_SCAN_STATE_FILE CORRUPTION_SCAN_STRIKES_FILE RADARR_TRACKED_COUNT_FILE \
|
||||
TRANSCODE_DAILY_LOG BANDWIDTH_LOG ARR_CLEANUP_STATS ARR_RECOVERY_STATS \
|
||||
ARR_RECOVERY_FAILURE_COUNTS; do
|
||||
sed -i -E "s|^(\s*${v}=\")\\\$\{?DATA_DIR\}?/|\1\${DB_DIR}/|" "$CONF"
|
||||
done
|
||||
for v in AI_INDEX_DB AI_MEMORY_FILE AI_TOKEN_DB; do
|
||||
sed -i -E "s|^(\s*${v}=\")\\\$\{?DATA_DIR\}?/|\1\${AI_DATA_DIR}/|" "$CONF"
|
||||
done
|
||||
sed -i -E 's|^(\s*LIDARR_CACHE_FILE=")\$\{?DATA_DIR\}?/|\1${ARR_CACHE_BACKUP_DIR}/|' "$CONF"
|
||||
sed -i -E 's|^(\s*ZFS_REPORT_LOG=")\$\{?DATA_DIR\}?/|\1${LOG_ARCHIVE_DIR}/|' "$CONF"
|
||||
say "rewritten — backup kept beside it"
|
||||
else
|
||||
say "would rewrite STATE_DIR, PERSISTENT_CONF_CACHE and 25 file paths"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── 2. Build the tree ─────────────────────────────────────────────────────────
|
||||
step "Step 2: directory tree"
|
||||
for d in "$DB_DIR" "$STATE_DIR" "$AI_DATA_DIR" "$ARR_CACHE_BACKUP_DIR" "$LOG_ARCHIVE_DIR"; do
|
||||
if [[ -d "$d" ]]; then say "exists ${d#$DATA_DIR/}"
|
||||
elif [[ "$DRY_RUN" == true ]]; then say "would create ${d#$DATA_DIR/}"
|
||||
else mkdir -p "$d" && say "created ${d#$DATA_DIR/}"
|
||||
fi
|
||||
done
|
||||
# The conf cache carries partner credentials and keeps its restrictive mode.
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
mkdir -p "$CONF_CACHE_BACKUP_DIR" && chmod 700 "$CONF_CACHE_BACKUP_DIR"
|
||||
say "created cache/conf (0700)"
|
||||
fi
|
||||
|
||||
# ── 3. State files ────────────────────────────────────────────────────────────
|
||||
step "Step 3: State_Files → data/state"
|
||||
if [[ -d "$OLD_STATE" ]]; then
|
||||
shopt -s nullglob dotglob
|
||||
for f in "$OLD_STATE"/*; do move "$f" "$STATE_DIR"; done
|
||||
shopt -u nullglob dotglob
|
||||
if [[ "$DRY_RUN" == false && -d "$OLD_STATE" ]]; then
|
||||
rmdir "$OLD_STATE" 2>/dev/null && say "removed empty State_Files/" \
|
||||
|| say "State_Files/ not empty — left in place, inspect it"
|
||||
fi
|
||||
else
|
||||
say "no State_Files/ — nothing to do"
|
||||
fi
|
||||
|
||||
# ── 4. Sort the data root ─────────────────────────────────────────────────────
|
||||
step "Step 4: sort data/ into subfolders"
|
||||
classify() {
|
||||
local f="$1" base; base="$(basename "$f")"
|
||||
case "$base" in
|
||||
ai_*) move "$f" "$AI_DATA_DIR" ;;
|
||||
*_tracked_cache.json) move "$f" "$ARR_CACHE_BACKUP_DIR" ;;
|
||||
*.log) move "$f" "$LOG_ARCHIVE_DIR" ;;
|
||||
*.db|*.count|*.tsv|*.list|*.json) move "$f" "$DB_DIR" ;;
|
||||
*) say "leave $base — unclassified, left in data/" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# Two passes, sidecars first. A SQLite database is three files, and the -wal holds committed
|
||||
# transactions that have not been checkpointed into the .db yet. Move the .db first and any
|
||||
# process that opens it during the gap sees a database with no write-ahead log, creates a fresh
|
||||
# one at the old path, and everything still in the old -wal is lost when it is moved over the
|
||||
# top. Sidecars ahead of their base closes that ordering: the worst case becomes a database
|
||||
# opened without its log still sitting beside it, which SQLite handles.
|
||||
shopt -s nullglob
|
||||
for f in "$DATA_DIR"/*-wal "$DATA_DIR"/*-shm; do
|
||||
[[ -d "$f" ]] && continue
|
||||
classify "$f"
|
||||
done
|
||||
for f in "$DATA_DIR"/*; do
|
||||
[[ -d "$f" ]] && continue # subfolders are the destinations
|
||||
case "$(basename "$f")" in *-wal|*-shm) continue ;; esac
|
||||
classify "$f"
|
||||
done
|
||||
shopt -u nullglob
|
||||
|
||||
# ── 5. Conf cache backup ──────────────────────────────────────────────────────
|
||||
step "Step 5: conf cache backup"
|
||||
if [[ -d "$OLD_CONFCACHE" ]]; then
|
||||
shopt -s nullglob dotglob
|
||||
for f in "$OLD_CONFCACHE"/*; do move "$f" "$CONF_CACHE_BACKUP_DIR"; done
|
||||
shopt -u nullglob dotglob
|
||||
[[ "$DRY_RUN" == false ]] && rmdir "$OLD_CONFCACHE" "$ROOT/.cache/vv" "$ROOT/.cache" 2>/dev/null
|
||||
else
|
||||
say "no $OLD_CONFCACHE — nothing to do"
|
||||
fi
|
||||
|
||||
# ── Summary ───────────────────────────────────────────────────────────────────
|
||||
printf '\n━━━━━ SUMMARY ━━━━━\n'
|
||||
printf ' %-10s %s\n' "moved:" "$moved"
|
||||
printf ' %-10s %s\n' "skipped:" "$skipped"
|
||||
printf ' %-10s %s\n' "failed:" "$failed"
|
||||
[[ "$DRY_RUN" == true ]] && printf '\n DRY RUN — nothing was changed.\n'
|
||||
[[ "$failed" -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
Reference in New Issue
Block a user