diff --git a/Old_Arch_Still_Works/arr_cleanup.py b/Old_Arch_Still_Works/arr_cleanup.py new file mode 100644 index 0000000..8ba2e3f --- /dev/null +++ b/Old_Arch_Still_Works/arr_cleanup.py @@ -0,0 +1,872 @@ +#!/usr/bin/env python3 +# ============================================================================================== +# ================================= Arr Cleanup ================================================ +# ============================================================================================== +# +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Delete orphaned media files not tracked by an arr (Lidarr, Radarr, Sonarr, +# or any future arr). Queries the API for all tracked file paths, walks the +# library on disk, and removes anything untracked that is old enough to be +# past the import window. Triggers an Emby library clean after each deletion +# run so ghost entries disappear immediately. +# +# Called exclusively by arr_cleanup.sh, which sources shell config and exports +# all configuration as environment variables before exec'ing this script. +# +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== +# +# Every file encountered on disk is classified into one of five categories: +# +# TRACKED — arr API knows this exact path → leave it alone +# PROTECTED — matches {ARR}_PROTECTED_PATTERNS → never delete +# ORPHAN — media file, not tracked, older than {ARR}_ORPHAN_AGE → delete +# JUNK — not a tracked extension, not protected → delete regardless of age +# RECENT — not tracked, under {ARR}_ORPHAN_AGE → skip (may be mid-import) +# +# Arr apps generate cover art (*.jpg), metadata (*.nfo), lyrics (*.lrc), and +# subtitles (*.srt) but do NOT include these in their tracked file API response. +# Without PROTECTED classification these would be deleted — breaking the arr +# app and Emby display. +# +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# Seven gates — ALL must pass before any file is touched: +# 1. Container running and not starting/unhealthy +# 2. API reachable +# 3. API version matches {ARR}_VERSION_MAJOR in master.conf +# 4. Parent count > 0 (artists / movies / series) +# 5. Tracked file count > 0 +# 6. Tracked count >= {ARR}_MIN_TRACKED_PCT % of last known (if configured) +# 7. Deletion size < {ARR}_MAX_DELETE_GB — or --i-know-what-im-doing required +# +# ============================================================================================== +# ADDING A NEW ARR +# ============================================================================================== +# +# 1. Add an entry to ARR_PROFILES below (6 values — API endpoint pattern only) +# 2. Add HOST*__URL, API_KEY, MEDIA_ROOT, PATH_MAP to master_host*.conf +# 3. Add _ORPHAN_AGE, MAX_DELETE_GB, EXTENSIONS, etc. to master.conf +# 4. Add an export block to arr_cleanup.sh (copy existing block, change prefix) +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master_host*.conf (host-specific, aliased by detect_hosts in arr_cleanup.sh) +# +# HOST*_{ARR}_URL — arr base URL +# HOST*_{ARR}_API_KEY — arr API key +# HOST*_{ARR}_MEDIA_ROOT — host-side library root (MUSIC_ROOT / MOVIES_ROOT / TV_ROOT) +# HOST*_{ARR}_PATH_MAP — container path → host path translation (assoc array) +# +# master.conf (shared thresholds) +# +# {ARR}_ORPHAN_AGE — days before untracked file is eligible for deletion +# {ARR}_MAX_DELETE_GB — require --i-know-what-im-doing above this +# {ARR}_MIN_TRACKED_PCT — abort if tracked count drops below this % of last run +# {ARR}_TRACKED_COUNT_FILE — persistent baseline file path (enables gate 6) +# {ARR}_EXTENSIONS — media file extensions for orphan classification +# {ARR}_PROTECTED_PATTERNS — glob patterns never deleted (cover art, metadata, etc.) +# {ARR}_VERSION_MAJOR — expected arr major version for API safety check +# {ARR}_IMPORT_SCAN_TIMEOUT — seconds to wait for pre-flight import scan (default 600) +# {ARR}_LOCK_WARN_AGE — override default lock warning age (large libraries) +# ARR_CLEANUP_STATS — stats file path (read by sunday_morning_coffee_report) +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# arr_cleanup.sh --arr lidarr — normal run +# arr_cleanup.sh --arr lidarr --dry-run — preview, no deletions +# arr_cleanup.sh --arr lidarr --log — verbose output +# arr_cleanup.sh --arr lidarr --status — show config and exit +# arr_cleanup.sh --arr lidarr --i-know-what-im-doing — bypass size threshold +# arr_cleanup.sh --arr lidarr --i-know-what-im-doing --skip-strike-list — NUCLEAR MODE +# +# NUCLEAR MODE: both flags bypass age check AND size threshold. Use when the arr +# has filled gaps and you want a clean one-pass wipe. Flag name is long and +# annoying by design. +# +# ============================================================================================== + +import argparse +import atexit +import datetime +import fnmatch +import json +import os +import subprocess +import sys +import time +import urllib.error +import urllib.request +from pathlib import Path + +# ── Arr profiles — only what cannot come from env vars ──────────────────────── +# API endpoint patterns and labels. Everything else is config in master.conf. +ARR_PROFILES = { + "lidarr": { + "api_version": "v1", + "parent_endpoint": "artist", + "parent_id_param": "artistId", + "file_endpoint": "trackFile", + "import_scan_cmd": "DownloadedAlbumsScan", + "parent_label": "artists", + }, + "radarr": { + "api_version": "v3", + "parent_endpoint": "movie", + "parent_id_param": "movieId", + "file_endpoint": "moviefile", + "import_scan_cmd": "DownloadedMoviesScan", + "parent_label": "movies", + }, + "sonarr": { + "api_version": "v3", + "parent_endpoint": "series", + "parent_id_param": "seriesId", + "file_endpoint": "episodefile", + "import_scan_cmd": "DownloadedEpisodesScan", + "parent_label": "series", + }, + # Add new arrs here. 6 values — everything else goes in master.conf. + # "readarr": { + # "api_version": "v1", + # "parent_endpoint": "author", + # "parent_id_param": "authorId", + # "file_endpoint": "bookfile", + # "import_scan_cmd": "DownloadedBooksScan", + # "parent_label": "authors", + # }, +} + +# ── Output helpers ───────────────────────────────────────────────────────────── + +VERBOSE = False + +def log(msg): + if VERBOSE: + print(f" {msg}") + +def warn(msg): + print(f" ⚠️ {msg}") + +def error(msg): + print(f" ❌ {msg}", file=sys.stderr) + +def success(msg): + print(f" ✅ {msg}") + +def die(msg, notify_fn=None): + error(msg) + if notify_fn: + notify_fn(msg) + sys.exit(1) + +# ── Env var readers ──────────────────────────────────────────────────────────── + +def _env(key, default=""): + return os.environ.get(key, default) + +def _env_int(key, default=0): + try: + return int(os.environ.get(key, str(default))) + except ValueError: + return default + +def _env_float(key, default=0.0): + try: + return float(os.environ.get(key, str(default))) + except ValueError: + return default + +def _env_list(key): + val = os.environ.get(key, "") + return val.split() if val else [] + +def _env_bool(key): + return os.environ.get(key, "false").lower() == "true" + +# ── Lock ─────────────────────────────────────────────────────────────────────── + +LOCK_DIR = _env("LOCK_DIR", "/tmp/unraid_locks") +LOCK_TIMEOUT = _env_int("LOCK_WAIT_TIMEOUT", 30) +SCRIPT_NAME = "arr_cleanup" + +def acquire_lock(warn_age=3600): + os.makedirs(LOCK_DIR, exist_ok=True) + lockfile = Path(LOCK_DIR) / f"{SCRIPT_NAME}.lock" + + if lockfile.exists(): + try: + content = lockfile.read_text().strip() + pid_str, locked_name = content.split(":", 1) + pid = int(pid_str) + + try: + os.kill(pid, 0) + pid_alive = True + except (ProcessLookupError, PermissionError): + pid_alive = False + + if not pid_alive or locked_name != SCRIPT_NAME: + warn(f"Stale lock (PID {pid} gone) — clearing") + lockfile.unlink(missing_ok=True) + else: + age = time.time() - lockfile.stat().st_mtime + if age > warn_age: + warn(f"{SCRIPT_NAME} has been running for {int(age)}s — may be stuck (PID {pid})") + print(f" Another instance of {SCRIPT_NAME} is running — waiting up to {LOCK_TIMEOUT}s...") + waited = 0 + while lockfile.exists() and waited < LOCK_TIMEOUT: + time.sleep(1) + waited += 1 + if lockfile.exists(): + die(f"{SCRIPT_NAME} still locked after {LOCK_TIMEOUT}s — exiting") + except (ValueError, OSError): + lockfile.unlink(missing_ok=True) + + lockfile.write_text(f"{os.getpid()}:{SCRIPT_NAME}") + atexit.register(lambda: lockfile.unlink(missing_ok=True)) + log(f"🔏 Lock acquired: {SCRIPT_NAME} (PID {os.getpid()})") + +# ── HTTP helper ──────────────────────────────────────────────────────────────── + +def _http_get(url, api_key, timeout=30): + req = urllib.request.Request(url, headers={"X-Api-Key": api_key}) + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + return resp.status, resp.read().decode() + except urllib.error.HTTPError as e: + return e.code, "" + except Exception: + return 0, "" + +def _http_post(url, api_key, payload=None, timeout=30): + data = json.dumps(payload or {}).encode() + req = urllib.request.Request( + url, data=data, method="POST", + headers={"X-Api-Key": api_key, "Content-Type": "application/json"}, + ) + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + return resp.status, resp.read().decode() + except urllib.error.HTTPError as e: + return e.code, "" + except Exception: + return 0, "" + +# ── Arr API ──────────────────────────────────────────────────────────────────── + +def arr_get(base_url, api_key, api_version, endpoint): + url = f"{base_url}/api/{api_version}/{endpoint}" + status, body = _http_get(url, api_key) + if status != 200: + error(f"API HTTP {status} for: {endpoint}") + return None + try: + return json.loads(body) + except json.JSONDecodeError: + error(f"Failed to parse JSON for: {endpoint}") + return None + +# ── Path translation ─────────────────────────────────────────────────────────── +# Replicates common.sh translate_path() — longest-prefix match wins. + +def translate_path(api_path, path_map): + best_match = "" + best_len = 0 + for container_path, host_path in path_map.items(): + if api_path.startswith(container_path) and len(container_path) > best_len: + best_match = container_path + best_len = len(container_path) + if best_match: + return path_map[best_match] + api_path[best_len:] + return api_path + +# ── Container health check ───────────────────────────────────────────────────── + +def check_container(name, timeout=15): + def _inspect(fmt): + try: + r = subprocess.run( + ["timeout", str(timeout), "docker", "inspect", "-f", fmt, name], + capture_output=True, text=True, + ) + return r.stdout.strip() + except Exception: + return "" + + if _inspect("{{.State.Running}}") != "true": + return False, f"{name} is not running" + + health = _inspect("{{.State.Health.Status}}") + if health == "healthy": + log(f"{name} is healthy") + elif health == "": + log(f"{name} has no health check — proceeding") + elif health == "starting": + return False, f"{name} is still starting" + elif health == "unhealthy": + return False, f"{name} is unhealthy" + else: + warn(f"{name} health: {health} — proceeding with caution") + + return True, "" + +# ── API reachability ─────────────────────────────────────────────────────────── + +def check_api(url, label, timeout=10): + req = urllib.request.Request(url) + try: + urllib.request.urlopen(req, timeout=timeout) + log(f"{label} API reachable: {url}") + return True + except Exception: + error(f"{label} API not reachable: {url}") + return False + +# ── API version check ────────────────────────────────────────────────────────── + +def check_arr_version(base_url, api_key, api_version, expected_major, label): + url = f"{base_url}/api/{api_version}/system/status" + status, body = _http_get(url, api_key, timeout=10) + if status != 200 or not body: + warn(f"{label} version check failed — proceeding without verification") + return True + try: + data = json.loads(body) + version = data.get("version", "") + major = version.split(".")[0] + if major == str(expected_major): + success(f"{label} version: {version} (major {major} — tested ✅)") + return True + else: + error(f"{label} version mismatch — running v{major}, tested against v{expected_major}") + error(f"API structure may have changed — update {label.upper()}_VERSION_MAJOR in master.conf after verifying") + return False + except Exception: + warn(f"{label} version check failed — could not parse response") + return True + +# ── Import scan (pre-flight) ─────────────────────────────────────────────────── + +def run_import_scan(base_url, api_key, api_version, scan_cmd, media_root, path_map, timeout=600, label="Arr"): + container_root = next( + (cp for cp, hp in path_map.items() if hp == media_root), + None, + ) + + if container_root: + log(f"Triggering {scan_cmd} on: {container_root}") + payload = {"name": scan_cmd, "path": container_root} + else: + log(f"No path map match — triggering {scan_cmd} (all root folders)") + payload = {"name": scan_cmd} + + url = f"{base_url}/api/{api_version}/command" + status, body = _http_post(url, api_key, payload) + if status not in (200, 201): + warn(f"Could not trigger import scan (HTTP {status}) — proceeding without pre-flight") + return + + try: + cmd_id = json.loads(body).get("id") + except Exception: + cmd_id = None + + if not cmd_id: + warn("Could not get scan command ID — proceeding without pre-flight") + return + + print(f" Import scan queued (command ID: {cmd_id}) — waiting for completion...") + polled = 0 + poll_url = f"{base_url}/api/{api_version}/command/{cmd_id}" + while polled < timeout: + _, body = _http_get(poll_url, api_key, timeout=10) + try: + state = json.loads(body).get("status", "") + except Exception: + state = "" + + if state == "completed": + log("Import scan complete ✅") + return + if state == "failed": + warn("Import scan reported failed — proceeding anyway") + return + + time.sleep(10) + polled += 10 + if polled % 60 == 0: + log(f" Still scanning... ({polled}s elapsed)") + + warn(f"Import scan timed out after {timeout}s — proceeding anyway") + +# ── Fetch tracked paths ──────────────────────────────────────────────────────── + +def fetch_tracked_paths(base_url, api_key, api_version, profile, path_map, label): + parent_endpoint = profile["parent_endpoint"] + parent_id_param = profile["parent_id_param"] + file_endpoint = profile["file_endpoint"] + parent_label = profile["parent_label"] + + parents = arr_get(base_url, api_key, api_version, parent_endpoint) + if parents is None: + return None, 0, 0 + + parent_ids = [p["id"] for p in parents] + parent_count = len(parent_ids) + if parent_count == 0: + return None, 0, 0 + + log(f"Found {parent_count} {parent_label} — fetching tracked files...") + + tracked = set() + for i, pid in enumerate(parent_ids): + if i > 0 and i % 100 == 0: + log(f"Fetching files: {i}/{parent_count} {parent_label}...") + files = arr_get(base_url, api_key, api_version, f"{file_endpoint}?{parent_id_param}={pid}") + if not files: + continue + if isinstance(files, dict): + files = [files] + for f in files: + api_path = f.get("path", "") + if api_path: + tracked.add(translate_path(api_path, path_map)) + + return tracked, parent_count, len(tracked) + +# ── File classification helpers ──────────────────────────────────────────────── + +def is_media_file(path, extensions): + ext = Path(path).suffix.lstrip(".").lower() + return ext in extensions + +def is_protected(path, patterns): + name = Path(path).name + return any(fnmatch.fnmatch(name, p) for p in patterns) + +# ── Notify Emby ─────────────────────────────────────────────────────────────── + +def notify_emby_scan(emby_url, emby_api_key, my_id): + if not emby_url or not emby_api_key: + log(f"Emby not configured on {my_id} — skipping library scan notification") + return + + log("Notifying Emby to clean missing files...") + status, body = _http_get(f"{emby_url}/ScheduledTasks", emby_api_key, timeout=15) + if status != 200 or not body: + warn(f"Could not reach Emby scheduled tasks API — skipping scan") + return + + try: + tasks = json.loads(body) + except Exception: + warn("Could not parse Emby tasks response") + return + + task_id = None + for task in tasks: + if "Clean Missing" in task.get("Name", ""): + task_id = task.get("Id") + break + if not task_id: + for task in tasks: + if "Scan Media Library" in task.get("Name", ""): + task_id = task.get("Id") + log("Clean Missing Files not found — using Scan Media Library") + break + + if not task_id: + warn("Could not find Emby Clean Missing Files or Scan Media Library task") + warn("Ghost entries will persist until next Emby scan") + return + + status, _ = _http_post(f"{emby_url}/ScheduledTasks/Running/{task_id}", emby_api_key) + if status in (200, 204): + warn("🎬 Emby Clean Missing Files triggered — ghost entries will be removed") + else: + warn(f"Emby task trigger returned HTTP {status} — ghost entries may persist") + +# ── Unraid + Discord notifications ──────────────────────────────────────────── + +def notify(msg, subject, notify_unraid, hostname, discord_webhook): + log(f"🔔 Notification: {subject} — {msg}") + if notify_unraid: + notify_script = "/usr/local/emhttp/plugins/dynamix/scripts/notify" + if os.path.isfile(notify_script) and os.access(notify_script, os.X_OK): + subprocess.run([notify_script, "-s", subject, "-d", msg, "-i", "warning"], + capture_output=True) + if discord_webhook: + payload = json.dumps({"content": f"🔔 **{subject}**\n{msg}"}).encode() + req = urllib.request.Request( + discord_webhook, data=payload, method="POST", + headers={"Content-Type": "application/json"}, + ) + try: + urllib.request.urlopen(req, timeout=10) + except Exception: + pass + +# ── Format helpers ───────────────────────────────────────────────────────────── + +def format_bytes(b): + if b > 1_073_741_824: + return f"{b / 1_073_741_824:.1f}GB" + if b > 1_048_576: + return f"{b / 1_048_576:.1f}MB" + return f"{b}B" + +def format_duration(secs): + if secs >= 3600: + return f"{secs // 3600}h{(secs % 3600) // 60}m{secs % 60}s" + if secs >= 60: + return f"{secs // 60}m{secs % 60}s" + return f"{secs}s" + +# ══════════════════════════════════════════════════════════════════════════════ +# MAIN +# ══════════════════════════════════════════════════════════════════════════════ + +def main(): + global VERBOSE + + parser = argparse.ArgumentParser(prog="arr_cleanup.py") + parser.add_argument("--arr", required=True, choices=list(ARR_PROFILES.keys())) + parser.add_argument("--dry-run", action="store_true") + parser.add_argument("--log", action="store_true") + parser.add_argument("--status", action="store_true") + parser.add_argument("--i-know-what-im-doing", action="store_true", dest="i_know") + parser.add_argument("--skip-strike-list", action="store_true", dest="skip_strikes") + args = parser.parse_args() + + VERBOSE = args.log + ARR = args.arr + PREFIX = ARR.upper() + profile = ARR_PROFILES[ARR] + arr_label = ARR.capitalize() + + # ── Read config from env ─────────────────────────────────────────────────── + url = _env(f"{PREFIX}_URL") + api_key = _env(f"{PREFIX}_API_KEY") + media_root = _env(f"{PREFIX}_MEDIA_ROOT") + extensions = set(_env_list(f"{PREFIX}_EXTENSIONS")) + protected = _env_list(f"{PREFIX}_PROTECTED_PATTERNS") + orphan_age = _env_int(f"{PREFIX}_ORPHAN_AGE", 3) + max_del_gb = _env_float(f"{PREFIX}_MAX_DELETE_GB", 10.0) + min_pct = _env_int(f"{PREFIX}_MIN_TRACKED_PCT", 0) + count_file = _env(f"{PREFIX}_TRACKED_COUNT_FILE") + ver_major = _env(f"{PREFIX}_VERSION_MAJOR", "0") + scan_tmout = _env_int(f"{PREFIX}_IMPORT_SCAN_TIMEOUT", 600) + lock_warn = _env_int(f"{PREFIX}_LOCK_WARN_AGE", 3600) + container = _env(f"{PREFIX}_CONTAINER") or arr_label + + path_map_json = _env(f"{PREFIX}_PATH_MAP_JSON", "{}") + try: + path_map = json.loads(path_map_json) + except json.JSONDecodeError: + path_map = {} + + my_id = _env("MY_ID", "HOST1") + server_name = _env("LOCAL_SERVER_NAME") + notify_unraid = _env_bool("NOTIFY_UNRAID") + emby_url = _env("EMBY_URL") + emby_api_key = _env("EMBY_API_KEY") + discord = _env("MY_DISCORD_WEBHOOK") + stats_file = _env("ARR_CLEANUP_STATS") + hostname = _env("ARR_HOSTNAME") + api_version = profile["api_version"] + + def _notify(msg, subject=f"{arr_label} Cleanup"): + notify(msg, subject, notify_unraid, hostname, discord) + + # ── Nuclear mode warning ─────────────────────────────────────────────────── + if args.i_know and args.skip_strikes and not args.dry_run: + print() + print("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━") + print("⚠️ WARNING — NUCLEAR MODE ACTIVE") + print("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━") + print(" Flags: --i-know-what-im-doing --skip-strike-list") + print(" Strike system: BYPASSED — deletes on first pass") + print(" Size threshold: BYPASSED — no GB limit") + print(" Data recovery: NOT POSSIBLE after deletion") + print() + print(" Review --dry-run output before proceeding.") + print(" You have 10 seconds to cancel (Ctrl+C)...") + print("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━") + time.sleep(10) + print(" Proceeding...") + print() + + # ── Setup ────────────────────────────────────────────────────────────────── + print() + print("━━━ ⚙️ Setup ━━━") + + if not url or not api_key: + log(f"{arr_label} not configured on {my_id} ({server_name}) — skipping") + sys.exit(0) + + if not media_root: + die(f"{PREFIX}_MEDIA_ROOT not set — check master_host*.conf", _notify) + + if not Path(media_root).is_dir(): + die(f"Media root not found: {media_root}", _notify) + + acquire_lock(warn_age=lock_warn) + + print(f" {my_id} ({server_name}) — {url}") + + if args.dry_run: warn("DRY RUN — no files will be deleted") + if args.i_know: warn("OVERRIDE — --i-know-what-im-doing active") + if args.skip_strikes: warn("OVERRIDE — --skip-strike-list active — age check bypassed") + + # ── Status ───────────────────────────────────────────────────────────────── + if args.status: + print() + print(f"━━━━━ 📋 STATUS ━━━━━") + print(f"⚙️ Identity: {my_id} ({server_name})") + print(f"⚙️ {arr_label} URL: {url}") + print(f"⚙️ Media root: {media_root}") + print(f"⏱️ Orphan age: {orphan_age} days") + print(f"⚙️ Max delete: {max_del_gb}GB (requires --i-know-what-im-doing)") + if min_pct: + print(f"⚙️ Min tracked %: {min_pct}%") + print(f"⚙️ {arr_label} ver: v{ver_major} expected") + print(f"⚙️ Extensions: {' '.join(sorted(extensions))}") + print(f"⚙️ Protected patterns: {' '.join(protected)}") + print(f"⚙️ Dry Run: {args.dry_run}") + print(f"⚙️ I know: {args.i_know}") + print(f"⚙️ Skip strikes: {args.skip_strikes}") + print("━━━━━━━━━━━━━━━━━━━━━━━") + sys.exit(0) + + # ── Safety gate 1 — container health ────────────────────────────────────── + print() + print("━━━ 🛡️ Safety Checks ━━━") + + ok, reason = check_container(container) + if not ok: + _notify(f"{arr_label} cleanup aborted on {hostname} — {reason}") + die(f"{reason} — aborting") + + log("Safety gate 1 passed — container healthy") + + # ── Pre-flight import scan ───────────────────────────────────────────────── + print() + print(f"━━━ 🔄 Pre-flight: {arr_label} Import Scan ━━━") + run_import_scan(url, api_key, api_version, + profile["import_scan_cmd"], media_root, path_map, + timeout=scan_tmout, label=arr_label) + + # ── Safety gate 2 — API reachability ────────────────────────────────────── + print() + print(f"━━━ 🔄 Fetching {arr_label} Tracked Files ━━━") + + if not check_api(url, arr_label): + _notify(f"{arr_label} cleanup aborted on {hostname} — API unreachable") + sys.exit(1) + + # ── Safety gate 3 — API version ─────────────────────────────────────────── + if not check_arr_version(url, api_key, api_version, ver_major, arr_label): + _notify(f"{arr_label} version mismatch on {hostname} — check master.conf") + sys.exit(1) + + # ── Fetch tracked paths ──────────────────────────────────────────────────── + print(f" Querying {arr_label} API...") + tracked, parent_count, tracked_count = fetch_tracked_paths( + url, api_key, api_version, profile, path_map, arr_label, + ) + + # ── Safety gate 4 — parent count > 0 ────────────────────────────────────── + if tracked is None or parent_count == 0: + msg = f"API returned 0 {profile['parent_label']} — aborting to prevent mass deletion" + _notify(f"{arr_label} cleanup aborted on {hostname} — 0 {profile['parent_label']} returned") + die(msg) + + # ── Safety gate 5 — tracked count > 0 ──────────────────────────────────── + if tracked_count == 0: + _notify(f"{arr_label} cleanup aborted on {hostname} — 0 tracked files returned") + die("API returned 0 tracked files — aborting to prevent mass deletion") + + print(f" {parent_count} {profile['parent_label']} | {tracked_count} tracked files") + log(f"Built in-memory lookup set: {tracked_count} tracked paths") + + # ── Safety gate 6 — tracked % drop (only if configured) ────────────────── + if count_file and min_pct > 0: + count_path = Path(count_file) + if count_path.exists(): + try: + last = int(count_path.read_text().strip()) + if last > 0: + pct = int((tracked_count / last) * 100) + if pct < min_pct: + error(f"Tracked count dropped to {pct}% of last run ({tracked_count} vs {last})") + error(f"Suggests API issue — aborting to prevent mass deletion") + error(f"If expected (large removal) delete: {count_file}") + _notify(f"{arr_label} cleanup aborted on {hostname} — tracked count dropped to {pct}%") + sys.exit(1) + log(f"Tracked count: {pct}% of last run ({tracked_count} vs {last}) ✅") + except (ValueError, OSError): + log("Could not read previous count — skipping % check") + else: + log("No previous count on record — first run, saving baseline") + try: + count_path.write_text(str(tracked_count)) + except OSError as e: + warn(f"Could not write tracked count file: {e}") + + # ── Scan media root ──────────────────────────────────────────────────────── + print() + print(f"━━━ 🧹 Scanning Media Root ━━━") + print(f" Root: {media_root} | Orphan age: {orphan_age} days") + print() + + start = time.time() + orphan_count = junk_count = recent_count = protected_count = 0 + orphan_bytes = junk_bytes = 0 + age_threshold = orphan_age * 86400 + now = time.time() + max_del_bytes = int(max_del_gb * 1_073_741_824) + + scan_roots = set(path_map.values()) | {media_root} + + all_files = [] + for root in scan_roots: + if Path(root).is_dir(): + for fp in Path(root).rglob("*"): + if fp.is_file(): + all_files.append(str(fp)) + all_files = sorted(set(all_files)) + + for filepath in all_files: + if filepath in tracked: + log(f"TRACKED: {filepath}") + continue + + if is_protected(filepath, protected): + log(f"🔰 PROTECTED: {filepath}") + protected_count += 1 + continue + + try: + st = Path(filepath).stat() + except OSError: + continue + + file_size = st.st_size + + if is_media_file(filepath, extensions): + file_age = now - st.st_mtime + if file_age < age_threshold and not args.skip_strikes: + log(f"RECENT (skipping): {filepath}") + recent_count += 1 + continue + warn(f"🗑️ ORPHAN: {filepath}") + orphan_count += 1 + orphan_bytes += file_size + else: + log(f"JUNK: {filepath}") + junk_count += 1 + junk_bytes += file_size + + total_del_bytes = orphan_bytes + junk_bytes + total_removed = orphan_count + junk_count + + # ── Safety gate 7 — deletion size threshold ─────────────────────────────── + if total_del_bytes > max_del_bytes: + total_human = format_bytes(total_del_bytes) + if not args.i_know: + print() + error(f"Deletion would exceed {max_del_gb}GB — {total_human} would be deleted") + error("Review ORPHAN lines above carefully before proceeding") + error("Rerun with: --i-know-what-im-doing") + error("To also bypass age check: add --skip-strike-list") + _notify(f"{arr_label} cleanup halted on {hostname} — {total_human} requires --i-know-what-im-doing") + sys.exit(1) + else: + warn(f"OVERRIDE — deletion is {total_human} — proceeding with --i-know-what-im-doing") + + # ── Execute deletions ────────────────────────────────────────────────────── + if not args.dry_run: + for filepath in all_files: + if filepath in tracked: + continue + if is_protected(filepath, protected): + continue + try: + st = Path(filepath).stat() + file_age = now - st.st_mtime + except OSError: + continue + + if is_media_file(filepath, extensions): + if file_age < age_threshold and not args.skip_strikes: + continue + + try: + Path(filepath).unlink() + except OSError as e: + error(f"Failed to delete: {filepath} — {e}") + + log("Cleaning up empty folders...") + for root in scan_roots: + if Path(root).is_dir(): + for d in sorted(Path(root).rglob("*"), key=lambda p: len(p.parts), reverse=True): + if d.is_dir(): + try: + d.rmdir() + except OSError: + pass + log("Empty folders removed") + + elapsed = int(time.time() - start) + + # ── Summary ──────────────────────────────────────────────────────────────── + orphan_human = format_bytes(orphan_bytes) + junk_human = format_bytes(junk_bytes) + + print() + print(f"━━━━━ 📋 {arr_label.upper()} CLEANUP SUMMARY ━━━━━") + print(f"🖥️ Identity: {my_id} ({server_name})") + print(f"🔄 Tracked: {tracked_count} files ({parent_count} {profile['parent_label']})") + print(f"🛡️ Protected: {protected_count} files (cover art, metadata)") + print(f"🗑️ Orphans: {orphan_count} files ({orphan_human})") + print(f"🗑️ Junk: {junk_count} files ({junk_human})") + print(f"⏭️ Recent skipped: {recent_count} files (under {orphan_age} days)") + print(f"⏱️ Duration: {format_duration(elapsed)}") + print() + + if args.dry_run: + warn("DRY RUN — no files deleted") + elif total_removed == 0: + success("Clean — nothing to remove") + else: + warn(f"🏁 Removed {total_removed} files (orphans: {orphan_human} junk: {junk_human})") + _notify( + f"{arr_label} cleanup on {hostname} — removed {total_removed} files " + f"(orphans: {orphan_human} junk: {junk_human})" + ) + notify_emby_scan(emby_url, emby_api_key, my_id) + + print("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━") + + # ── Write stats for coffee report ───────────────────────────────────────── + if not args.dry_run and stats_file: + today = datetime.date.today().strftime("%Y-%m-%d") + line = f"{today}|{ARR}|{orphan_count}|{orphan_bytes}|{junk_count}|{junk_bytes}|{recent_count}|{tracked_count}\n" + try: + with open(stats_file, "a") as f: + f.write(line) + except OSError: + pass + + +if __name__ == "__main__": + main() diff --git a/Old_Arch_Still_Works/arr_cleanup.sh b/Old_Arch_Still_Works/arr_cleanup.sh new file mode 100755 index 0000000..70911da --- /dev/null +++ b/Old_Arch_Still_Works/arr_cleanup.sh @@ -0,0 +1,114 @@ +#!/bin/bash +# ============================================================================================== +# ================================= Arr Cleanup Launcher ======================================= +# ============================================================================================== +# +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Thin bash launcher for arr_cleanup.py. Handles everything bash is uniquely +# suited for: sourcing shell config, detect_hosts(), exporting env vars. +# All logic lives in Python. +# +# USAGE +# ───────────────────────────────────────────────────────────────────────────── +# arr_cleanup.sh --arr lidarr [--dry-run] [--log] [--status] +# arr_cleanup.sh --arr radarr [--i-know-what-im-doing] [--skip-strike-list] +# arr_cleanup.sh --arr sonarr +# +# ADDING A NEW ARR +# ───────────────────────────────────────────────────────────────────────────── +# 1. Add HOST*__URL, API_KEY, MEDIA_ROOT, PATH_MAP to master_host*.conf +# 2. Add _ORPHAN_AGE, MAX_DELETE_GB, EXTENSIONS, etc. to master.conf +# 3. Add a profile entry to ARR_PROFILES in arr_cleanup.py (6 values) +# 4. Add an export block for the new arr below (copy Sonarr block, change prefix) +# +# ============================================================================================== + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "$SCRIPT_DIR/../load_config.sh" + +if [[ "$EUID" -ne 0 ]]; then + echo "ERROR: Must be run as root" >&2 + exit 1 +fi + +if ! command -v python3 >/dev/null 2>&1; then + echo "ERROR: python3 not found — required for arr_cleanup" >&2 + exit 1 +fi + +detect_hosts + +# ── Helper: serialize host-specific associative array to JSON ────────────────── +# Reads ${MY_ID}_${ARR_UPPER}_PATH_MAP and emits {"container_path":"host_path",...} +# Paths with double-quotes in names are not supported (not a real-world constraint). +_path_map_json() { + local arr_upper="$1" + local map_var="${MY_ID}_${arr_upper}_PATH_MAP" + local json="{" + local sep="" keys k v + + if ! declare -p "$map_var" 2>/dev/null | grep -q "declare -A"; then + echo "{}" + return + fi + + eval "keys=(\"\${!${map_var}[@]}\")" + for k in "${keys[@]}"; do + eval "v=\"\${${map_var}[\$k]}\"" + json+="${sep}\"${k}\":\"${v}\"" + sep="," + done + json+="}" + echo "$json" +} + +# ── Host identity ────────────────────────────────────────────────────────────── +export MY_ID LOCAL_SERVER_NAME +export ARR_HOSTNAME +ARR_HOSTNAME=$(hostname) + +# ── Notifications ────────────────────────────────────────────────────────────── +export NOTIFY_UNRAID +export EMBY_URL EMBY_API_KEY +export MY_DISCORD_WEBHOOK + +# ── Shared ───────────────────────────────────────────────────────────────────── +export ARR_CLEANUP_STATS +export LOCK_DIR LOCK_WAIT_TIMEOUT + +# ── Lidarr ───────────────────────────────────────────────────────────────────── +export LIDARR_URL LIDARR_API_KEY +export LIDARR_MEDIA_ROOT="${LIDARR_MUSIC_ROOT:-}" +export LIDARR_EXTENSIONS="${LIDARR_EXTENSIONS[*]:-}" +export LIDARR_PROTECTED_PATTERNS="${LIDARR_PROTECTED_PATTERNS[*]:-}" +export LIDARR_ORPHAN_AGE LIDARR_MAX_DELETE_GB +export LIDARR_MIN_TRACKED_PCT LIDARR_TRACKED_COUNT_FILE +export LIDARR_VERSION_MAJOR LIDARR_IMPORT_SCAN_TIMEOUT LIDARR_LOCK_WARN_AGE +export LIDARR_PATH_MAP_JSON +LIDARR_PATH_MAP_JSON=$(_path_map_json "LIDARR") + +# ── Radarr ───────────────────────────────────────────────────────────────────── +export RADARR_URL RADARR_API_KEY +export RADARR_MEDIA_ROOT="${RADARR_MOVIES_ROOT:-}" +export RADARR_EXTENSIONS="${RADARR_EXTENSIONS[*]:-}" +export RADARR_PROTECTED_PATTERNS="${RADARR_PROTECTED_PATTERNS[*]:-}" +export RADARR_ORPHAN_AGE RADARR_MAX_DELETE_GB +export RADARR_MIN_TRACKED_PCT RADARR_TRACKED_COUNT_FILE +export RADARR_VERSION_MAJOR RADARR_IMPORT_SCAN_TIMEOUT RADARR_LOCK_WARN_AGE +export RADARR_PATH_MAP_JSON +RADARR_PATH_MAP_JSON=$(_path_map_json "RADARR") + +# ── Sonarr ───────────────────────────────────────────────────────────────────── +export SONARR_URL SONARR_API_KEY +export SONARR_MEDIA_ROOT="${SONARR_TV_ROOT:-}" +export SONARR_EXTENSIONS="${SONARR_EXTENSIONS[*]:-}" +export SONARR_PROTECTED_PATTERNS="${SONARR_PROTECTED_PATTERNS[*]:-}" +export SONARR_ORPHAN_AGE SONARR_MAX_DELETE_GB +export SONARR_MIN_TRACKED_PCT SONARR_TRACKED_COUNT_FILE +export SONARR_VERSION_MAJOR SONARR_IMPORT_SCAN_TIMEOUT SONARR_LOCK_WARN_AGE +export SONARR_PATH_MAP_JSON +SONARR_PATH_MAP_JSON=$(_path_map_json "SONARR") + +exec python3 "$SCRIPT_DIR/arr_cleanup.py" "$@" diff --git a/Tools/continuous_scripts_status.sh b/Old_Arch_Still_Works/continuous_scripts_status.sh similarity index 93% rename from Tools/continuous_scripts_status.sh rename to Old_Arch_Still_Works/continuous_scripts_status.sh index ff7d433..9195016 100644 --- a/Tools/continuous_scripts_status.sh +++ b/Old_Arch_Still_Works/continuous_scripts_status.sh @@ -2,30 +2,44 @@ # ============================================================================================== # ========================= Continuous Scripts Status ========================================== # ============================================================================================== -# Live status dashboard for all continuously running scripts in the ecosystem. -# Run manually anytime — no schedule, no cron. # -# ── WHAT IT SHOWS ───────────────────────────────────────────────────────────────────────────── -# For each continuous script (system_watchdog, docker_watchdog, failover): -# Running state, PID, uptime, approximate cycle count -# Active strikes and skip list -# Recent restart history -# Live health snapshot +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Live status dashboard for all continuously running scripts in the ecosystem. +# Run manually at any time — no schedule, no cron. +# +# For each script shows: running state, PID, uptime, approximate cycle count, +# active strikes, skip list, recent restart history, and a live health snapshot. # # system_watchdog — rootfs, RAM, ZFS ARC, load, zombie count, CPU temp # docker_watchdog — running/stopped/unhealthy containers, required containers, -# monitored memory containers, recent restart history +# memory-monitored containers, recent restart history # failover — current state, tier status, remote Tailscale visibility # # If a script is mid-cycle, state files are read as-is — reflects last completed cycle. # -# ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── -# detect_hosts() sets MY_ID and aliases all HOST*_WATCHDOG_* arrays. -# Required containers and tier delays use MY_ID/REMOTE_ID correctly. +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# Host-Aware Output +# detect_hosts() sets MY_ID and aliases all HOST*_WATCHDOG_* arrays. +# Required containers and tier delays are shown for the correct host. +# +# Read-Only +# Reads state files and docker inspect output only — makes no changes to any +# running script, container, or state file. +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# continuous_scripts_status.sh +# Show the full dashboard for all continuous scripts. +# +# continuous_scripts_status.sh --log +# Verbose output with additional detail per script section. # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# continuous_scripts_status.sh — show dashboard -# continuous_scripts_status.sh --log — verbose output # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"