diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 22d54c7..0000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "files.exclude": { - "**/.cache/**": true, - "**/.next/**": true, - "**/build/**": true, - "**/coverage/**": true, - "**/dist/**": true, - "**/node_modules/**": true - } -} \ No newline at end of file diff --git a/Deployment/deploy.sh b/Deployment/deploy.sh deleted file mode 100755 index f6aa515..0000000 --- a/Deployment/deploy.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/bin/bash -# ============================================================================================== -# deploy.sh — Sync dev working tree → prod runtime for Plugin PHP and conf files. -# ============================================================================================== -# -# Direct call (interactive): ! bash Deployment/deploy.sh -# Deploys both Plugin and Configurations with full output. -# -# Hook call (Claude Code PostToolUse): -# Reads the tool JSON from stdin, deploys only the component that was actually changed. -# Plugin/ → rsync --delete into prod Plugin/unraid/ -# Configurations/ → conf_upgrade.sh (adds new keys, preserves prod values) -# -# ============================================================================================== - -DEV_ROOT="/mnt/cloud-storage/Important Shit/Git/Development/Varaverk" -PROD_ROOT="/boot/config/plugins/varaverk" -UPGRADE="$PROD_ROOT/Deployment/conf_upgrade.sh" -LOG="/tmp/vv_deploy.log" - -# ── Deploy functions ────────────────────────────────────────────────────────── - -deploy_plugin() { - rsync -a --delete --exclude='.git' \ - "$DEV_ROOT/Plugin/unraid/" \ - "$PROD_ROOT/Plugin/unraid/" -} - -deploy_conf() { - for conf in master.conf host1.conf host2.conf; do - tmpl="$DEV_ROOT/Configurations/$conf" - target="$PROD_ROOT/Configurations/$conf" - [[ -f "$tmpl" && -f "$target" ]] && \ - "$UPGRADE" --template "$tmpl" --target "$target" - done -} - -# ── Entry point ─────────────────────────────────────────────────────────────── - -# Read stdin (non-blocking — empty when called directly or via '!') -STDIN=$(cat 2>/dev/null || true) - -# Extract file_path from hook JSON if present -FILE=$(python3 -c " -import json, sys -try: - d = json.loads(sys.stdin.read()) - print(d.get('tool_input', {}).get('file_path', '')) -except: - print('') -" <<< "$STDIN" 2>/dev/null || echo "") - -if [[ -z "$FILE" ]]; then - # Direct call (interactive or via !) — deploy everything with output - echo "=== deploy: dev → prod ===" - echo "" - echo "── Plugin ──" - deploy_plugin && echo " rsync done" - echo "" - echo "── Configurations ──" - deploy_conf - echo "" - echo "=== done ===" - exit 0 -fi - -# Hook call — deploy only the affected component, log silently -{ - echo "[$(date '+%H:%M:%S')] hook: $FILE" - - if [[ "$FILE" == "$DEV_ROOT/Plugin/"* ]]; then - deploy_plugin && echo " plugin synced" - fi - - if [[ "$FILE" == "$DEV_ROOT/Configurations/"* ]]; then - deploy_conf && echo " conf upgraded" - fi -} >> "$LOG" 2>&1 - -exit 0 diff --git a/Old_Arch_Still_Works/arr_cleanup.py b/Old_Arch_Still_Works/arr_cleanup.py deleted file mode 100644 index 8ba2e3f..0000000 --- a/Old_Arch_Still_Works/arr_cleanup.py +++ /dev/null @@ -1,872 +0,0 @@ -#!/usr/bin/env python3 -# ============================================================================================== -# ================================= Arr Cleanup ================================================ -# ============================================================================================== -# -# PURPOSE -# ───────────────────────────────────────────────────────────────────────────── -# Delete orphaned media files not tracked by an arr (Lidarr, Radarr, Sonarr, -# or any future arr). Queries the API for all tracked file paths, walks the -# library on disk, and removes anything untracked that is old enough to be -# past the import window. Triggers an Emby library clean after each deletion -# run so ghost entries disappear immediately. -# -# Called exclusively by arr_cleanup.sh, which sources shell config and exports -# all configuration as environment variables before exec'ing this script. -# -# ============================================================================================== -# OPERATIONAL MODEL -# ============================================================================================== -# -# Every file encountered on disk is classified into one of five categories: -# -# TRACKED — arr API knows this exact path → leave it alone -# PROTECTED — matches {ARR}_PROTECTED_PATTERNS → never delete -# ORPHAN — media file, not tracked, older than {ARR}_ORPHAN_AGE → delete -# JUNK — not a tracked extension, not protected → delete regardless of age -# RECENT — not tracked, under {ARR}_ORPHAN_AGE → skip (may be mid-import) -# -# Arr apps generate cover art (*.jpg), metadata (*.nfo), lyrics (*.lrc), and -# subtitles (*.srt) but do NOT include these in their tracked file API response. -# Without PROTECTED classification these would be deleted — breaking the arr -# app and Emby display. -# -# ============================================================================================== -# OPERATIONAL SAFEGUARDS -# ============================================================================================== -# -# Seven gates — ALL must pass before any file is touched: -# 1. Container running and not starting/unhealthy -# 2. API reachable -# 3. API version matches {ARR}_VERSION_MAJOR in master.conf -# 4. Parent count > 0 (artists / movies / series) -# 5. Tracked file count > 0 -# 6. Tracked count >= {ARR}_MIN_TRACKED_PCT % of last known (if configured) -# 7. Deletion size < {ARR}_MAX_DELETE_GB — or --i-know-what-im-doing required -# -# ============================================================================================== -# ADDING A NEW ARR -# ============================================================================================== -# -# 1. Add an entry to ARR_PROFILES below (6 values — API endpoint pattern only) -# 2. Add HOST*__URL, API_KEY, MEDIA_ROOT, PATH_MAP to master_host*.conf -# 3. Add _ORPHAN_AGE, MAX_DELETE_GB, EXTENSIONS, etc. to master.conf -# 4. Add an export block to arr_cleanup.sh (copy existing block, change prefix) -# -# ============================================================================================== -# CONFIGURATION -# ============================================================================================== -# -# master_host*.conf (host-specific, aliased by detect_hosts in arr_cleanup.sh) -# -# HOST*_{ARR}_URL — arr base URL -# HOST*_{ARR}_API_KEY — arr API key -# HOST*_{ARR}_MEDIA_ROOT — host-side library root (MUSIC_ROOT / MOVIES_ROOT / TV_ROOT) -# HOST*_{ARR}_PATH_MAP — container path → host path translation (assoc array) -# -# master.conf (shared thresholds) -# -# {ARR}_ORPHAN_AGE — days before untracked file is eligible for deletion -# {ARR}_MAX_DELETE_GB — require --i-know-what-im-doing above this -# {ARR}_MIN_TRACKED_PCT — abort if tracked count drops below this % of last run -# {ARR}_TRACKED_COUNT_FILE — persistent baseline file path (enables gate 6) -# {ARR}_EXTENSIONS — media file extensions for orphan classification -# {ARR}_PROTECTED_PATTERNS — glob patterns never deleted (cover art, metadata, etc.) -# {ARR}_VERSION_MAJOR — expected arr major version for API safety check -# {ARR}_IMPORT_SCAN_TIMEOUT — seconds to wait for pre-flight import scan (default 600) -# {ARR}_LOCK_WARN_AGE — override default lock warning age (large libraries) -# ARR_CLEANUP_STATS — stats file path (read by sunday_morning_coffee_report) -# -# ============================================================================================== -# RUNTIME MODES -# ============================================================================================== -# -# arr_cleanup.sh --arr lidarr — normal run -# arr_cleanup.sh --arr lidarr --dry-run — preview, no deletions -# arr_cleanup.sh --arr lidarr --log — verbose output -# arr_cleanup.sh --arr lidarr --status — show config and exit -# arr_cleanup.sh --arr lidarr --i-know-what-im-doing — bypass size threshold -# arr_cleanup.sh --arr lidarr --i-know-what-im-doing --skip-strike-list — NUCLEAR MODE -# -# NUCLEAR MODE: both flags bypass age check AND size threshold. Use when the arr -# has filled gaps and you want a clean one-pass wipe. Flag name is long and -# annoying by design. -# -# ============================================================================================== - -import argparse -import atexit -import datetime -import fnmatch -import json -import os -import subprocess -import sys -import time -import urllib.error -import urllib.request -from pathlib import Path - -# ── Arr profiles — only what cannot come from env vars ──────────────────────── -# API endpoint patterns and labels. Everything else is config in master.conf. -ARR_PROFILES = { - "lidarr": { - "api_version": "v1", - "parent_endpoint": "artist", - "parent_id_param": "artistId", - "file_endpoint": "trackFile", - "import_scan_cmd": "DownloadedAlbumsScan", - "parent_label": "artists", - }, - "radarr": { - "api_version": "v3", - "parent_endpoint": "movie", - "parent_id_param": "movieId", - "file_endpoint": "moviefile", - "import_scan_cmd": "DownloadedMoviesScan", - "parent_label": "movies", - }, - "sonarr": { - "api_version": "v3", - "parent_endpoint": "series", - "parent_id_param": "seriesId", - "file_endpoint": "episodefile", - "import_scan_cmd": "DownloadedEpisodesScan", - "parent_label": "series", - }, - # Add new arrs here. 6 values — everything else goes in master.conf. - # "readarr": { - # "api_version": "v1", - # "parent_endpoint": "author", - # "parent_id_param": "authorId", - # "file_endpoint": "bookfile", - # "import_scan_cmd": "DownloadedBooksScan", - # "parent_label": "authors", - # }, -} - -# ── Output helpers ───────────────────────────────────────────────────────────── - -VERBOSE = False - -def log(msg): - if VERBOSE: - print(f" {msg}") - -def warn(msg): - print(f" ⚠️ {msg}") - -def error(msg): - print(f" ❌ {msg}", file=sys.stderr) - -def success(msg): - print(f" ✅ {msg}") - -def die(msg, notify_fn=None): - error(msg) - if notify_fn: - notify_fn(msg) - sys.exit(1) - -# ── Env var readers ──────────────────────────────────────────────────────────── - -def _env(key, default=""): - return os.environ.get(key, default) - -def _env_int(key, default=0): - try: - return int(os.environ.get(key, str(default))) - except ValueError: - return default - -def _env_float(key, default=0.0): - try: - return float(os.environ.get(key, str(default))) - except ValueError: - return default - -def _env_list(key): - val = os.environ.get(key, "") - return val.split() if val else [] - -def _env_bool(key): - return os.environ.get(key, "false").lower() == "true" - -# ── Lock ─────────────────────────────────────────────────────────────────────── - -LOCK_DIR = _env("LOCK_DIR", "/tmp/unraid_locks") -LOCK_TIMEOUT = _env_int("LOCK_WAIT_TIMEOUT", 30) -SCRIPT_NAME = "arr_cleanup" - -def acquire_lock(warn_age=3600): - os.makedirs(LOCK_DIR, exist_ok=True) - lockfile = Path(LOCK_DIR) / f"{SCRIPT_NAME}.lock" - - if lockfile.exists(): - try: - content = lockfile.read_text().strip() - pid_str, locked_name = content.split(":", 1) - pid = int(pid_str) - - try: - os.kill(pid, 0) - pid_alive = True - except (ProcessLookupError, PermissionError): - pid_alive = False - - if not pid_alive or locked_name != SCRIPT_NAME: - warn(f"Stale lock (PID {pid} gone) — clearing") - lockfile.unlink(missing_ok=True) - else: - age = time.time() - lockfile.stat().st_mtime - if age > warn_age: - warn(f"{SCRIPT_NAME} has been running for {int(age)}s — may be stuck (PID {pid})") - print(f" Another instance of {SCRIPT_NAME} is running — waiting up to {LOCK_TIMEOUT}s...") - waited = 0 - while lockfile.exists() and waited < LOCK_TIMEOUT: - time.sleep(1) - waited += 1 - if lockfile.exists(): - die(f"{SCRIPT_NAME} still locked after {LOCK_TIMEOUT}s — exiting") - except (ValueError, OSError): - lockfile.unlink(missing_ok=True) - - lockfile.write_text(f"{os.getpid()}:{SCRIPT_NAME}") - atexit.register(lambda: lockfile.unlink(missing_ok=True)) - log(f"🔏 Lock acquired: {SCRIPT_NAME} (PID {os.getpid()})") - -# ── HTTP helper ──────────────────────────────────────────────────────────────── - -def _http_get(url, api_key, timeout=30): - req = urllib.request.Request(url, headers={"X-Api-Key": api_key}) - try: - with urllib.request.urlopen(req, timeout=timeout) as resp: - return resp.status, resp.read().decode() - except urllib.error.HTTPError as e: - return e.code, "" - except Exception: - return 0, "" - -def _http_post(url, api_key, payload=None, timeout=30): - data = json.dumps(payload or {}).encode() - req = urllib.request.Request( - url, data=data, method="POST", - headers={"X-Api-Key": api_key, "Content-Type": "application/json"}, - ) - try: - with urllib.request.urlopen(req, timeout=timeout) as resp: - return resp.status, resp.read().decode() - except urllib.error.HTTPError as e: - return e.code, "" - except Exception: - return 0, "" - -# ── Arr API ──────────────────────────────────────────────────────────────────── - -def arr_get(base_url, api_key, api_version, endpoint): - url = f"{base_url}/api/{api_version}/{endpoint}" - status, body = _http_get(url, api_key) - if status != 200: - error(f"API HTTP {status} for: {endpoint}") - return None - try: - return json.loads(body) - except json.JSONDecodeError: - error(f"Failed to parse JSON for: {endpoint}") - return None - -# ── Path translation ─────────────────────────────────────────────────────────── -# Replicates common.sh translate_path() — longest-prefix match wins. - -def translate_path(api_path, path_map): - best_match = "" - best_len = 0 - for container_path, host_path in path_map.items(): - if api_path.startswith(container_path) and len(container_path) > best_len: - best_match = container_path - best_len = len(container_path) - if best_match: - return path_map[best_match] + api_path[best_len:] - return api_path - -# ── Container health check ───────────────────────────────────────────────────── - -def check_container(name, timeout=15): - def _inspect(fmt): - try: - r = subprocess.run( - ["timeout", str(timeout), "docker", "inspect", "-f", fmt, name], - capture_output=True, text=True, - ) - return r.stdout.strip() - except Exception: - return "" - - if _inspect("{{.State.Running}}") != "true": - return False, f"{name} is not running" - - health = _inspect("{{.State.Health.Status}}") - if health == "healthy": - log(f"{name} is healthy") - elif health == "": - log(f"{name} has no health check — proceeding") - elif health == "starting": - return False, f"{name} is still starting" - elif health == "unhealthy": - return False, f"{name} is unhealthy" - else: - warn(f"{name} health: {health} — proceeding with caution") - - return True, "" - -# ── API reachability ─────────────────────────────────────────────────────────── - -def check_api(url, label, timeout=10): - req = urllib.request.Request(url) - try: - urllib.request.urlopen(req, timeout=timeout) - log(f"{label} API reachable: {url}") - return True - except Exception: - error(f"{label} API not reachable: {url}") - return False - -# ── API version check ────────────────────────────────────────────────────────── - -def check_arr_version(base_url, api_key, api_version, expected_major, label): - url = f"{base_url}/api/{api_version}/system/status" - status, body = _http_get(url, api_key, timeout=10) - if status != 200 or not body: - warn(f"{label} version check failed — proceeding without verification") - return True - try: - data = json.loads(body) - version = data.get("version", "") - major = version.split(".")[0] - if major == str(expected_major): - success(f"{label} version: {version} (major {major} — tested ✅)") - return True - else: - error(f"{label} version mismatch — running v{major}, tested against v{expected_major}") - error(f"API structure may have changed — update {label.upper()}_VERSION_MAJOR in master.conf after verifying") - return False - except Exception: - warn(f"{label} version check failed — could not parse response") - return True - -# ── Import scan (pre-flight) ─────────────────────────────────────────────────── - -def run_import_scan(base_url, api_key, api_version, scan_cmd, media_root, path_map, timeout=600, label="Arr"): - container_root = next( - (cp for cp, hp in path_map.items() if hp == media_root), - None, - ) - - if container_root: - log(f"Triggering {scan_cmd} on: {container_root}") - payload = {"name": scan_cmd, "path": container_root} - else: - log(f"No path map match — triggering {scan_cmd} (all root folders)") - payload = {"name": scan_cmd} - - url = f"{base_url}/api/{api_version}/command" - status, body = _http_post(url, api_key, payload) - if status not in (200, 201): - warn(f"Could not trigger import scan (HTTP {status}) — proceeding without pre-flight") - return - - try: - cmd_id = json.loads(body).get("id") - except Exception: - cmd_id = None - - if not cmd_id: - warn("Could not get scan command ID — proceeding without pre-flight") - return - - print(f" Import scan queued (command ID: {cmd_id}) — waiting for completion...") - polled = 0 - poll_url = f"{base_url}/api/{api_version}/command/{cmd_id}" - while polled < timeout: - _, body = _http_get(poll_url, api_key, timeout=10) - try: - state = json.loads(body).get("status", "") - except Exception: - state = "" - - if state == "completed": - log("Import scan complete ✅") - return - if state == "failed": - warn("Import scan reported failed — proceeding anyway") - return - - time.sleep(10) - polled += 10 - if polled % 60 == 0: - log(f" Still scanning... ({polled}s elapsed)") - - warn(f"Import scan timed out after {timeout}s — proceeding anyway") - -# ── Fetch tracked paths ──────────────────────────────────────────────────────── - -def fetch_tracked_paths(base_url, api_key, api_version, profile, path_map, label): - parent_endpoint = profile["parent_endpoint"] - parent_id_param = profile["parent_id_param"] - file_endpoint = profile["file_endpoint"] - parent_label = profile["parent_label"] - - parents = arr_get(base_url, api_key, api_version, parent_endpoint) - if parents is None: - return None, 0, 0 - - parent_ids = [p["id"] for p in parents] - parent_count = len(parent_ids) - if parent_count == 0: - return None, 0, 0 - - log(f"Found {parent_count} {parent_label} — fetching tracked files...") - - tracked = set() - for i, pid in enumerate(parent_ids): - if i > 0 and i % 100 == 0: - log(f"Fetching files: {i}/{parent_count} {parent_label}...") - files = arr_get(base_url, api_key, api_version, f"{file_endpoint}?{parent_id_param}={pid}") - if not files: - continue - if isinstance(files, dict): - files = [files] - for f in files: - api_path = f.get("path", "") - if api_path: - tracked.add(translate_path(api_path, path_map)) - - return tracked, parent_count, len(tracked) - -# ── File classification helpers ──────────────────────────────────────────────── - -def is_media_file(path, extensions): - ext = Path(path).suffix.lstrip(".").lower() - return ext in extensions - -def is_protected(path, patterns): - name = Path(path).name - return any(fnmatch.fnmatch(name, p) for p in patterns) - -# ── Notify Emby ─────────────────────────────────────────────────────────────── - -def notify_emby_scan(emby_url, emby_api_key, my_id): - if not emby_url or not emby_api_key: - log(f"Emby not configured on {my_id} — skipping library scan notification") - return - - log("Notifying Emby to clean missing files...") - status, body = _http_get(f"{emby_url}/ScheduledTasks", emby_api_key, timeout=15) - if status != 200 or not body: - warn(f"Could not reach Emby scheduled tasks API — skipping scan") - return - - try: - tasks = json.loads(body) - except Exception: - warn("Could not parse Emby tasks response") - return - - task_id = None - for task in tasks: - if "Clean Missing" in task.get("Name", ""): - task_id = task.get("Id") - break - if not task_id: - for task in tasks: - if "Scan Media Library" in task.get("Name", ""): - task_id = task.get("Id") - log("Clean Missing Files not found — using Scan Media Library") - break - - if not task_id: - warn("Could not find Emby Clean Missing Files or Scan Media Library task") - warn("Ghost entries will persist until next Emby scan") - return - - status, _ = _http_post(f"{emby_url}/ScheduledTasks/Running/{task_id}", emby_api_key) - if status in (200, 204): - warn("🎬 Emby Clean Missing Files triggered — ghost entries will be removed") - else: - warn(f"Emby task trigger returned HTTP {status} — ghost entries may persist") - -# ── Unraid + Discord notifications ──────────────────────────────────────────── - -def notify(msg, subject, notify_unraid, hostname, discord_webhook): - log(f"🔔 Notification: {subject} — {msg}") - if notify_unraid: - notify_script = "/usr/local/emhttp/plugins/dynamix/scripts/notify" - if os.path.isfile(notify_script) and os.access(notify_script, os.X_OK): - subprocess.run([notify_script, "-s", subject, "-d", msg, "-i", "warning"], - capture_output=True) - if discord_webhook: - payload = json.dumps({"content": f"🔔 **{subject}**\n{msg}"}).encode() - req = urllib.request.Request( - discord_webhook, data=payload, method="POST", - headers={"Content-Type": "application/json"}, - ) - try: - urllib.request.urlopen(req, timeout=10) - except Exception: - pass - -# ── Format helpers ───────────────────────────────────────────────────────────── - -def format_bytes(b): - if b > 1_073_741_824: - return f"{b / 1_073_741_824:.1f}GB" - if b > 1_048_576: - return f"{b / 1_048_576:.1f}MB" - return f"{b}B" - -def format_duration(secs): - if secs >= 3600: - return f"{secs // 3600}h{(secs % 3600) // 60}m{secs % 60}s" - if secs >= 60: - return f"{secs // 60}m{secs % 60}s" - return f"{secs}s" - -# ══════════════════════════════════════════════════════════════════════════════ -# MAIN -# ══════════════════════════════════════════════════════════════════════════════ - -def main(): - global VERBOSE - - parser = argparse.ArgumentParser(prog="arr_cleanup.py") - parser.add_argument("--arr", required=True, choices=list(ARR_PROFILES.keys())) - parser.add_argument("--dry-run", action="store_true") - parser.add_argument("--log", action="store_true") - parser.add_argument("--status", action="store_true") - parser.add_argument("--i-know-what-im-doing", action="store_true", dest="i_know") - parser.add_argument("--skip-strike-list", action="store_true", dest="skip_strikes") - args = parser.parse_args() - - VERBOSE = args.log - ARR = args.arr - PREFIX = ARR.upper() - profile = ARR_PROFILES[ARR] - arr_label = ARR.capitalize() - - # ── Read config from env ─────────────────────────────────────────────────── - url = _env(f"{PREFIX}_URL") - api_key = _env(f"{PREFIX}_API_KEY") - media_root = _env(f"{PREFIX}_MEDIA_ROOT") - extensions = set(_env_list(f"{PREFIX}_EXTENSIONS")) - protected = _env_list(f"{PREFIX}_PROTECTED_PATTERNS") - orphan_age = _env_int(f"{PREFIX}_ORPHAN_AGE", 3) - max_del_gb = _env_float(f"{PREFIX}_MAX_DELETE_GB", 10.0) - min_pct = _env_int(f"{PREFIX}_MIN_TRACKED_PCT", 0) - count_file = _env(f"{PREFIX}_TRACKED_COUNT_FILE") - ver_major = _env(f"{PREFIX}_VERSION_MAJOR", "0") - scan_tmout = _env_int(f"{PREFIX}_IMPORT_SCAN_TIMEOUT", 600) - lock_warn = _env_int(f"{PREFIX}_LOCK_WARN_AGE", 3600) - container = _env(f"{PREFIX}_CONTAINER") or arr_label - - path_map_json = _env(f"{PREFIX}_PATH_MAP_JSON", "{}") - try: - path_map = json.loads(path_map_json) - except json.JSONDecodeError: - path_map = {} - - my_id = _env("MY_ID", "HOST1") - server_name = _env("LOCAL_SERVER_NAME") - notify_unraid = _env_bool("NOTIFY_UNRAID") - emby_url = _env("EMBY_URL") - emby_api_key = _env("EMBY_API_KEY") - discord = _env("MY_DISCORD_WEBHOOK") - stats_file = _env("ARR_CLEANUP_STATS") - hostname = _env("ARR_HOSTNAME") - api_version = profile["api_version"] - - def _notify(msg, subject=f"{arr_label} Cleanup"): - notify(msg, subject, notify_unraid, hostname, discord) - - # ── Nuclear mode warning ─────────────────────────────────────────────────── - if args.i_know and args.skip_strikes and not args.dry_run: - print() - print("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━") - print("⚠️ WARNING — NUCLEAR MODE ACTIVE") - print("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━") - print(" Flags: --i-know-what-im-doing --skip-strike-list") - print(" Strike system: BYPASSED — deletes on first pass") - print(" Size threshold: BYPASSED — no GB limit") - print(" Data recovery: NOT POSSIBLE after deletion") - print() - print(" Review --dry-run output before proceeding.") - print(" You have 10 seconds to cancel (Ctrl+C)...") - print("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━") - time.sleep(10) - print(" Proceeding...") - print() - - # ── Setup ────────────────────────────────────────────────────────────────── - print() - print("━━━ ⚙️ Setup ━━━") - - if not url or not api_key: - log(f"{arr_label} not configured on {my_id} ({server_name}) — skipping") - sys.exit(0) - - if not media_root: - die(f"{PREFIX}_MEDIA_ROOT not set — check master_host*.conf", _notify) - - if not Path(media_root).is_dir(): - die(f"Media root not found: {media_root}", _notify) - - acquire_lock(warn_age=lock_warn) - - print(f" {my_id} ({server_name}) — {url}") - - if args.dry_run: warn("DRY RUN — no files will be deleted") - if args.i_know: warn("OVERRIDE — --i-know-what-im-doing active") - if args.skip_strikes: warn("OVERRIDE — --skip-strike-list active — age check bypassed") - - # ── Status ───────────────────────────────────────────────────────────────── - if args.status: - print() - print(f"━━━━━ 📋 STATUS ━━━━━") - print(f"⚙️ Identity: {my_id} ({server_name})") - print(f"⚙️ {arr_label} URL: {url}") - print(f"⚙️ Media root: {media_root}") - print(f"⏱️ Orphan age: {orphan_age} days") - print(f"⚙️ Max delete: {max_del_gb}GB (requires --i-know-what-im-doing)") - if min_pct: - print(f"⚙️ Min tracked %: {min_pct}%") - print(f"⚙️ {arr_label} ver: v{ver_major} expected") - print(f"⚙️ Extensions: {' '.join(sorted(extensions))}") - print(f"⚙️ Protected patterns: {' '.join(protected)}") - print(f"⚙️ Dry Run: {args.dry_run}") - print(f"⚙️ I know: {args.i_know}") - print(f"⚙️ Skip strikes: {args.skip_strikes}") - print("━━━━━━━━━━━━━━━━━━━━━━━") - sys.exit(0) - - # ── Safety gate 1 — container health ────────────────────────────────────── - print() - print("━━━ 🛡️ Safety Checks ━━━") - - ok, reason = check_container(container) - if not ok: - _notify(f"{arr_label} cleanup aborted on {hostname} — {reason}") - die(f"{reason} — aborting") - - log("Safety gate 1 passed — container healthy") - - # ── Pre-flight import scan ───────────────────────────────────────────────── - print() - print(f"━━━ 🔄 Pre-flight: {arr_label} Import Scan ━━━") - run_import_scan(url, api_key, api_version, - profile["import_scan_cmd"], media_root, path_map, - timeout=scan_tmout, label=arr_label) - - # ── Safety gate 2 — API reachability ────────────────────────────────────── - print() - print(f"━━━ 🔄 Fetching {arr_label} Tracked Files ━━━") - - if not check_api(url, arr_label): - _notify(f"{arr_label} cleanup aborted on {hostname} — API unreachable") - sys.exit(1) - - # ── Safety gate 3 — API version ─────────────────────────────────────────── - if not check_arr_version(url, api_key, api_version, ver_major, arr_label): - _notify(f"{arr_label} version mismatch on {hostname} — check master.conf") - sys.exit(1) - - # ── Fetch tracked paths ──────────────────────────────────────────────────── - print(f" Querying {arr_label} API...") - tracked, parent_count, tracked_count = fetch_tracked_paths( - url, api_key, api_version, profile, path_map, arr_label, - ) - - # ── Safety gate 4 — parent count > 0 ────────────────────────────────────── - if tracked is None or parent_count == 0: - msg = f"API returned 0 {profile['parent_label']} — aborting to prevent mass deletion" - _notify(f"{arr_label} cleanup aborted on {hostname} — 0 {profile['parent_label']} returned") - die(msg) - - # ── Safety gate 5 — tracked count > 0 ──────────────────────────────────── - if tracked_count == 0: - _notify(f"{arr_label} cleanup aborted on {hostname} — 0 tracked files returned") - die("API returned 0 tracked files — aborting to prevent mass deletion") - - print(f" {parent_count} {profile['parent_label']} | {tracked_count} tracked files") - log(f"Built in-memory lookup set: {tracked_count} tracked paths") - - # ── Safety gate 6 — tracked % drop (only if configured) ────────────────── - if count_file and min_pct > 0: - count_path = Path(count_file) - if count_path.exists(): - try: - last = int(count_path.read_text().strip()) - if last > 0: - pct = int((tracked_count / last) * 100) - if pct < min_pct: - error(f"Tracked count dropped to {pct}% of last run ({tracked_count} vs {last})") - error(f"Suggests API issue — aborting to prevent mass deletion") - error(f"If expected (large removal) delete: {count_file}") - _notify(f"{arr_label} cleanup aborted on {hostname} — tracked count dropped to {pct}%") - sys.exit(1) - log(f"Tracked count: {pct}% of last run ({tracked_count} vs {last}) ✅") - except (ValueError, OSError): - log("Could not read previous count — skipping % check") - else: - log("No previous count on record — first run, saving baseline") - try: - count_path.write_text(str(tracked_count)) - except OSError as e: - warn(f"Could not write tracked count file: {e}") - - # ── Scan media root ──────────────────────────────────────────────────────── - print() - print(f"━━━ 🧹 Scanning Media Root ━━━") - print(f" Root: {media_root} | Orphan age: {orphan_age} days") - print() - - start = time.time() - orphan_count = junk_count = recent_count = protected_count = 0 - orphan_bytes = junk_bytes = 0 - age_threshold = orphan_age * 86400 - now = time.time() - max_del_bytes = int(max_del_gb * 1_073_741_824) - - scan_roots = set(path_map.values()) | {media_root} - - all_files = [] - for root in scan_roots: - if Path(root).is_dir(): - for fp in Path(root).rglob("*"): - if fp.is_file(): - all_files.append(str(fp)) - all_files = sorted(set(all_files)) - - for filepath in all_files: - if filepath in tracked: - log(f"TRACKED: {filepath}") - continue - - if is_protected(filepath, protected): - log(f"🔰 PROTECTED: {filepath}") - protected_count += 1 - continue - - try: - st = Path(filepath).stat() - except OSError: - continue - - file_size = st.st_size - - if is_media_file(filepath, extensions): - file_age = now - st.st_mtime - if file_age < age_threshold and not args.skip_strikes: - log(f"RECENT (skipping): {filepath}") - recent_count += 1 - continue - warn(f"🗑️ ORPHAN: {filepath}") - orphan_count += 1 - orphan_bytes += file_size - else: - log(f"JUNK: {filepath}") - junk_count += 1 - junk_bytes += file_size - - total_del_bytes = orphan_bytes + junk_bytes - total_removed = orphan_count + junk_count - - # ── Safety gate 7 — deletion size threshold ─────────────────────────────── - if total_del_bytes > max_del_bytes: - total_human = format_bytes(total_del_bytes) - if not args.i_know: - print() - error(f"Deletion would exceed {max_del_gb}GB — {total_human} would be deleted") - error("Review ORPHAN lines above carefully before proceeding") - error("Rerun with: --i-know-what-im-doing") - error("To also bypass age check: add --skip-strike-list") - _notify(f"{arr_label} cleanup halted on {hostname} — {total_human} requires --i-know-what-im-doing") - sys.exit(1) - else: - warn(f"OVERRIDE — deletion is {total_human} — proceeding with --i-know-what-im-doing") - - # ── Execute deletions ────────────────────────────────────────────────────── - if not args.dry_run: - for filepath in all_files: - if filepath in tracked: - continue - if is_protected(filepath, protected): - continue - try: - st = Path(filepath).stat() - file_age = now - st.st_mtime - except OSError: - continue - - if is_media_file(filepath, extensions): - if file_age < age_threshold and not args.skip_strikes: - continue - - try: - Path(filepath).unlink() - except OSError as e: - error(f"Failed to delete: {filepath} — {e}") - - log("Cleaning up empty folders...") - for root in scan_roots: - if Path(root).is_dir(): - for d in sorted(Path(root).rglob("*"), key=lambda p: len(p.parts), reverse=True): - if d.is_dir(): - try: - d.rmdir() - except OSError: - pass - log("Empty folders removed") - - elapsed = int(time.time() - start) - - # ── Summary ──────────────────────────────────────────────────────────────── - orphan_human = format_bytes(orphan_bytes) - junk_human = format_bytes(junk_bytes) - - print() - print(f"━━━━━ 📋 {arr_label.upper()} CLEANUP SUMMARY ━━━━━") - print(f"🖥️ Identity: {my_id} ({server_name})") - print(f"🔄 Tracked: {tracked_count} files ({parent_count} {profile['parent_label']})") - print(f"🛡️ Protected: {protected_count} files (cover art, metadata)") - print(f"🗑️ Orphans: {orphan_count} files ({orphan_human})") - print(f"🗑️ Junk: {junk_count} files ({junk_human})") - print(f"⏭️ Recent skipped: {recent_count} files (under {orphan_age} days)") - print(f"⏱️ Duration: {format_duration(elapsed)}") - print() - - if args.dry_run: - warn("DRY RUN — no files deleted") - elif total_removed == 0: - success("Clean — nothing to remove") - else: - warn(f"🏁 Removed {total_removed} files (orphans: {orphan_human} junk: {junk_human})") - _notify( - f"{arr_label} cleanup on {hostname} — removed {total_removed} files " - f"(orphans: {orphan_human} junk: {junk_human})" - ) - notify_emby_scan(emby_url, emby_api_key, my_id) - - print("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━") - - # ── Write stats for coffee report ───────────────────────────────────────── - if not args.dry_run and stats_file: - today = datetime.date.today().strftime("%Y-%m-%d") - line = f"{today}|{ARR}|{orphan_count}|{orphan_bytes}|{junk_count}|{junk_bytes}|{recent_count}|{tracked_count}\n" - try: - with open(stats_file, "a") as f: - f.write(line) - except OSError: - pass - - -if __name__ == "__main__": - main() diff --git a/Old_Arch_Still_Works/arr_cleanup.sh b/Old_Arch_Still_Works/arr_cleanup.sh deleted file mode 100755 index e57c77f..0000000 --- a/Old_Arch_Still_Works/arr_cleanup.sh +++ /dev/null @@ -1,116 +0,0 @@ -#!/bin/bash -# ============================================================================================== -# ================================= Arr Cleanup Launcher ======================================= -# ============================================================================================== -# -# PURPOSE -# ───────────────────────────────────────────────────────────────────────────── -# Thin bash launcher for arr_cleanup.py. Handles everything bash is uniquely -# suited for: sourcing shell config, detect_hosts(), exporting env vars. -# All logic lives in Python. -# -# USAGE -# ───────────────────────────────────────────────────────────────────────────── -# arr_cleanup.sh --arr lidarr [--dry-run] [--log] [--status] -# arr_cleanup.sh --arr radarr [--i-know-what-im-doing] [--skip-strike-list] -# arr_cleanup.sh --arr sonarr -# -# ADDING A NEW ARR -# ───────────────────────────────────────────────────────────────────────────── -# 1. Add HOST*__URL, API_KEY, MEDIA_ROOT, PATH_MAP to host*.conf -# 2. Add _ORPHAN_AGE, MAX_DELETE_GB, EXTENSIONS, etc. to master.conf -# 3. Add a profile entry to ARR_PROFILES in arr_cleanup.py (6 values) -# 4. Add an export block for the new arr below (copy Sonarr block, change prefix) -# -# ============================================================================================== - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - -source "$SCRIPT_DIR/../load_config.sh" - -if [[ "$EUID" -ne 0 ]]; then - echo "ERROR: Must be run as root" >&2 - exit 1 -fi - -acquire_lock - -if ! command -v python3 >/dev/null 2>&1; then - echo "ERROR: python3 not found — required for arr_cleanup" >&2 - exit 1 -fi - -detect_hosts - -# ── Helper: serialize host-specific associative array to JSON ────────────────── -# Reads ${MY_ID}_${ARR_UPPER}_PATH_MAP and emits {"container_path":"host_path",...} -# Paths with double-quotes in names are not supported (not a real-world constraint). -_path_map_json() { - local arr_upper="$1" - local map_var="${MY_ID}_${arr_upper}_PATH_MAP" - local json="{" - local sep="" keys k v - - if ! declare -p "$map_var" 2>/dev/null | grep -q "declare -A"; then - echo "{}" - return - fi - - eval "keys=(\"\${!${map_var}[@]}\")" - for k in "${keys[@]}"; do - eval "v=\"\${${map_var}[\$k]}\"" - json+="${sep}\"${k}\":\"${v}\"" - sep="," - done - json+="}" - echo "$json" -} - -# ── Host identity ────────────────────────────────────────────────────────────── -export MY_ID LOCAL_SERVER_NAME -export ARR_HOSTNAME -ARR_HOSTNAME=$(hostname) - -# ── Notifications ────────────────────────────────────────────────────────────── -export NOTIFY_UNRAID -export EMBY_URL EMBY_API_KEY -export MY_DISCORD_WEBHOOK - -# ── Shared ───────────────────────────────────────────────────────────────────── -export ARR_CLEANUP_STATS -export LOCK_DIR LOCK_WAIT_TIMEOUT - -# ── Lidarr ───────────────────────────────────────────────────────────────────── -export LIDARR_URL LIDARR_API_KEY -export LIDARR_MEDIA_ROOT="${LIDARR_MUSIC_ROOT:-}" -export LIDARR_EXTENSIONS="${LIDARR_EXTENSIONS[*]:-}" -export LIDARR_PROTECTED_PATTERNS="${LIDARR_PROTECTED_PATTERNS[*]:-}" -export LIDARR_ORPHAN_AGE LIDARR_MAX_DELETE_GB -export LIDARR_MIN_TRACKED_PCT LIDARR_TRACKED_COUNT_FILE -export LIDARR_VERSION_MAJOR LIDARR_IMPORT_SCAN_TIMEOUT LIDARR_LOCK_WARN_AGE -export LIDARR_PATH_MAP_JSON -LIDARR_PATH_MAP_JSON=$(_path_map_json "LIDARR") - -# ── Radarr ───────────────────────────────────────────────────────────────────── -export RADARR_URL RADARR_API_KEY -export RADARR_MEDIA_ROOT="${RADARR_MOVIES_ROOT:-}" -export RADARR_EXTENSIONS="${RADARR_EXTENSIONS[*]:-}" -export RADARR_PROTECTED_PATTERNS="${RADARR_PROTECTED_PATTERNS[*]:-}" -export RADARR_ORPHAN_AGE RADARR_MAX_DELETE_GB -export RADARR_MIN_TRACKED_PCT RADARR_TRACKED_COUNT_FILE -export RADARR_VERSION_MAJOR RADARR_IMPORT_SCAN_TIMEOUT RADARR_LOCK_WARN_AGE -export RADARR_PATH_MAP_JSON -RADARR_PATH_MAP_JSON=$(_path_map_json "RADARR") - -# ── Sonarr ───────────────────────────────────────────────────────────────────── -export SONARR_URL SONARR_API_KEY -export SONARR_MEDIA_ROOT="${SONARR_TV_ROOT:-}" -export SONARR_EXTENSIONS="${SONARR_EXTENSIONS[*]:-}" -export SONARR_PROTECTED_PATTERNS="${SONARR_PROTECTED_PATTERNS[*]:-}" -export SONARR_ORPHAN_AGE SONARR_MAX_DELETE_GB -export SONARR_MIN_TRACKED_PCT SONARR_TRACKED_COUNT_FILE -export SONARR_VERSION_MAJOR SONARR_IMPORT_SCAN_TIMEOUT SONARR_LOCK_WARN_AGE -export SONARR_PATH_MAP_JSON -SONARR_PATH_MAP_JSON=$(_path_map_json "SONARR") - -exec python3 "$SCRIPT_DIR/arr_cleanup.py" "$@" diff --git a/Old_Arch_Still_Works/continuous_scripts_status.sh b/Old_Arch_Still_Works/continuous_scripts_status.sh deleted file mode 100755 index 327c11a..0000000 --- a/Old_Arch_Still_Works/continuous_scripts_status.sh +++ /dev/null @@ -1,517 +0,0 @@ -#!/bin/bash -# ============================================================================================== -# ========================= Continuous Scripts Status ========================================== -# ============================================================================================== -# -# PURPOSE -# ───────────────────────────────────────────────────────────────────────────── -# Live status dashboard for all continuously running scripts in the ecosystem. -# Run manually at any time — no schedule, no cron. -# -# For each script shows: running state, PID, uptime, approximate cycle count, -# active strikes, skip list, recent restart history, and a live health snapshot. -# -# system_watchdog — rootfs, RAM, ZFS ARC, load, zombie count, CPU temp -# docker_watchdog — running/stopped/unhealthy containers, required containers, -# memory-monitored containers, recent restart history -# failover — current state, tier status, remote Tailscale visibility -# -# If a script is mid-cycle, state files are read as-is — reflects last completed cycle. -# -# ============================================================================================== -# OPERATIONAL SAFEGUARDS -# ============================================================================================== -# -# Host-Aware Output -# detect_hosts() sets MY_ID and aliases all HOST*_WATCHDOG_* arrays. -# Required containers and tier delays are shown for the correct host. -# -# Read-Only -# Reads state files and docker inspect output only — makes no changes to any -# running script, container, or state file. -# -# ============================================================================================== -# RUNTIME MODES -# ============================================================================================== -# -# continuous_scripts_status.sh -# Show the full dashboard for all continuous scripts. -# -# continuous_scripts_status.sh --log -# Verbose output with additional detail per script section. -# -# ============================================================================================== - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - -source "$SCRIPT_DIR/../load_config.sh" - -# Dashboard script — output is the point -SILENT_MODE=false - -parse_args "$@" - -DOCKER_TIMEOUT=15 - -# ============================================================================================== -# ━━━ Setup ━━━ -# ============================================================================================== -if [[ "$EUID" -ne 0 ]]; then - error "Must be run as root" - exit 1 -fi - -acquire_lock - -# detect_hosts() sets MY_ID and aliases all HOST*_WATCHDOG_* arrays -detect_hosts - -# ============================================================================================== -# ── HELPER FUNCTIONS ────────────────────────────────────────────────────────────────────────── -# ============================================================================================== - -get_lock_pid() { - local script_name="$1" - local lockfile="$LOCK_DIR/${script_name}.lock" - if [[ -f "$lockfile" ]]; then - local content - content=$(cat "$lockfile" 2>/dev/null) - echo "${content%%:*}" - fi -} - -get_lock_name() { - local script_name="$1" - local lockfile="$LOCK_DIR/${script_name}.lock" - if [[ -f "$lockfile" ]]; then - local content - content=$(cat "$lockfile" 2>/dev/null) - echo "${content##*:}" - fi -} - -is_script_running() { - local script_name="$1" - local pid locked_name - pid=$(get_lock_pid "$script_name") - locked_name=$(get_lock_name "$script_name") - [[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null && [[ "$locked_name" == "$script_name" ]] -} - -get_lock_age() { - local script_name="$1" - local lockfile="$LOCK_DIR/${script_name}.lock" - if [[ -f "$lockfile" ]]; then - local mtime now - mtime=$(stat -c %Y "$lockfile" 2>/dev/null || echo 0) - now=$(date +%s) - echo $(( now - mtime )) - else - echo 0 - fi -} - -# Human readable uptime — days/hours/mins -format_uptime() { - local seconds=$1 - local days=$(( seconds / 86400 )) - local hours=$(( (seconds % 86400) / 3600 )) - local mins=$(( (seconds % 3600) / 60 )) - if (( days > 0 )); then - echo "${days}d ${hours}h ${mins}m" - elif (( hours > 0 )); then - echo "${hours}h ${mins}m" - else - echo "${mins}m" - fi -} - -divider() { printf '%.0s─' {1..57}; echo; } -section() { echo ""; echo " $1"; divider; } - -# ============================================================================================== -# ━━━ Header ━━━ -# ============================================================================================== -clear -echo "" -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -echo " 🛡️ WATCHDOG STATUS — $(date '+%A, %B %-d at %-I:%M%p')" -echo " $ICON_HOST $MY_ID — $LOCAL_SERVER_NAME" -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - -# ============================================================================================== -# ━━━ System Watchdog ━━━ -# ============================================================================================== -section "⚙️ SYSTEM WATCHDOG" - -SYS_PID=$(get_lock_pid "system_watchdog") -SYS_RUNNING=false - -if is_script_running "system_watchdog"; then - SYS_RUNNING=true - SYS_AGE=$(get_lock_age "system_watchdog") - SYS_UPTIME=$(format_uptime "$SYS_AGE") - SYS_CYCLE=$(( SYS_AGE / SYSTEM_WATCHDOG_INTERVAL )) - echo " ✅ Running │ PID: $SYS_PID │ Uptime: $SYS_UPTIME │ ~Cycle: $SYS_CYCLE" - echo " ⏱️ Interval: ${SYSTEM_WATCHDOG_INTERVAL}s │ Heartbeat every: ${SYSTEM_WATCHDOG_HEARTBEAT_HOURS}hr" -else - echo " ❌ NOT RUNNING — system_watchdog.sh is not active" - echo " Start via: bash Orchestrators/array_started.sh" -fi - -echo "" - -# System strikes -if [[ -f "$SYS_WATCHDOG_STATE_FILE" ]]; then - ACTIVE_STRIKES=$(grep -v ":0$" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$") - if [[ -n "$ACTIVE_STRIKES" ]]; then - echo " ⚠️ Active strikes:" - while IFS=: read -r key count; do - [[ -z "$key" ]] && continue - echo " → $key: $count/$SYS_WATCHDOG_STRIKE_LIMIT" - done <<< "$ACTIVE_STRIKES" - else - echo " ✅ Strikes: none" - fi -else - echo " ℹ️ Strike state file not found (watchdog may not have run yet)" -fi - -# Reboot log -if [[ -f "$SYS_WATCHDOG_REBOOT_LOG" ]]; then - TOTAL_REBOOTS=$(grep -c "." "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null || echo 0) - TOTAL_REBOOTS="${TOTAL_REBOOTS//[^0-9]/}" - TOTAL_REBOOTS="${TOTAL_REBOOTS:-0}" - WEEK_CUTOFF=$(date -d "7 days ago" '+%Y-%m-%d %H:%M:%S') - WEEK_REBOOTS=$(awk -v cutoff="$WEEK_CUTOFF" '$0 >= cutoff' \ - "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null | wc -l) - echo " 🔄 Watchdog reboots: $WEEK_REBOOTS this week / $TOTAL_REBOOTS total" -fi - -# Container skip list -if [[ -f "$DOCKER_WATCHDOG_FAILED_FILE" ]] && [[ -s "$DOCKER_WATCHDOG_FAILED_FILE" ]]; then - SKIP_COUNT=$(wc -l < "$DOCKER_WATCHDOG_FAILED_FILE") - echo "" - echo " ⛔ Skip list ($SKIP_COUNT — manual intervention needed):" - while IFS= read -r container; do - [[ -z "$container" ]] && continue - echo " → $container" - done < "$DOCKER_WATCHDOG_FAILED_FILE" -else - echo " ✅ Skip list: empty" -fi - -# Live system health snapshot -echo "" -echo " 📊 Current system state:" - -ROOTFS_PCT=$(df / --output=pcent 2>/dev/null | tail -1 | tr -d ' %') -[[ "${ROOTFS_PCT:-0}" -ge "${SYS_WATCHDOG_ROOTFS_PCT:-95}" ]] && \ - ROOTFS_ICON="⚠️ " || ROOTFS_ICON="✅" -echo " ${ROOTFS_ICON} rootfs: ${ROOTFS_PCT}% (threshold: ${SYS_WATCHDOG_ROOTFS_PCT}%)" - -MEM_AVAIL_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo) -MEM_FREE_GB=$(awk "BEGIN {printf \"%.1f\", $MEM_AVAIL_KB / 1048576}") -MEM_TOTAL_GB=$(awk '/MemTotal/ {printf "%.0f", $2/1048576}' /proc/meminfo) -[[ $(printf "%.0f" "$MEM_FREE_GB") -lt "${SYS_WATCHDOG_MEM_GB:-4}" ]] && \ - MEM_ICON="⚠️ " || MEM_ICON="✅" -echo " ${MEM_ICON} RAM: ${MEM_FREE_GB}GB free / ${MEM_TOTAL_GB}GB total (threshold: ${SYS_WATCHDOG_MEM_GB}GB free)" - -if [[ -f /proc/spl/kstat/zfs/arcstats ]]; then - ARC_SIZE=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats) - ARC_MAX=$(awk '/^c_max / {print $3}' /proc/spl/kstat/zfs/arcstats) - ARC_PCT=$(( ARC_SIZE * 100 / ARC_MAX )) - ARC_GB=$(awk "BEGIN {printf \"%.1f\", $ARC_SIZE / 1073741824}") - [[ "$ARC_PCT" -ge "${SYS_WATCHDOG_ARC_PINNED_PCT:-98}" ]] && \ - ARC_ICON="⚠️ " || ARC_ICON="✅" - echo " ${ARC_ICON} ZFS ARC: ${ARC_GB}GB (${ARC_PCT}% of max, threshold: ${SYS_WATCHDOG_ARC_PINNED_PCT}%)" -fi - -LOAD=$(awk '{print $1}' /proc/loadavg) -CORES=$(nproc) -LOAD_THRESH=$(( CORES * ${SYS_WATCHDOG_LOAD_MULTIPLIER:-3} )) -LOAD_INT=$(printf "%.0f" "$LOAD") -[[ "$LOAD_INT" -ge "$LOAD_THRESH" ]] && LOAD_ICON="⚠️ " || LOAD_ICON="✅" -echo " ${LOAD_ICON} Load avg: $LOAD (threshold: ${LOAD_THRESH} = ${SYS_WATCHDOG_LOAD_MULTIPLIER}x ${CORES} cores)" - -ZOMBIE_COUNT=$(ps aux | awk '{print $8}' | grep -c "^Z$" 2>/dev/null || echo 0) -ZOMBIE_COUNT="${ZOMBIE_COUNT//[^0-9]/}" -ZOMBIE_COUNT="${ZOMBIE_COUNT:-0}" -[[ "$ZOMBIE_COUNT" -ge "${SYS_WATCHDOG_ZOMBIE_LIMIT:-50}" ]] && \ - ZOMBIE_ICON="⚠️ " || ZOMBIE_ICON="✅" -echo " ${ZOMBIE_ICON} Zombies: $ZOMBIE_COUNT (threshold: ${SYS_WATCHDOG_ZOMBIE_LIMIT})" - -if command -v sensors >/dev/null 2>&1; then - CPU_TEMP=$(sensors 2>/dev/null | \ - grep -i "Package id 0\|Tctl\|CPU Temp" | \ - awk '{print $NF}' | tr -d '+°C' | head -1) - if [[ -n "$CPU_TEMP" ]]; then - CPU_TEMP_INT=$(printf "%.0f" "$CPU_TEMP") - [[ "$CPU_TEMP_INT" -ge "${SYS_WATCHDOG_CPU_TEMP_MAX:-95}" ]] && \ - TEMP_ICON="⚠️ " || TEMP_ICON="✅" - echo " ${TEMP_ICON} CPU temp: ${CPU_TEMP_INT}°C (threshold: ${SYS_WATCHDOG_CPU_TEMP_MAX}°C)" - fi -fi - -# ============================================================================================== -# ━━━ Docker Watchdog ━━━ -# ============================================================================================== -section "🐳 DOCKER WATCHDOG" - -DOCKER_PID=$(get_lock_pid "docker_watchdog") -DOCKER_RUNNING=false - -if is_script_running "docker_watchdog"; then - DOCKER_RUNNING=true - DOCKER_AGE=$(get_lock_age "docker_watchdog") - DOCKER_UPTIME=$(format_uptime "$DOCKER_AGE") - DOCKER_CYCLE=$(( DOCKER_AGE / DOCKER_WATCHDOG_INTERVAL )) - echo " ✅ Running │ PID: $DOCKER_PID │ Uptime: $DOCKER_UPTIME │ ~Cycle: $DOCKER_CYCLE" - echo " ⏱️ Interval: ${DOCKER_WATCHDOG_INTERVAL}s │ Heartbeat every: ${DOCKER_WATCHDOG_HEARTBEAT_HOURS}hr" -else - echo " ❌ NOT RUNNING — docker_watchdog.sh is not active" - echo " Start via: bash Orchestrators/array_started.sh" -fi - -echo "" - -# Container strikes -if [[ -f "$WATCHDOG_STATE_FILE" ]]; then - ACTIVE_CONTAINER_STRIKES=$(grep -v ":0$" "$WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$") - if [[ -n "$ACTIVE_CONTAINER_STRIKES" ]]; then - echo " ⚠️ Active container strikes:" - while IFS=: read -r key count; do - [[ -z "$key" ]] && continue - echo " → $key: $count" - done <<< "$ACTIVE_CONTAINER_STRIKES" - else - echo " ✅ Container strikes: none" - fi -fi - -# Container restart history -if [[ -f "$WATCHDOG_CONTAINER_RESTART_LOG" ]]; then - WEEK_CUTOFF=$(date -d "7 days ago" '+%Y-%m-%d %H:%M:%S') - WEEK_RESTARTS=$(awk -F'|' -v cutoff="$WEEK_CUTOFF" \ - '$2 >= cutoff' "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | wc -l) - if [[ "${WEEK_RESTARTS:-0}" -gt 0 ]]; then - echo "" - echo " 🔄 Container restarts this week: $WEEK_RESTARTS" - awk -F'|' -v cutoff="$WEEK_CUTOFF" \ - '$2 >= cutoff {print $1}' "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | \ - sort | uniq -c | sort -rn | head -5 | \ - while read -r count name; do - echo " → $name: $count restart(s)" - done - else - echo " ✅ Container restarts this week: none" - fi -fi - -# Container overview -echo "" -echo " 📦 Container overview:" - -if command -v docker >/dev/null 2>&1; then - RUNNING=$(timeout "$DOCKER_TIMEOUT" docker ps -q 2>/dev/null | wc -l) - TOTAL=$(timeout "$DOCKER_TIMEOUT" docker ps -aq 2>/dev/null | wc -l) - UNHEALTHY=$(timeout "$DOCKER_TIMEOUT" docker ps \ - --filter health=unhealthy -q 2>/dev/null | wc -l) - - # Stopped containers — bucket into clean vs unexpected, skip SCAN_IGNORE entirely - CLEAN_STOPPED=() - UNEXPECTED_STOPPED=() - while IFS= read -r name; do - [[ -z "$name" ]] && continue - SKIP=false - for ignore in "${WATCHDOG_SCAN_IGNORE[@]}"; do - [[ "$name" == "$ignore" ]] && SKIP=true && break - done - [[ "$SKIP" == true ]] && continue - exit_code=$(docker inspect --format '{{.State.ExitCode}}' "$name" 2>/dev/null) - if [[ "$exit_code" == "0" || "$exit_code" == "143" ]]; then - CLEAN_STOPPED+=("$name") - else - UNEXPECTED_STOPPED+=("$name") - fi - done < <(timeout "$DOCKER_TIMEOUT" docker ps -af "status=exited" \ - --format "{{.Names}}" 2>/dev/null) - - echo " Running: $RUNNING / $TOTAL total" - [[ "$UNHEALTHY" -gt 0 ]] && echo " ⚠️ Unhealthy: $UNHEALTHY" - - if [[ "${#UNEXPECTED_STOPPED[@]}" -gt 0 ]]; then - echo " ⚠️ Stopped (unexpected):" - for name in "${UNEXPECTED_STOPPED[@]}"; do - echo " → $name" - done - fi - - if [[ "${#CLEAN_STOPPED[@]}" -gt 0 ]]; then - echo " ⏸️ Stopped (clean):" - for name in "${CLEAN_STOPPED[@]}"; do - echo " → $name" - done - fi - - if [[ "${#UNEXPECTED_STOPPED[@]}" -eq 0 && "${#CLEAN_STOPPED[@]}" -eq 0 ]]; then - echo " ✅ All containers running" - fi - - # Required containers — aliased by detect_hosts() → WATCHDOG_REQUIRED_CONTAINERS - REQUIRED_ISSUES=0 - if [[ ${#WATCHDOG_REQUIRED_CONTAINERS[@]} -gt 0 ]]; then - echo "" - echo " 🔐 Required containers:" - for container in "${WATCHDOG_REQUIRED_CONTAINERS[@]}"; do - [[ -z "$container" ]] && continue - STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \ - '{{.State.Running}}' "$container" 2>/dev/null || echo "not found") - if [[ "$STATUS" == "true" ]]; then - echo " ✅ $container" - else - echo " ❌ $container — $STATUS" - (( REQUIRED_ISSUES++ )) - fi - done - fi - - # Memory-monitored containers — aliased by detect_hosts() → WATCHDOG_CONTAINERS - if [[ ${#WATCHDOG_CONTAINERS[@]} -gt 0 ]]; then - echo "" - echo " 📊 Monitored containers (memory):" - for container in "${!WATCHDOG_CONTAINERS[@]}"; do - LIMIT_MB="${WATCHDOG_CONTAINERS[$container]}" - LIMIT_GB=$(awk "BEGIN {printf \"%.0f\", $LIMIT_MB / 1024}") - USAGE=$(timeout "$DOCKER_TIMEOUT" docker stats --no-stream \ - --format "{{.MemUsage}}" "$container" 2>/dev/null | awk '{print $1}') - STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \ - '{{.State.Running}}' "$container" 2>/dev/null || echo "not found") - if [[ "$STATUS" == "true" ]]; then - echo " ✅ $container: ${USAGE:-?} (limit: ${LIMIT_GB}GB)" - else - echo " ❌ $container: not running (limit: ${LIMIT_GB}GB)" - fi - done - fi -else - echo " Docker not available" -fi - -# ============================================================================================== -# ━━━ Failover ━━━ -# ============================================================================================== -section "🔀 FALLBACK" - -FALLBACK_PID=$(get_lock_pid "fallback") -FALLBACK_RUNNING=false - -if is_script_running "fallback"; then - FALLBACK_RUNNING=true - FALLBACK_AGE=$(get_lock_age "fallback") - FALLBACK_UPTIME=$(format_uptime "$FALLBACK_AGE") - echo " ✅ Running │ PID: $FALLBACK_PID │ Uptime: $FALLBACK_UPTIME" -else - if [[ "${FALLBACK_ENABLED:-true}" == false ]]; then - echo " ⏸️ Disabled — FALLBACK_ENABLED=false in master.conf" - else - echo " ❌ NOT RUNNING — fallback.sh is not active" - echo " Start via: bash Orchestrators/array_started.sh" - fi -fi - -echo "" - -# Fallback state -FALLBACK_STATE="UNKNOWN" -FALLBACK_STATE_SECONDS=0 - -if [[ -f "$FALLBACK_STATE_FILE" ]]; then - FALLBACK_STATE=$(grep "^state=" "$FALLBACK_STATE_FILE" 2>/dev/null | cut -d= -f2) - FALLBACK_LAST_EPOCH=$(grep "^fallback_start=" "$FALLBACK_STATE_FILE" \ - 2>/dev/null | cut -d= -f2) - if [[ -n "$FALLBACK_LAST_EPOCH" && "$FALLBACK_LAST_EPOCH" -gt 0 ]]; then - FALLBACK_STATE_SECONDS=$(( $(date +%s) - FALLBACK_LAST_EPOCH )) - fi -fi - -STATE_DURATION=$(format_uptime "${FALLBACK_STATE_SECONDS:-0}") - -# Tier delays via REMOTE_ID — same logic as fallback.sh -REMOTE_TIER2_VAR="${REMOTE_ID}_TIER2_DELAY" -REMOTE_TIER3_VAR="${REMOTE_ID}_TIER3_DELAY" -REMOTE_TIER4_VAR="${REMOTE_ID}_TIER4_DELAY" -TIER2_DELAY="${!REMOTE_TIER2_VAR:-240}" -TIER3_DELAY="${!REMOTE_TIER3_VAR:-720}" -TIER4_DELAY="${!REMOTE_TIER4_VAR:-1440}" - -case "$FALLBACK_STATE" in - NORMAL) - echo " ✅ State: NORMAL" - ;; - FALLBACK) - echo " ⚠️ State: FALLBACK — $REMOTE_SERVER_NAME is down" - echo " ⏱️ Duration: $STATE_DURATION" - FALLBACK_MINS=$(( FALLBACK_STATE_SECONDS / 60 )) - echo "" - echo " 🔄 Tier status:" - echo " Tier 1 (immediate): ✅ active" - if (( FALLBACK_MINS >= TIER2_DELAY )); then - echo " Tier 2 (${TIER2_DELAY}min): ✅ active" - else - REMAINING=$(( TIER2_DELAY - FALLBACK_MINS )) - echo " Tier 2 (${TIER2_DELAY}min): ⏳ in ${REMAINING}min" - fi - if (( FALLBACK_MINS >= TIER3_DELAY )); then - echo " Tier 3 (${TIER3_DELAY}min): ✅ active" - else - REMAINING=$(( TIER3_DELAY - FALLBACK_MINS )) - echo " Tier 3 (${TIER3_DELAY}min): ⏳ in ${REMAINING}min" - fi - if (( FALLBACK_MINS >= TIER4_DELAY )); then - echo " Tier 4 (${TIER4_DELAY}min): ✅ active" - else - REMAINING=$(( TIER4_DELAY - FALLBACK_MINS )) - echo " Tier 4 (${TIER4_DELAY}min): ⏳ in ${REMAINING}min" - fi - ;; - NO_INTERNET) - echo " ❌ State: NO_INTERNET — DDNS stopped" - echo " ⏱️ Down for: $STATE_DURATION" - ;; - DARK) - echo " ❌ State: DARK — $REMOTE_SERVER_NAME down AND no internet" - echo " ⏱️ Duration: $STATE_DURATION" - ;; - *) - echo " ❓ State: ${FALLBACK_STATE:-unknown}" - ;; -esac - - -echo " 📡 Check interval: ${FALLBACK_CHECK_INTERVAL}s │ Handback strikes: ${FALLBACK_HANDBACK_STRIKES}" - -# ============================================================================================== -# ━━━ Footer ━━━ -# ============================================================================================== -echo "" -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - -ISSUES=0 -[[ "$SYS_RUNNING" == false ]] && (( ISSUES++ )) -[[ "$DOCKER_RUNNING" == false ]] && (( ISSUES++ )) -[[ "$FALLBACK_RUNNING" == false && "${FALLBACK_ENABLED:-true}" != false ]] && (( ISSUES++ )) -[[ -n "$ACTIVE_STRIKES" ]] && (( ISSUES++ )) -[[ -n "$ACTIVE_CONTAINER_STRIKES" ]] && (( ISSUES++ )) -[[ "${REQUIRED_ISSUES:-0}" -gt 0 ]] && (( ISSUES++ )) -[[ "$FALLBACK_STATE" != "NORMAL" && "$FALLBACK_STATE" != "UNKNOWN" ]] && (( ISSUES++ )) - -if [[ "$ISSUES" -eq 0 ]]; then - echo " ✅ $MY_ID — all continuous scripts healthy" -else - echo " ⚠️ $ISSUES issue(s) detected — review above" -fi - -echo " 🕐 Checked: $(date '+%H:%M:%S')" -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -echo "" \ No newline at end of file diff --git a/Plugin/Manual-Plugin.md b/Plugin/Manual-Plugin.md index 77fcac7..4c58e98 100644 --- a/Plugin/Manual-Plugin.md +++ b/Plugin/Manual-Plugin.md @@ -18,7 +18,7 @@ For folder overview see `README-Plugin.md`. For web app logic see the headers in ```bash cd Plugin/ -./dev_install.sh +./plugin_setup.sh ``` This creates: @@ -94,7 +94,7 @@ All other configuration lives in `Configurations/master.conf` and `Configuration If the repo is cloned to a new path: -1. Re-run `dev_install.sh` — removes the stale symlink and creates a new one pointing at the new path +1. Re-run `plugin_setup.sh` — removes the stale symlink and creates a new one pointing at the new path 2. Update `SCRIPTS_DIR` in Settings → Other Settings → Varaverk (or edit `varaverk.cfg` directly on flash) The `.plg` on flash does not need to change — it has no path references. @@ -115,10 +115,10 @@ management page to reflect when the plugin was last changed: ## ━━━ ADDING A NEW OS ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -`dev_install.sh` is built to support multiple OS targets. To add one: +`plugin_setup.sh` is built to support multiple OS targets. To add one: 1. Create `Plugin//` with the OS-appropriate web app files -2. Add the OS marker to `detect_os()` in `dev_install.sh`: +2. Add the OS marker to `detect_os()` in `plugin_setup.sh`: ```bash elif [[ -f /etc/-marker ]]; then echo "" ``` diff --git a/Plugin/README-Plugin.md b/Plugin/README-Plugin.md index 0e6c522..8ea1fef 100644 --- a/Plugin/README-Plugin.md +++ b/Plugin/README-Plugin.md @@ -31,7 +31,7 @@ no separate monitoring stack, no third-party dashboards. ``` Plugin/ -├── dev_install.sh # One-time developer setup: symlinks plugin into web server +├── plugin_setup.sh # One-time setup: symlinks plugin into web server ├── Icons/ # Source icon assets (1024px master files) └── unraid/ # The Unraid platform adapter + plugin application ├── adapter.sh # Platform adapter — provides platform_*() API to all scripts @@ -75,7 +75,7 @@ so behaviour stays consistent without a shell dependency. | Script | Role | When It Runs | |--------|------|--------------| -| `dev_install.sh` | Symlinks `Plugin/unraid/` into Unraid's web server | Once, manually, after cloning or moving the repo | +| `plugin_setup.sh` | Symlinks `Plugin/unraid/` into Unraid's web server | Once, manually, after cloning or moving the repo | | `build.sh` | Packages the plugin as a `.txz` for release | Before publishing a new plugin version | --- diff --git a/Plugin/build.sh b/Plugin/build.sh index da57b3d..d754d84 100755 --- a/Plugin/build.sh +++ b/Plugin/build.sh @@ -7,7 +7,7 @@ # ───────────────────────────────────────────────────────────────────────────── # Packages the plugin web files (Plugin/unraid/) into a Slackware .txz, the # format unRAID re-installs from flash on every boot. This is the RELEASE path — -# for day-to-day development use dev_install.sh (symlink, instant edits). +# for day-to-day development use plugin_setup.sh (symlink, instant edits). # # What it produces (in Plugin/dist/): # varaverk--noarch-1.txz the package unRAID installs to diff --git a/Plugin/dev_install.sh b/Plugin/plugin_setup.sh similarity index 94% rename from Plugin/dev_install.sh rename to Plugin/plugin_setup.sh index dea60df..90a46d6 100755 --- a/Plugin/dev_install.sh +++ b/Plugin/plugin_setup.sh @@ -1,6 +1,6 @@ #!/bin/bash # ============================================================================================== -# ============================= dev_install.sh ================================================= +# ============================= plugin_setup.sh ================================================= # ============================================================================================== # # PURPOSE @@ -47,10 +47,10 @@ # RUNTIME MODES # ============================================================================================== # -# ./dev_install.sh +# ./plugin_setup.sh # Auto-detects the running OS and installs. # -# ./dev_install.sh +# ./plugin_setup.sh # Overrides OS detection. Valid values: unraid, debian, arch. # Useful when testing on a machine where the marker files differ. # @@ -70,7 +70,7 @@ detect_os() { else echo "unknown"; fi } -OS="${1:-$(detect_os)}" # accept override: ./dev_install.sh unraid +OS="${1:-$(detect_os)}" # accept override: ./plugin_setup.sh unraid # ── OS-specific install target ──────────────────────────────────────────────── @@ -81,7 +81,7 @@ case "$OS" in ;; debian|arch) echo "OS '$OS' detected but plugin target path not yet defined." - echo "Add the TARGET= line for this OS in dev_install.sh." + echo "Add the TARGET= line for this OS in plugin_setup.sh." exit 1 ;; unknown|*)