FailedProxy/Varaverk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

remove stale dev artifacts; rename dev_install.sh to plugin_setup.sh

Browse Source
This commit is contained in:
Gmer4Lfe
2026-06-14 13:08:14 -04:00
parent 39b8e47ff5
commit 722e688783
9 changed files with 12 additions and 1607 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.vscode/settings.json
Vendored
-10
View File
@@ -1,10 +0,0 @@
{
"files.exclude": {
"**/.cache/**": true,
"**/.next/**": true,
"**/build/**": true,
"**/coverage/**": true,
"**/dist/**": true,
"**/node_modules/**": true
}
}
Deployment/deploy.sh
-80
View File
@@ -1,80 +0,0 @@
#!/bin/bash
# ==============================================================================================
# deploy.sh — Sync dev working tree → prod runtime for Plugin PHP and conf files.
# ==============================================================================================
#
# Direct call (interactive): ! bash Deployment/deploy.sh
# Deploys both Plugin and Configurations with full output.
#
# Hook call (Claude Code PostToolUse):
# Reads the tool JSON from stdin, deploys only the component that was actually changed.
# Plugin/ → rsync --delete into prod Plugin/unraid/
# Configurations/ → conf_upgrade.sh (adds new keys, preserves prod values)
#
# ==============================================================================================
DEV_ROOT="/mnt/cloud-storage/Important Shit/Git/Development/Varaverk"
PROD_ROOT="/boot/config/plugins/varaverk"
UPGRADE="$PROD_ROOT/Deployment/conf_upgrade.sh"
LOG="/tmp/vv_deploy.log"
# ── Deploy functions ──────────────────────────────────────────────────────────
deploy_plugin() {
rsync -a --delete --exclude='.git' \
"$DEV_ROOT/Plugin/unraid/" \
"$PROD_ROOT/Plugin/unraid/"
}
deploy_conf() {
for conf in master.conf host1.conf host2.conf; do
tmpl="$DEV_ROOT/Configurations/$conf"
target="$PROD_ROOT/Configurations/$conf"
[[ -f "$tmpl" && -f "$target" ]] && \
"$UPGRADE" --template "$tmpl" --target "$target"
done
}
# ── Entry point ───────────────────────────────────────────────────────────────
# Read stdin (non-blocking — empty when called directly or via '!')
STDIN=$(cat 2>/dev/null || true)
# Extract file_path from hook JSON if present
FILE=$(python3 -c "
import json, sys
try:
d = json.loads(sys.stdin.read())
print(d.get('tool_input', {}).get('file_path', ''))
except:
print('')
" <<< "$STDIN" 2>/dev/null || echo "")
if [[ -z "$FILE" ]]; then
# Direct call (interactive or via !) — deploy everything with output
echo "=== deploy: dev → prod ==="
echo ""
echo "── Plugin ──"
deploy_plugin && echo " rsync done"
echo ""
echo "── Configurations ──"
deploy_conf
echo ""
echo "=== done ==="
exit 0
fi
# Hook call — deploy only the affected component, log silently
{
echo "[$(date '+%H:%M:%S')] hook: $FILE"
if [[ "$FILE" == "$DEV_ROOT/Plugin/"* ]]; then
deploy_plugin && echo " plugin synced"
fi
if [[ "$FILE" == "$DEV_ROOT/Configurations/"* ]]; then
deploy_conf && echo " conf upgraded"
fi
} >> "$LOG" 2>&1
exit 0
Old_Arch_Still_Works/arr_cleanup.py
-872
View File
@@ -1,872 +0,0 @@
#!/usr/bin/env python3
# ==============================================================================================
# ================================= Arr Cleanup ================================================
# ==============================================================================================
#
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# Delete orphaned media files not tracked by an arr (Lidarr, Radarr, Sonarr,
# or any future arr). Queries the API for all tracked file paths, walks the
# library on disk, and removes anything untracked that is old enough to be
# past the import window. Triggers an Emby library clean after each deletion
# run so ghost entries disappear immediately.
#
# Called exclusively by arr_cleanup.sh, which sources shell config and exports
# all configuration as environment variables before exec'ing this script.
#
# ==============================================================================================
# OPERATIONAL MODEL
# ==============================================================================================
#
# Every file encountered on disk is classified into one of five categories:
#
# TRACKED — arr API knows this exact path → leave it alone
# PROTECTED — matches {ARR}_PROTECTED_PATTERNS → never delete
# ORPHAN — media file, not tracked, older than {ARR}_ORPHAN_AGE → delete
# JUNK — not a tracked extension, not protected → delete regardless of age
# RECENT — not tracked, under {ARR}_ORPHAN_AGE → skip (may be mid-import)
#
# Arr apps generate cover art (*.jpg), metadata (*.nfo), lyrics (*.lrc), and
# subtitles (*.srt) but do NOT include these in their tracked file API response.
# Without PROTECTED classification these would be deleted — breaking the arr
# app and Emby display.
#
# ==============================================================================================
# OPERATIONAL SAFEGUARDS
# ==============================================================================================
#
# Seven gates — ALL must pass before any file is touched:
# 1. Container running and not starting/unhealthy
# 2. API reachable
# 3. API version matches {ARR}_VERSION_MAJOR in master.conf
# 4. Parent count > 0 (artists / movies / series)
# 5. Tracked file count > 0
# 6. Tracked count >= {ARR}_MIN_TRACKED_PCT % of last known (if configured)
# 7. Deletion size < {ARR}_MAX_DELETE_GB — or --i-know-what-im-doing required
#
# ==============================================================================================
# ADDING A NEW ARR
# ==============================================================================================
#
# 1. Add an entry to ARR_PROFILES below (6 values — API endpoint pattern only)
# 2. Add HOST*_<ARR>_URL, API_KEY, MEDIA_ROOT, PATH_MAP to master_host*.conf
# 3. Add <ARR>_ORPHAN_AGE, MAX_DELETE_GB, EXTENSIONS, etc. to master.conf
# 4. Add an export block to arr_cleanup.sh (copy existing block, change prefix)
#
# ==============================================================================================
# CONFIGURATION
# ==============================================================================================
#
# master_host*.conf (host-specific, aliased by detect_hosts in arr_cleanup.sh)
#
# HOST*_{ARR}_URL — arr base URL
# HOST*_{ARR}_API_KEY — arr API key
# HOST*_{ARR}_MEDIA_ROOT — host-side library root (MUSIC_ROOT / MOVIES_ROOT / TV_ROOT)
# HOST*_{ARR}_PATH_MAP — container path → host path translation (assoc array)
#
# master.conf (shared thresholds)
#
# {ARR}_ORPHAN_AGE — days before untracked file is eligible for deletion
# {ARR}_MAX_DELETE_GB — require --i-know-what-im-doing above this
# {ARR}_MIN_TRACKED_PCT — abort if tracked count drops below this % of last run
# {ARR}_TRACKED_COUNT_FILE — persistent baseline file path (enables gate 6)
# {ARR}_EXTENSIONS — media file extensions for orphan classification
# {ARR}_PROTECTED_PATTERNS — glob patterns never deleted (cover art, metadata, etc.)
# {ARR}_VERSION_MAJOR — expected arr major version for API safety check
# {ARR}_IMPORT_SCAN_TIMEOUT — seconds to wait for pre-flight import scan (default 600)
# {ARR}_LOCK_WARN_AGE — override default lock warning age (large libraries)
# ARR_CLEANUP_STATS — stats file path (read by sunday_morning_coffee_report)
#
# ==============================================================================================
# RUNTIME MODES
# ==============================================================================================
#
# arr_cleanup.sh --arr lidarr — normal run
# arr_cleanup.sh --arr lidarr --dry-run — preview, no deletions
# arr_cleanup.sh --arr lidarr --log — verbose output
# arr_cleanup.sh --arr lidarr --status — show config and exit
# arr_cleanup.sh --arr lidarr --i-know-what-im-doing — bypass size threshold
# arr_cleanup.sh --arr lidarr --i-know-what-im-doing --skip-strike-list — NUCLEAR MODE
#
# NUCLEAR MODE: both flags bypass age check AND size threshold. Use when the arr
# has filled gaps and you want a clean one-pass wipe. Flag name is long and
# annoying by design.
#
# ==============================================================================================
import argparse
import atexit
import datetime
import fnmatch
import json
import os
import subprocess
import sys
import time
import urllib.error
import urllib.request
from pathlib import Path
# ── Arr profiles — only what cannot come from env vars ────────────────────────
# API endpoint patterns and labels. Everything else is config in master.conf.
ARR_PROFILES = {
"lidarr": {
"api_version": "v1",
"parent_endpoint": "artist",
"parent_id_param": "artistId",
"file_endpoint": "trackFile",
"import_scan_cmd": "DownloadedAlbumsScan",
"parent_label": "artists",
},
"radarr": {
"api_version": "v3",
"parent_endpoint": "movie",
"parent_id_param": "movieId",
"file_endpoint": "moviefile",
"import_scan_cmd": "DownloadedMoviesScan",
"parent_label": "movies",
},
"sonarr": {
"api_version": "v3",
"parent_endpoint": "series",
"parent_id_param": "seriesId",
"file_endpoint": "episodefile",
"import_scan_cmd": "DownloadedEpisodesScan",
"parent_label": "series",
},
# Add new arrs here. 6 values — everything else goes in master.conf.
# "readarr": {
# "api_version": "v1",
# "parent_endpoint": "author",
# "parent_id_param": "authorId",
# "file_endpoint": "bookfile",
# "import_scan_cmd": "DownloadedBooksScan",
# "parent_label": "authors",
# },
}
# ── Output helpers ─────────────────────────────────────────────────────────────
VERBOSE = False
def log(msg):
if VERBOSE:
print(f" {msg}")
def warn(msg):
print(f" ⚠️ {msg}")
def error(msg):
print(f" ❌ {msg}", file=sys.stderr)
def success(msg):
print(f" ✅ {msg}")
def die(msg, notify_fn=None):
error(msg)
if notify_fn:
notify_fn(msg)
sys.exit(1)
# ── Env var readers ────────────────────────────────────────────────────────────
def _env(key, default=""):
return os.environ.get(key, default)
def _env_int(key, default=0):
try:
return int(os.environ.get(key, str(default)))
except ValueError:
return default
def _env_float(key, default=0.0):
try:
return float(os.environ.get(key, str(default)))
except ValueError:
return default
def _env_list(key):
val = os.environ.get(key, "")
return val.split() if val else []
def _env_bool(key):
return os.environ.get(key, "false").lower() == "true"
# ── Lock ───────────────────────────────────────────────────────────────────────
LOCK_DIR = _env("LOCK_DIR", "/tmp/unraid_locks")
LOCK_TIMEOUT = _env_int("LOCK_WAIT_TIMEOUT", 30)
SCRIPT_NAME = "arr_cleanup"
def acquire_lock(warn_age=3600):
os.makedirs(LOCK_DIR, exist_ok=True)
lockfile = Path(LOCK_DIR) / f"{SCRIPT_NAME}.lock"
if lockfile.exists():
try:
content = lockfile.read_text().strip()
pid_str, locked_name = content.split(":", 1)
pid = int(pid_str)
try:
os.kill(pid, 0)
pid_alive = True
except (ProcessLookupError, PermissionError):
pid_alive = False
if not pid_alive or locked_name != SCRIPT_NAME:
warn(f"Stale lock (PID {pid} gone) — clearing")
lockfile.unlink(missing_ok=True)
else:
age = time.time() - lockfile.stat().st_mtime
if age > warn_age:
warn(f"{SCRIPT_NAME} has been running for {int(age)}s — may be stuck (PID {pid})")
print(f" Another instance of {SCRIPT_NAME} is running — waiting up to {LOCK_TIMEOUT}s...")
waited = 0
while lockfile.exists() and waited < LOCK_TIMEOUT:
time.sleep(1)
waited += 1
if lockfile.exists():
die(f"{SCRIPT_NAME} still locked after {LOCK_TIMEOUT}s — exiting")
except (ValueError, OSError):
lockfile.unlink(missing_ok=True)
lockfile.write_text(f"{os.getpid()}:{SCRIPT_NAME}")
atexit.register(lambda: lockfile.unlink(missing_ok=True))
log(f"🔏 Lock acquired: {SCRIPT_NAME} (PID {os.getpid()})")
# ── HTTP helper ────────────────────────────────────────────────────────────────
def _http_get(url, api_key, timeout=30):
req = urllib.request.Request(url, headers={"X-Api-Key": api_key})
try:
with urllib.request.urlopen(req, timeout=timeout) as resp:
return resp.status, resp.read().decode()
except urllib.error.HTTPError as e:
return e.code, ""
except Exception:
return 0, ""
def _http_post(url, api_key, payload=None, timeout=30):
data = json.dumps(payload or {}).encode()
req = urllib.request.Request(
url, data=data, method="POST",
headers={"X-Api-Key": api_key, "Content-Type": "application/json"},
)
try:
with urllib.request.urlopen(req, timeout=timeout) as resp:
return resp.status, resp.read().decode()
except urllib.error.HTTPError as e:
return e.code, ""
except Exception:
return 0, ""
# ── Arr API ────────────────────────────────────────────────────────────────────
def arr_get(base_url, api_key, api_version, endpoint):
url = f"{base_url}/api/{api_version}/{endpoint}"
status, body = _http_get(url, api_key)
if status != 200:
error(f"API HTTP {status} for: {endpoint}")
return None
try:
return json.loads(body)
except json.JSONDecodeError:
error(f"Failed to parse JSON for: {endpoint}")
return None
# ── Path translation ───────────────────────────────────────────────────────────
# Replicates common.sh translate_path() — longest-prefix match wins.
def translate_path(api_path, path_map):
best_match = ""
best_len = 0
for container_path, host_path in path_map.items():
if api_path.startswith(container_path) and len(container_path) > best_len:
best_match = container_path
best_len = len(container_path)
if best_match:
return path_map[best_match] + api_path[best_len:]
return api_path
# ── Container health check ─────────────────────────────────────────────────────
def check_container(name, timeout=15):
def _inspect(fmt):
try:
r = subprocess.run(
["timeout", str(timeout), "docker", "inspect", "-f", fmt, name],
capture_output=True, text=True,
)
return r.stdout.strip()
except Exception:
return ""
if _inspect("{{.State.Running}}") != "true":
return False, f"{name} is not running"
health = _inspect("{{.State.Health.Status}}")
if health == "healthy":
log(f"{name} is healthy")
elif health == "":
log(f"{name} has no health check — proceeding")
elif health == "starting":
return False, f"{name} is still starting"
elif health == "unhealthy":
return False, f"{name} is unhealthy"
else:
warn(f"{name} health: {health} — proceeding with caution")
return True, ""
# ── API reachability ───────────────────────────────────────────────────────────
def check_api(url, label, timeout=10):
req = urllib.request.Request(url)
try:
urllib.request.urlopen(req, timeout=timeout)
log(f"{label} API reachable: {url}")
return True
except Exception:
error(f"{label} API not reachable: {url}")
return False
# ── API version check ──────────────────────────────────────────────────────────
def check_arr_version(base_url, api_key, api_version, expected_major, label):
url = f"{base_url}/api/{api_version}/system/status"
status, body = _http_get(url, api_key, timeout=10)
if status != 200 or not body:
warn(f"{label} version check failed — proceeding without verification")
return True
try:
data = json.loads(body)
version = data.get("version", "")
major = version.split(".")[0]
if major == str(expected_major):
success(f"{label} version: {version} (major {major} — tested ✅)")
return True
else:
error(f"{label} version mismatch — running v{major}, tested against v{expected_major}")
error(f"API structure may have changed — update {label.upper()}_VERSION_MAJOR in master.conf after verifying")
return False
except Exception:
warn(f"{label} version check failed — could not parse response")
return True
# ── Import scan (pre-flight) ───────────────────────────────────────────────────
def run_import_scan(base_url, api_key, api_version, scan_cmd, media_root, path_map, timeout=600, label="Arr"):
container_root = next(
(cp for cp, hp in path_map.items() if hp == media_root),
None,
)
if container_root:
log(f"Triggering {scan_cmd} on: {container_root}")
payload = {"name": scan_cmd, "path": container_root}
else:
log(f"No path map match — triggering {scan_cmd} (all root folders)")
payload = {"name": scan_cmd}
url = f"{base_url}/api/{api_version}/command"
status, body = _http_post(url, api_key, payload)
if status not in (200, 201):
warn(f"Could not trigger import scan (HTTP {status}) — proceeding without pre-flight")
return
try:
cmd_id = json.loads(body).get("id")
except Exception:
cmd_id = None
if not cmd_id:
warn("Could not get scan command ID — proceeding without pre-flight")
return
print(f" Import scan queued (command ID: {cmd_id}) — waiting for completion...")
polled = 0
poll_url = f"{base_url}/api/{api_version}/command/{cmd_id}"
while polled < timeout:
_, body = _http_get(poll_url, api_key, timeout=10)
try:
state = json.loads(body).get("status", "")
except Exception:
state = ""
if state == "completed":
log("Import scan complete ✅")
return
if state == "failed":
warn("Import scan reported failed — proceeding anyway")
return
time.sleep(10)
polled += 10
if polled % 60 == 0:
log(f" Still scanning... ({polled}s elapsed)")
warn(f"Import scan timed out after {timeout}s — proceeding anyway")
# ── Fetch tracked paths ────────────────────────────────────────────────────────
def fetch_tracked_paths(base_url, api_key, api_version, profile, path_map, label):
parent_endpoint = profile["parent_endpoint"]
parent_id_param = profile["parent_id_param"]
file_endpoint = profile["file_endpoint"]
parent_label = profile["parent_label"]
parents = arr_get(base_url, api_key, api_version, parent_endpoint)
if parents is None:
return None, 0, 0
parent_ids = [p["id"] for p in parents]
parent_count = len(parent_ids)
if parent_count == 0:
return None, 0, 0
log(f"Found {parent_count} {parent_label} — fetching tracked files...")
tracked = set()
for i, pid in enumerate(parent_ids):
if i > 0 and i % 100 == 0:
log(f"Fetching files: {i}/{parent_count} {parent_label}...")
files = arr_get(base_url, api_key, api_version, f"{file_endpoint}?{parent_id_param}={pid}")
if not files:
continue
if isinstance(files, dict):
files = [files]
for f in files:
api_path = f.get("path", "")
if api_path:
tracked.add(translate_path(api_path, path_map))
return tracked, parent_count, len(tracked)
# ── File classification helpers ────────────────────────────────────────────────
def is_media_file(path, extensions):
ext = Path(path).suffix.lstrip(".").lower()
return ext in extensions
def is_protected(path, patterns):
name = Path(path).name
return any(fnmatch.fnmatch(name, p) for p in patterns)
# ── Notify Emby ───────────────────────────────────────────────────────────────
def notify_emby_scan(emby_url, emby_api_key, my_id):
if not emby_url or not emby_api_key:
log(f"Emby not configured on {my_id} — skipping library scan notification")
return
log("Notifying Emby to clean missing files...")
status, body = _http_get(f"{emby_url}/ScheduledTasks", emby_api_key, timeout=15)
if status != 200 or not body:
warn(f"Could not reach Emby scheduled tasks API — skipping scan")
return
try:
tasks = json.loads(body)
except Exception:
warn("Could not parse Emby tasks response")
return
task_id = None
for task in tasks:
if "Clean Missing" in task.get("Name", ""):
task_id = task.get("Id")
break
if not task_id:
for task in tasks:
if "Scan Media Library" in task.get("Name", ""):
task_id = task.get("Id")
log("Clean Missing Files not found — using Scan Media Library")
break
if not task_id:
warn("Could not find Emby Clean Missing Files or Scan Media Library task")
warn("Ghost entries will persist until next Emby scan")
return
status, _ = _http_post(f"{emby_url}/ScheduledTasks/Running/{task_id}", emby_api_key)
if status in (200, 204):
warn("🎬 Emby Clean Missing Files triggered — ghost entries will be removed")
else:
warn(f"Emby task trigger returned HTTP {status} — ghost entries may persist")
# ── Unraid + Discord notifications ────────────────────────────────────────────
def notify(msg, subject, notify_unraid, hostname, discord_webhook):
log(f"🔔 Notification: {subject} — {msg}")
if notify_unraid:
notify_script = "/usr/local/emhttp/plugins/dynamix/scripts/notify"
if os.path.isfile(notify_script) and os.access(notify_script, os.X_OK):
subprocess.run([notify_script, "-s", subject, "-d", msg, "-i", "warning"],
capture_output=True)
if discord_webhook:
payload = json.dumps({"content": f"🔔 **{subject}**\n{msg}"}).encode()
req = urllib.request.Request(
discord_webhook, data=payload, method="POST",
headers={"Content-Type": "application/json"},
)
try:
urllib.request.urlopen(req, timeout=10)
except Exception:
pass
# ── Format helpers ─────────────────────────────────────────────────────────────
def format_bytes(b):
if b > 1_073_741_824:
return f"{b / 1_073_741_824:.1f}GB"
if b > 1_048_576:
return f"{b / 1_048_576:.1f}MB"
return f"{b}B"
def format_duration(secs):
if secs >= 3600:
return f"{secs // 3600}h{(secs % 3600) // 60}m{secs % 60}s"
if secs >= 60:
return f"{secs // 60}m{secs % 60}s"
return f"{secs}s"
# ══════════════════════════════════════════════════════════════════════════════
# MAIN
# ══════════════════════════════════════════════════════════════════════════════
def main():
global VERBOSE
parser = argparse.ArgumentParser(prog="arr_cleanup.py")
parser.add_argument("--arr", required=True, choices=list(ARR_PROFILES.keys()))
parser.add_argument("--dry-run", action="store_true")
parser.add_argument("--log", action="store_true")
parser.add_argument("--status", action="store_true")
parser.add_argument("--i-know-what-im-doing", action="store_true", dest="i_know")
parser.add_argument("--skip-strike-list", action="store_true", dest="skip_strikes")
args = parser.parse_args()
VERBOSE = args.log
ARR = args.arr
PREFIX = ARR.upper()
profile = ARR_PROFILES[ARR]
arr_label = ARR.capitalize()
# ── Read config from env ───────────────────────────────────────────────────
url = _env(f"{PREFIX}_URL")
api_key = _env(f"{PREFIX}_API_KEY")
media_root = _env(f"{PREFIX}_MEDIA_ROOT")
extensions = set(_env_list(f"{PREFIX}_EXTENSIONS"))
protected = _env_list(f"{PREFIX}_PROTECTED_PATTERNS")
orphan_age = _env_int(f"{PREFIX}_ORPHAN_AGE", 3)
max_del_gb = _env_float(f"{PREFIX}_MAX_DELETE_GB", 10.0)
min_pct = _env_int(f"{PREFIX}_MIN_TRACKED_PCT", 0)
count_file = _env(f"{PREFIX}_TRACKED_COUNT_FILE")
ver_major = _env(f"{PREFIX}_VERSION_MAJOR", "0")
scan_tmout = _env_int(f"{PREFIX}_IMPORT_SCAN_TIMEOUT", 600)
lock_warn = _env_int(f"{PREFIX}_LOCK_WARN_AGE", 3600)
container = _env(f"{PREFIX}_CONTAINER") or arr_label
path_map_json = _env(f"{PREFIX}_PATH_MAP_JSON", "{}")
try:
path_map = json.loads(path_map_json)
except json.JSONDecodeError:
path_map = {}
my_id = _env("MY_ID", "HOST1")
server_name = _env("LOCAL_SERVER_NAME")
notify_unraid = _env_bool("NOTIFY_UNRAID")
emby_url = _env("EMBY_URL")
emby_api_key = _env("EMBY_API_KEY")
discord = _env("MY_DISCORD_WEBHOOK")
stats_file = _env("ARR_CLEANUP_STATS")
hostname = _env("ARR_HOSTNAME")
api_version = profile["api_version"]
def _notify(msg, subject=f"{arr_label} Cleanup"):
notify(msg, subject, notify_unraid, hostname, discord)
# ── Nuclear mode warning ───────────────────────────────────────────────────
if args.i_know and args.skip_strikes and not args.dry_run:
print()
print("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━")
print("⚠️ WARNING — NUCLEAR MODE ACTIVE")
print("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━")
print(" Flags: --i-know-what-im-doing --skip-strike-list")
print(" Strike system: BYPASSED — deletes on first pass")
print(" Size threshold: BYPASSED — no GB limit")
print(" Data recovery: NOT POSSIBLE after deletion")
print()
print(" Review --dry-run output before proceeding.")
print(" You have 10 seconds to cancel (Ctrl+C)...")
print("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━")
time.sleep(10)
print(" Proceeding...")
print()
# ── Setup ──────────────────────────────────────────────────────────────────
print()
print("━━━ ⚙️ Setup ━━━")
if not url or not api_key:
log(f"{arr_label} not configured on {my_id} ({server_name}) — skipping")
sys.exit(0)
if not media_root:
die(f"{PREFIX}_MEDIA_ROOT not set — check master_host*.conf", _notify)
if not Path(media_root).is_dir():
die(f"Media root not found: {media_root}", _notify)
acquire_lock(warn_age=lock_warn)
print(f" {my_id} ({server_name}) — {url}")
if args.dry_run: warn("DRY RUN — no files will be deleted")
if args.i_know: warn("OVERRIDE — --i-know-what-im-doing active")
if args.skip_strikes: warn("OVERRIDE — --skip-strike-list active — age check bypassed")
# ── Status ─────────────────────────────────────────────────────────────────
if args.status:
print()
print(f"━━━━━ 📋 STATUS ━━━━━")
print(f"⚙️ Identity: {my_id} ({server_name})")
print(f"⚙️ {arr_label} URL: {url}")
print(f"⚙️ Media root: {media_root}")
print(f"⏱️ Orphan age: {orphan_age} days")
print(f"⚙️ Max delete: {max_del_gb}GB (requires --i-know-what-im-doing)")
if min_pct:
print(f"⚙️ Min tracked %: {min_pct}%")
print(f"⚙️ {arr_label} ver: v{ver_major} expected")
print(f"⚙️ Extensions: {' '.join(sorted(extensions))}")
print(f"⚙️ Protected patterns: {' '.join(protected)}")
print(f"⚙️ Dry Run: {args.dry_run}")
print(f"⚙️ I know: {args.i_know}")
print(f"⚙️ Skip strikes: {args.skip_strikes}")
print("━━━━━━━━━━━━━━━━━━━━━━━")
sys.exit(0)
# ── Safety gate 1 — container health ──────────────────────────────────────
print()
print("━━━ 🛡️ Safety Checks ━━━")
ok, reason = check_container(container)
if not ok:
_notify(f"{arr_label} cleanup aborted on {hostname} — {reason}")
die(f"{reason} — aborting")
log("Safety gate 1 passed — container healthy")
# ── Pre-flight import scan ─────────────────────────────────────────────────
print()
print(f"━━━ 🔄 Pre-flight: {arr_label} Import Scan ━━━")
run_import_scan(url, api_key, api_version,
profile["import_scan_cmd"], media_root, path_map,
timeout=scan_tmout, label=arr_label)
# ── Safety gate 2 — API reachability ──────────────────────────────────────
print()
print(f"━━━ 🔄 Fetching {arr_label} Tracked Files ━━━")
if not check_api(url, arr_label):
_notify(f"{arr_label} cleanup aborted on {hostname} — API unreachable")
sys.exit(1)
# ── Safety gate 3 — API version ───────────────────────────────────────────
if not check_arr_version(url, api_key, api_version, ver_major, arr_label):
_notify(f"{arr_label} version mismatch on {hostname} — check master.conf")
sys.exit(1)
# ── Fetch tracked paths ────────────────────────────────────────────────────
print(f" Querying {arr_label} API...")
tracked, parent_count, tracked_count = fetch_tracked_paths(
url, api_key, api_version, profile, path_map, arr_label,
)
# ── Safety gate 4 — parent count > 0 ──────────────────────────────────────
if tracked is None or parent_count == 0:
msg = f"API returned 0 {profile['parent_label']} — aborting to prevent mass deletion"
_notify(f"{arr_label} cleanup aborted on {hostname} — 0 {profile['parent_label']} returned")
die(msg)
# ── Safety gate 5 — tracked count > 0 ────────────────────────────────────
if tracked_count == 0:
_notify(f"{arr_label} cleanup aborted on {hostname} — 0 tracked files returned")
die("API returned 0 tracked files — aborting to prevent mass deletion")
print(f" {parent_count} {profile['parent_label']} | {tracked_count} tracked files")
log(f"Built in-memory lookup set: {tracked_count} tracked paths")
# ── Safety gate 6 — tracked % drop (only if configured) ──────────────────
if count_file and min_pct > 0:
count_path = Path(count_file)
if count_path.exists():
try:
last = int(count_path.read_text().strip())
if last > 0:
pct = int((tracked_count / last) * 100)
if pct < min_pct:
error(f"Tracked count dropped to {pct}% of last run ({tracked_count} vs {last})")
error(f"Suggests API issue — aborting to prevent mass deletion")
error(f"If expected (large removal) delete: {count_file}")
_notify(f"{arr_label} cleanup aborted on {hostname} — tracked count dropped to {pct}%")
sys.exit(1)
log(f"Tracked count: {pct}% of last run ({tracked_count} vs {last}) ✅")
except (ValueError, OSError):
log("Could not read previous count — skipping % check")
else:
log("No previous count on record — first run, saving baseline")
try:
count_path.write_text(str(tracked_count))
except OSError as e:
warn(f"Could not write tracked count file: {e}")
# ── Scan media root ────────────────────────────────────────────────────────
print()
print(f"━━━ 🧹 Scanning Media Root ━━━")
print(f" Root: {media_root} | Orphan age: {orphan_age} days")
print()
start = time.time()
orphan_count = junk_count = recent_count = protected_count = 0
orphan_bytes = junk_bytes = 0
age_threshold = orphan_age * 86400
now = time.time()
max_del_bytes = int(max_del_gb * 1_073_741_824)
scan_roots = set(path_map.values()) | {media_root}
all_files = []
for root in scan_roots:
if Path(root).is_dir():
for fp in Path(root).rglob("*"):
if fp.is_file():
all_files.append(str(fp))
all_files = sorted(set(all_files))
for filepath in all_files:
if filepath in tracked:
log(f"TRACKED: {filepath}")
continue
if is_protected(filepath, protected):
log(f"🔰 PROTECTED: {filepath}")
protected_count += 1
continue
try:
st = Path(filepath).stat()
except OSError:
continue
file_size = st.st_size
if is_media_file(filepath, extensions):
file_age = now - st.st_mtime
if file_age < age_threshold and not args.skip_strikes:
log(f"RECENT (skipping): {filepath}")
recent_count += 1
continue
warn(f"🗑️ ORPHAN: {filepath}")
orphan_count += 1
orphan_bytes += file_size
else:
log(f"JUNK: {filepath}")
junk_count += 1
junk_bytes += file_size
total_del_bytes = orphan_bytes + junk_bytes
total_removed = orphan_count + junk_count
# ── Safety gate 7 — deletion size threshold ───────────────────────────────
if total_del_bytes > max_del_bytes:
total_human = format_bytes(total_del_bytes)
if not args.i_know:
print()
error(f"Deletion would exceed {max_del_gb}GB — {total_human} would be deleted")
error("Review ORPHAN lines above carefully before proceeding")
error("Rerun with: --i-know-what-im-doing")
error("To also bypass age check: add --skip-strike-list")
_notify(f"{arr_label} cleanup halted on {hostname} — {total_human} requires --i-know-what-im-doing")
sys.exit(1)
else:
warn(f"OVERRIDE — deletion is {total_human} — proceeding with --i-know-what-im-doing")
# ── Execute deletions ──────────────────────────────────────────────────────
if not args.dry_run:
for filepath in all_files:
if filepath in tracked:
continue
if is_protected(filepath, protected):
continue
try:
st = Path(filepath).stat()
file_age = now - st.st_mtime
except OSError:
continue
if is_media_file(filepath, extensions):
if file_age < age_threshold and not args.skip_strikes:
continue
try:
Path(filepath).unlink()
except OSError as e:
error(f"Failed to delete: {filepath} — {e}")
log("Cleaning up empty folders...")
for root in scan_roots:
if Path(root).is_dir():
for d in sorted(Path(root).rglob("*"), key=lambda p: len(p.parts), reverse=True):
if d.is_dir():
try:
d.rmdir()
except OSError:
pass
log("Empty folders removed")
elapsed = int(time.time() - start)
# ── Summary ────────────────────────────────────────────────────────────────
orphan_human = format_bytes(orphan_bytes)
junk_human = format_bytes(junk_bytes)
print()
print(f"━━━━━ 📋 {arr_label.upper()} CLEANUP SUMMARY ━━━━━")
print(f"🖥️ Identity: {my_id} ({server_name})")
print(f"🔄 Tracked: {tracked_count} files ({parent_count} {profile['parent_label']})")
print(f"🛡️ Protected: {protected_count} files (cover art, metadata)")
print(f"🗑️ Orphans: {orphan_count} files ({orphan_human})")
print(f"🗑️ Junk: {junk_count} files ({junk_human})")
print(f"⏭️ Recent skipped: {recent_count} files (under {orphan_age} days)")
print(f"⏱️ Duration: {format_duration(elapsed)}")
print()
if args.dry_run:
warn("DRY RUN — no files deleted")
elif total_removed == 0:
success("Clean — nothing to remove")
else:
warn(f"🏁 Removed {total_removed} files (orphans: {orphan_human} junk: {junk_human})")
_notify(
f"{arr_label} cleanup on {hostname} — removed {total_removed} files "
f"(orphans: {orphan_human} junk: {junk_human})"
)
notify_emby_scan(emby_url, emby_api_key, my_id)
print("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━")
# ── Write stats for coffee report ─────────────────────────────────────────
if not args.dry_run and stats_file:
today = datetime.date.today().strftime("%Y-%m-%d")
line = f"{today}|{ARR}|{orphan_count}|{orphan_bytes}|{junk_count}|{junk_bytes}|{recent_count}|{tracked_count}\n"
try:
with open(stats_file, "a") as f:
f.write(line)
except OSError:
pass
if __name__ == "__main__":
main()
Old_Arch_Still_Works/arr_cleanup.sh
-116
View File
@@ -1,116 +0,0 @@
#!/bin/bash
# ==============================================================================================
# ================================= Arr Cleanup Launcher =======================================
# ==============================================================================================
#
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# Thin bash launcher for arr_cleanup.py. Handles everything bash is uniquely
# suited for: sourcing shell config, detect_hosts(), exporting env vars.
# All logic lives in Python.
#
# USAGE
# ─────────────────────────────────────────────────────────────────────────────
# arr_cleanup.sh --arr lidarr [--dry-run] [--log] [--status]
# arr_cleanup.sh --arr radarr [--i-know-what-im-doing] [--skip-strike-list]
# arr_cleanup.sh --arr sonarr
#
# ADDING A NEW ARR
# ─────────────────────────────────────────────────────────────────────────────
# 1. Add HOST*_<ARR>_URL, API_KEY, MEDIA_ROOT, PATH_MAP to host*.conf
# 2. Add <ARR>_ORPHAN_AGE, MAX_DELETE_GB, EXTENSIONS, etc. to master.conf
# 3. Add a profile entry to ARR_PROFILES in arr_cleanup.py (6 values)
# 4. Add an export block for the new arr below (copy Sonarr block, change prefix)
#
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../load_config.sh"
if [[ "$EUID" -ne 0 ]]; then
echo "ERROR: Must be run as root" >&2
exit 1
fi
acquire_lock
if ! command -v python3 >/dev/null 2>&1; then
echo "ERROR: python3 not found — required for arr_cleanup" >&2
exit 1
fi
detect_hosts
# ── Helper: serialize host-specific associative array to JSON ──────────────────
# Reads ${MY_ID}_${ARR_UPPER}_PATH_MAP and emits {"container_path":"host_path",...}
# Paths with double-quotes in names are not supported (not a real-world constraint).
_path_map_json() {
local arr_upper="$1"
local map_var="${MY_ID}_${arr_upper}_PATH_MAP"
local json="{"
local sep="" keys k v
if ! declare -p "$map_var" 2>/dev/null | grep -q "declare -A"; then
echo "{}"
return
fi
eval "keys=(\"\${!${map_var}[@]}\")"
for k in "${keys[@]}"; do
eval "v=\"\${${map_var}[\$k]}\""
json+="${sep}\"${k}\":\"${v}\""
sep=","
done
json+="}"
echo "$json"
}
# ── Host identity ──────────────────────────────────────────────────────────────
export MY_ID LOCAL_SERVER_NAME
export ARR_HOSTNAME
ARR_HOSTNAME=$(hostname)
# ── Notifications ──────────────────────────────────────────────────────────────
export NOTIFY_UNRAID
export EMBY_URL EMBY_API_KEY
export MY_DISCORD_WEBHOOK
# ── Shared ─────────────────────────────────────────────────────────────────────
export ARR_CLEANUP_STATS
export LOCK_DIR LOCK_WAIT_TIMEOUT
# ── Lidarr ─────────────────────────────────────────────────────────────────────
export LIDARR_URL LIDARR_API_KEY
export LIDARR_MEDIA_ROOT="${LIDARR_MUSIC_ROOT:-}"
export LIDARR_EXTENSIONS="${LIDARR_EXTENSIONS[*]:-}"
export LIDARR_PROTECTED_PATTERNS="${LIDARR_PROTECTED_PATTERNS[*]:-}"
export LIDARR_ORPHAN_AGE LIDARR_MAX_DELETE_GB
export LIDARR_MIN_TRACKED_PCT LIDARR_TRACKED_COUNT_FILE
export LIDARR_VERSION_MAJOR LIDARR_IMPORT_SCAN_TIMEOUT LIDARR_LOCK_WARN_AGE
export LIDARR_PATH_MAP_JSON
LIDARR_PATH_MAP_JSON=$(_path_map_json "LIDARR")
# ── Radarr ─────────────────────────────────────────────────────────────────────
export RADARR_URL RADARR_API_KEY
export RADARR_MEDIA_ROOT="${RADARR_MOVIES_ROOT:-}"
export RADARR_EXTENSIONS="${RADARR_EXTENSIONS[*]:-}"
export RADARR_PROTECTED_PATTERNS="${RADARR_PROTECTED_PATTERNS[*]:-}"
export RADARR_ORPHAN_AGE RADARR_MAX_DELETE_GB
export RADARR_MIN_TRACKED_PCT RADARR_TRACKED_COUNT_FILE
export RADARR_VERSION_MAJOR RADARR_IMPORT_SCAN_TIMEOUT RADARR_LOCK_WARN_AGE
export RADARR_PATH_MAP_JSON
RADARR_PATH_MAP_JSON=$(_path_map_json "RADARR")
# ── Sonarr ─────────────────────────────────────────────────────────────────────
export SONARR_URL SONARR_API_KEY
export SONARR_MEDIA_ROOT="${SONARR_TV_ROOT:-}"
export SONARR_EXTENSIONS="${SONARR_EXTENSIONS[*]:-}"
export SONARR_PROTECTED_PATTERNS="${SONARR_PROTECTED_PATTERNS[*]:-}"
export SONARR_ORPHAN_AGE SONARR_MAX_DELETE_GB
export SONARR_MIN_TRACKED_PCT SONARR_TRACKED_COUNT_FILE
export SONARR_VERSION_MAJOR SONARR_IMPORT_SCAN_TIMEOUT SONARR_LOCK_WARN_AGE
export SONARR_PATH_MAP_JSON
SONARR_PATH_MAP_JSON=$(_path_map_json "SONARR")
exec python3 "$SCRIPT_DIR/arr_cleanup.py" "$@"
Old_Arch_Still_Works/continuous_scripts_status.sh
-517
View File
@@ -1,517 +0,0 @@
#!/bin/bash
# ==============================================================================================
# ========================= Continuous Scripts Status ==========================================
# ==============================================================================================
#
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# Live status dashboard for all continuously running scripts in the ecosystem.
# Run manually at any time — no schedule, no cron.
#
# For each script shows: running state, PID, uptime, approximate cycle count,
# active strikes, skip list, recent restart history, and a live health snapshot.
#
# system_watchdog — rootfs, RAM, ZFS ARC, load, zombie count, CPU temp
# docker_watchdog — running/stopped/unhealthy containers, required containers,
# memory-monitored containers, recent restart history
# failover — current state, tier status, remote Tailscale visibility
#
# If a script is mid-cycle, state files are read as-is — reflects last completed cycle.
#
# ==============================================================================================
# OPERATIONAL SAFEGUARDS
# ==============================================================================================
#
# Host-Aware Output
# detect_hosts() sets MY_ID and aliases all HOST*_WATCHDOG_* arrays.
# Required containers and tier delays are shown for the correct host.
#
# Read-Only
# Reads state files and docker inspect output only — makes no changes to any
# running script, container, or state file.
#
# ==============================================================================================
# RUNTIME MODES
# ==============================================================================================
#
# continuous_scripts_status.sh
# Show the full dashboard for all continuous scripts.
#
# continuous_scripts_status.sh --log
# Verbose output with additional detail per script section.
#
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../load_config.sh"
# Dashboard script — output is the point
SILENT_MODE=false
parse_args "$@"
DOCKER_TIMEOUT=15
# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
error "Must be run as root"
exit 1
fi
acquire_lock
# detect_hosts() sets MY_ID and aliases all HOST*_WATCHDOG_* arrays
detect_hosts
# ==============================================================================================
# ── HELPER FUNCTIONS ──────────────────────────────────────────────────────────────────────────
# ==============================================================================================
get_lock_pid() {
local script_name="$1"
local lockfile="$LOCK_DIR/${script_name}.lock"
if [[ -f "$lockfile" ]]; then
local content
content=$(cat "$lockfile" 2>/dev/null)
echo "${content%%:*}"
fi
}
get_lock_name() {
local script_name="$1"
local lockfile="$LOCK_DIR/${script_name}.lock"
if [[ -f "$lockfile" ]]; then
local content
content=$(cat "$lockfile" 2>/dev/null)
echo "${content##*:}"
fi
}
is_script_running() {
local script_name="$1"
local pid locked_name
pid=$(get_lock_pid "$script_name")
locked_name=$(get_lock_name "$script_name")
[[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null && [[ "$locked_name" == "$script_name" ]]
}
get_lock_age() {
local script_name="$1"
local lockfile="$LOCK_DIR/${script_name}.lock"
if [[ -f "$lockfile" ]]; then
local mtime now
mtime=$(stat -c %Y "$lockfile" 2>/dev/null || echo 0)
now=$(date +%s)
echo $(( now - mtime ))
else
echo 0
fi
}
# Human readable uptime — days/hours/mins
format_uptime() {
local seconds=$1
local days=$(( seconds / 86400 ))
local hours=$(( (seconds % 86400) / 3600 ))
local mins=$(( (seconds % 3600) / 60 ))
if (( days > 0 )); then
echo "${days}d ${hours}h ${mins}m"
elif (( hours > 0 )); then
echo "${hours}h ${mins}m"
else
echo "${mins}m"
fi
}
divider() { printf '%.0s─' {1..57}; echo; }
section() { echo ""; echo " $1"; divider; }
# ==============================================================================================
# ━━━ Header ━━━
# ==============================================================================================
clear
echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo " 🛡️ WATCHDOG STATUS — $(date '+%A, %B %-d at %-I:%M%p')"
echo " $ICON_HOST $MY_ID — $LOCAL_SERVER_NAME"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
# ==============================================================================================
# ━━━ System Watchdog ━━━
# ==============================================================================================
section "⚙️ SYSTEM WATCHDOG"
SYS_PID=$(get_lock_pid "system_watchdog")
SYS_RUNNING=false
if is_script_running "system_watchdog"; then
SYS_RUNNING=true
SYS_AGE=$(get_lock_age "system_watchdog")
SYS_UPTIME=$(format_uptime "$SYS_AGE")
SYS_CYCLE=$(( SYS_AGE / SYSTEM_WATCHDOG_INTERVAL ))
echo " ✅ Running │ PID: $SYS_PID │ Uptime: $SYS_UPTIME │ ~Cycle: $SYS_CYCLE"
echo " ⏱️ Interval: ${SYSTEM_WATCHDOG_INTERVAL}s │ Heartbeat every: ${SYSTEM_WATCHDOG_HEARTBEAT_HOURS}hr"
else
echo " ❌ NOT RUNNING — system_watchdog.sh is not active"
echo " Start via: bash Orchestrators/array_started.sh"
fi
echo ""
# System strikes
if [[ -f "$SYS_WATCHDOG_STATE_FILE" ]]; then
ACTIVE_STRIKES=$(grep -v ":0$" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$")
if [[ -n "$ACTIVE_STRIKES" ]]; then
echo " ⚠️ Active strikes:"
while IFS=: read -r key count; do
[[ -z "$key" ]] && continue
echo " → $key: $count/$SYS_WATCHDOG_STRIKE_LIMIT"
done <<< "$ACTIVE_STRIKES"
else
echo " ✅ Strikes: none"
fi
else
echo " ℹ️ Strike state file not found (watchdog may not have run yet)"
fi
# Reboot log
if [[ -f "$SYS_WATCHDOG_REBOOT_LOG" ]]; then
TOTAL_REBOOTS=$(grep -c "." "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null || echo 0)
TOTAL_REBOOTS="${TOTAL_REBOOTS//[^0-9]/}"
TOTAL_REBOOTS="${TOTAL_REBOOTS:-0}"
WEEK_CUTOFF=$(date -d "7 days ago" '+%Y-%m-%d %H:%M:%S')
WEEK_REBOOTS=$(awk -v cutoff="$WEEK_CUTOFF" '$0 >= cutoff' \
"$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null | wc -l)
echo " 🔄 Watchdog reboots: $WEEK_REBOOTS this week / $TOTAL_REBOOTS total"
fi
# Container skip list
if [[ -f "$DOCKER_WATCHDOG_FAILED_FILE" ]] && [[ -s "$DOCKER_WATCHDOG_FAILED_FILE" ]]; then
SKIP_COUNT=$(wc -l < "$DOCKER_WATCHDOG_FAILED_FILE")
echo ""
echo " ⛔ Skip list ($SKIP_COUNT — manual intervention needed):"
while IFS= read -r container; do
[[ -z "$container" ]] && continue
echo " → $container"
done < "$DOCKER_WATCHDOG_FAILED_FILE"
else
echo " ✅ Skip list: empty"
fi
# Live system health snapshot
echo ""
echo " 📊 Current system state:"
ROOTFS_PCT=$(df / --output=pcent 2>/dev/null | tail -1 | tr -d ' %')
[[ "${ROOTFS_PCT:-0}" -ge "${SYS_WATCHDOG_ROOTFS_PCT:-95}" ]] && \
ROOTFS_ICON="⚠️ " || ROOTFS_ICON="✅"
echo " ${ROOTFS_ICON} rootfs: ${ROOTFS_PCT}% (threshold: ${SYS_WATCHDOG_ROOTFS_PCT}%)"
MEM_AVAIL_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo)
MEM_FREE_GB=$(awk "BEGIN {printf \"%.1f\", $MEM_AVAIL_KB / 1048576}")
MEM_TOTAL_GB=$(awk '/MemTotal/ {printf "%.0f", $2/1048576}' /proc/meminfo)
[[ $(printf "%.0f" "$MEM_FREE_GB") -lt "${SYS_WATCHDOG_MEM_GB:-4}" ]] && \
MEM_ICON="⚠️ " || MEM_ICON="✅"
echo " ${MEM_ICON} RAM: ${MEM_FREE_GB}GB free / ${MEM_TOTAL_GB}GB total (threshold: ${SYS_WATCHDOG_MEM_GB}GB free)"
if [[ -f /proc/spl/kstat/zfs/arcstats ]]; then
ARC_SIZE=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats)
ARC_MAX=$(awk '/^c_max / {print $3}' /proc/spl/kstat/zfs/arcstats)
ARC_PCT=$(( ARC_SIZE * 100 / ARC_MAX ))
ARC_GB=$(awk "BEGIN {printf \"%.1f\", $ARC_SIZE / 1073741824}")
[[ "$ARC_PCT" -ge "${SYS_WATCHDOG_ARC_PINNED_PCT:-98}" ]] && \
ARC_ICON="⚠️ " || ARC_ICON="✅"
echo " ${ARC_ICON} ZFS ARC: ${ARC_GB}GB (${ARC_PCT}% of max, threshold: ${SYS_WATCHDOG_ARC_PINNED_PCT}%)"
fi
LOAD=$(awk '{print $1}' /proc/loadavg)
CORES=$(nproc)
LOAD_THRESH=$(( CORES * ${SYS_WATCHDOG_LOAD_MULTIPLIER:-3} ))
LOAD_INT=$(printf "%.0f" "$LOAD")
[[ "$LOAD_INT" -ge "$LOAD_THRESH" ]] && LOAD_ICON="⚠️ " || LOAD_ICON="✅"
echo " ${LOAD_ICON} Load avg: $LOAD (threshold: ${LOAD_THRESH} = ${SYS_WATCHDOG_LOAD_MULTIPLIER}x ${CORES} cores)"
ZOMBIE_COUNT=$(ps aux | awk '{print $8}' | grep -c "^Z$" 2>/dev/null || echo 0)
ZOMBIE_COUNT="${ZOMBIE_COUNT//[^0-9]/}"
ZOMBIE_COUNT="${ZOMBIE_COUNT:-0}"
[[ "$ZOMBIE_COUNT" -ge "${SYS_WATCHDOG_ZOMBIE_LIMIT:-50}" ]] && \
ZOMBIE_ICON="⚠️ " || ZOMBIE_ICON="✅"
echo " ${ZOMBIE_ICON} Zombies: $ZOMBIE_COUNT (threshold: ${SYS_WATCHDOG_ZOMBIE_LIMIT})"
if command -v sensors >/dev/null 2>&1; then
CPU_TEMP=$(sensors 2>/dev/null | \
grep -i "Package id 0\|Tctl\|CPU Temp" | \
awk '{print $NF}' | tr -d '+°C' | head -1)
if [[ -n "$CPU_TEMP" ]]; then
CPU_TEMP_INT=$(printf "%.0f" "$CPU_TEMP")
[[ "$CPU_TEMP_INT" -ge "${SYS_WATCHDOG_CPU_TEMP_MAX:-95}" ]] && \
TEMP_ICON="⚠️ " || TEMP_ICON="✅"
echo " ${TEMP_ICON} CPU temp: ${CPU_TEMP_INT}°C (threshold: ${SYS_WATCHDOG_CPU_TEMP_MAX}°C)"
fi
fi
# ==============================================================================================
# ━━━ Docker Watchdog ━━━
# ==============================================================================================
section "🐳 DOCKER WATCHDOG"
DOCKER_PID=$(get_lock_pid "docker_watchdog")
DOCKER_RUNNING=false
if is_script_running "docker_watchdog"; then
DOCKER_RUNNING=true
DOCKER_AGE=$(get_lock_age "docker_watchdog")
DOCKER_UPTIME=$(format_uptime "$DOCKER_AGE")
DOCKER_CYCLE=$(( DOCKER_AGE / DOCKER_WATCHDOG_INTERVAL ))
echo " ✅ Running │ PID: $DOCKER_PID │ Uptime: $DOCKER_UPTIME │ ~Cycle: $DOCKER_CYCLE"
echo " ⏱️ Interval: ${DOCKER_WATCHDOG_INTERVAL}s │ Heartbeat every: ${DOCKER_WATCHDOG_HEARTBEAT_HOURS}hr"
else
echo " ❌ NOT RUNNING — docker_watchdog.sh is not active"
echo " Start via: bash Orchestrators/array_started.sh"
fi
echo ""
# Container strikes
if [[ -f "$WATCHDOG_STATE_FILE" ]]; then
ACTIVE_CONTAINER_STRIKES=$(grep -v ":0$" "$WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$")
if [[ -n "$ACTIVE_CONTAINER_STRIKES" ]]; then
echo " ⚠️ Active container strikes:"
while IFS=: read -r key count; do
[[ -z "$key" ]] && continue
echo " → $key: $count"
done <<< "$ACTIVE_CONTAINER_STRIKES"
else
echo " ✅ Container strikes: none"
fi
fi
# Container restart history
if [[ -f "$WATCHDOG_CONTAINER_RESTART_LOG" ]]; then
WEEK_CUTOFF=$(date -d "7 days ago" '+%Y-%m-%d %H:%M:%S')
WEEK_RESTARTS=$(awk -F'|' -v cutoff="$WEEK_CUTOFF" \
'$2 >= cutoff' "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | wc -l)
if [[ "${WEEK_RESTARTS:-0}" -gt 0 ]]; then
echo ""
echo " 🔄 Container restarts this week: $WEEK_RESTARTS"
awk -F'|' -v cutoff="$WEEK_CUTOFF" \
'$2 >= cutoff {print $1}' "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | \
sort | uniq -c | sort -rn | head -5 | \
while read -r count name; do
echo " → $name: $count restart(s)"
done
else
echo " ✅ Container restarts this week: none"
fi
fi
# Container overview
echo ""
echo " 📦 Container overview:"
if command -v docker >/dev/null 2>&1; then
RUNNING=$(timeout "$DOCKER_TIMEOUT" docker ps -q 2>/dev/null | wc -l)
TOTAL=$(timeout "$DOCKER_TIMEOUT" docker ps -aq 2>/dev/null | wc -l)
UNHEALTHY=$(timeout "$DOCKER_TIMEOUT" docker ps \
--filter health=unhealthy -q 2>/dev/null | wc -l)
# Stopped containers — bucket into clean vs unexpected, skip SCAN_IGNORE entirely
CLEAN_STOPPED=()
UNEXPECTED_STOPPED=()
while IFS= read -r name; do
[[ -z "$name" ]] && continue
SKIP=false
for ignore in "${WATCHDOG_SCAN_IGNORE[@]}"; do
[[ "$name" == "$ignore" ]] && SKIP=true && break
done
[[ "$SKIP" == true ]] && continue
exit_code=$(docker inspect --format '{{.State.ExitCode}}' "$name" 2>/dev/null)
if [[ "$exit_code" == "0" || "$exit_code" == "143" ]]; then
CLEAN_STOPPED+=("$name")
else
UNEXPECTED_STOPPED+=("$name")
fi
done < <(timeout "$DOCKER_TIMEOUT" docker ps -af "status=exited" \
--format "{{.Names}}" 2>/dev/null)
echo " Running: $RUNNING / $TOTAL total"
[[ "$UNHEALTHY" -gt 0 ]] && echo " ⚠️ Unhealthy: $UNHEALTHY"
if [[ "${#UNEXPECTED_STOPPED[@]}" -gt 0 ]]; then
echo " ⚠️ Stopped (unexpected):"
for name in "${UNEXPECTED_STOPPED[@]}"; do
echo " → $name"
done
fi
if [[ "${#CLEAN_STOPPED[@]}" -gt 0 ]]; then
echo " ⏸️ Stopped (clean):"
for name in "${CLEAN_STOPPED[@]}"; do
echo " → $name"
done
fi
if [[ "${#UNEXPECTED_STOPPED[@]}" -eq 0 && "${#CLEAN_STOPPED[@]}" -eq 0 ]]; then
echo " ✅ All containers running"
fi
# Required containers — aliased by detect_hosts() → WATCHDOG_REQUIRED_CONTAINERS
REQUIRED_ISSUES=0
if [[ ${#WATCHDOG_REQUIRED_CONTAINERS[@]} -gt 0 ]]; then
echo ""
echo " 🔐 Required containers:"
for container in "${WATCHDOG_REQUIRED_CONTAINERS[@]}"; do
[[ -z "$container" ]] && continue
STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
'{{.State.Running}}' "$container" 2>/dev/null || echo "not found")
if [[ "$STATUS" == "true" ]]; then
echo " ✅ $container"
else
echo " ❌ $container — $STATUS"
(( REQUIRED_ISSUES++ ))
fi
done
fi
# Memory-monitored containers — aliased by detect_hosts() → WATCHDOG_CONTAINERS
if [[ ${#WATCHDOG_CONTAINERS[@]} -gt 0 ]]; then
echo ""
echo " 📊 Monitored containers (memory):"
for container in "${!WATCHDOG_CONTAINERS[@]}"; do
LIMIT_MB="${WATCHDOG_CONTAINERS[$container]}"
LIMIT_GB=$(awk "BEGIN {printf \"%.0f\", $LIMIT_MB / 1024}")
USAGE=$(timeout "$DOCKER_TIMEOUT" docker stats --no-stream \
--format "{{.MemUsage}}" "$container" 2>/dev/null | awk '{print $1}')
STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
'{{.State.Running}}' "$container" 2>/dev/null || echo "not found")
if [[ "$STATUS" == "true" ]]; then
echo " ✅ $container: ${USAGE:-?} (limit: ${LIMIT_GB}GB)"
else
echo " ❌ $container: not running (limit: ${LIMIT_GB}GB)"
fi
done
fi
else
echo " Docker not available"
fi
# ==============================================================================================
# ━━━ Failover ━━━
# ==============================================================================================
section "🔀 FALLBACK"
FALLBACK_PID=$(get_lock_pid "fallback")
FALLBACK_RUNNING=false
if is_script_running "fallback"; then
FALLBACK_RUNNING=true
FALLBACK_AGE=$(get_lock_age "fallback")
FALLBACK_UPTIME=$(format_uptime "$FALLBACK_AGE")
echo " ✅ Running │ PID: $FALLBACK_PID │ Uptime: $FALLBACK_UPTIME"
else
if [[ "${FALLBACK_ENABLED:-true}" == false ]]; then
echo " ⏸️ Disabled — FALLBACK_ENABLED=false in master.conf"
else
echo " ❌ NOT RUNNING — fallback.sh is not active"
echo " Start via: bash Orchestrators/array_started.sh"
fi
fi
echo ""
# Fallback state
FALLBACK_STATE="UNKNOWN"
FALLBACK_STATE_SECONDS=0
if [[ -f "$FALLBACK_STATE_FILE" ]]; then
FALLBACK_STATE=$(grep "^state=" "$FALLBACK_STATE_FILE" 2>/dev/null | cut -d= -f2)
FALLBACK_LAST_EPOCH=$(grep "^fallback_start=" "$FALLBACK_STATE_FILE" \
2>/dev/null | cut -d= -f2)
if [[ -n "$FALLBACK_LAST_EPOCH" && "$FALLBACK_LAST_EPOCH" -gt 0 ]]; then
FALLBACK_STATE_SECONDS=$(( $(date +%s) - FALLBACK_LAST_EPOCH ))
fi
fi
STATE_DURATION=$(format_uptime "${FALLBACK_STATE_SECONDS:-0}")
# Tier delays via REMOTE_ID — same logic as fallback.sh
REMOTE_TIER2_VAR="${REMOTE_ID}_TIER2_DELAY"
REMOTE_TIER3_VAR="${REMOTE_ID}_TIER3_DELAY"
REMOTE_TIER4_VAR="${REMOTE_ID}_TIER4_DELAY"
TIER2_DELAY="${!REMOTE_TIER2_VAR:-240}"
TIER3_DELAY="${!REMOTE_TIER3_VAR:-720}"
TIER4_DELAY="${!REMOTE_TIER4_VAR:-1440}"
case "$FALLBACK_STATE" in
NORMAL)
echo " ✅ State: NORMAL"
;;
FALLBACK)
echo " ⚠️ State: FALLBACK — $REMOTE_SERVER_NAME is down"
echo " ⏱️ Duration: $STATE_DURATION"
FALLBACK_MINS=$(( FALLBACK_STATE_SECONDS / 60 ))
echo ""
echo " 🔄 Tier status:"
echo " Tier 1 (immediate): ✅ active"
if (( FALLBACK_MINS >= TIER2_DELAY )); then
echo " Tier 2 (${TIER2_DELAY}min): ✅ active"
else
REMAINING=$(( TIER2_DELAY - FALLBACK_MINS ))
echo " Tier 2 (${TIER2_DELAY}min): ⏳ in ${REMAINING}min"
fi
if (( FALLBACK_MINS >= TIER3_DELAY )); then
echo " Tier 3 (${TIER3_DELAY}min): ✅ active"
else
REMAINING=$(( TIER3_DELAY - FALLBACK_MINS ))
echo " Tier 3 (${TIER3_DELAY}min): ⏳ in ${REMAINING}min"
fi
if (( FALLBACK_MINS >= TIER4_DELAY )); then
echo " Tier 4 (${TIER4_DELAY}min): ✅ active"
else
REMAINING=$(( TIER4_DELAY - FALLBACK_MINS ))
echo " Tier 4 (${TIER4_DELAY}min): ⏳ in ${REMAINING}min"
fi
;;
NO_INTERNET)
echo " ❌ State: NO_INTERNET — DDNS stopped"
echo " ⏱️ Down for: $STATE_DURATION"
;;
DARK)
echo " ❌ State: DARK — $REMOTE_SERVER_NAME down AND no internet"
echo " ⏱️ Duration: $STATE_DURATION"
;;
*)
echo " ❓ State: ${FALLBACK_STATE:-unknown}"
;;
esac
echo " 📡 Check interval: ${FALLBACK_CHECK_INTERVAL}s │ Handback strikes: ${FALLBACK_HANDBACK_STRIKES}"
# ==============================================================================================
# ━━━ Footer ━━━
# ==============================================================================================
echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
ISSUES=0
[[ "$SYS_RUNNING" == false ]] && (( ISSUES++ ))
[[ "$DOCKER_RUNNING" == false ]] && (( ISSUES++ ))
[[ "$FALLBACK_RUNNING" == false && "${FALLBACK_ENABLED:-true}" != false ]] && (( ISSUES++ ))
[[ -n "$ACTIVE_STRIKES" ]] && (( ISSUES++ ))
[[ -n "$ACTIVE_CONTAINER_STRIKES" ]] && (( ISSUES++ ))
[[ "${REQUIRED_ISSUES:-0}" -gt 0 ]] && (( ISSUES++ ))
[[ "$FALLBACK_STATE" != "NORMAL" && "$FALLBACK_STATE" != "UNKNOWN" ]] && (( ISSUES++ ))
if [[ "$ISSUES" -eq 0 ]]; then
echo " ✅ $MY_ID — all continuous scripts healthy"
else
echo " ⚠️ $ISSUES issue(s) detected — review above"
fi
echo " 🕐 Checked: $(date '+%H:%M:%S')"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo ""
Plugin/Manual-Plugin.md
+4 -4
View File
@@ -18,7 +18,7 @@ For folder overview see `README-Plugin.md`. For web app logic see the headers in
```bash ```bash
cd Plugin/ cd Plugin/
./dev_install.sh ./plugin_setup.sh
``` ```
This creates: This creates:
@@ -94,7 +94,7 @@ All other configuration lives in `Configurations/master.conf` and `Configuration
If the repo is cloned to a new path: If the repo is cloned to a new path:
1. Re-run `dev_install.sh` — removes the stale symlink and creates a new one pointing at the new path 1. Re-run `plugin_setup.sh` — removes the stale symlink and creates a new one pointing at the new path
2. Update `SCRIPTS_DIR` in Settings → Other Settings → Varaverk (or edit `varaverk.cfg` directly on flash) 2. Update `SCRIPTS_DIR` in Settings → Other Settings → Varaverk (or edit `varaverk.cfg` directly on flash)
The `.plg` on flash does not need to change — it has no path references. The `.plg` on flash does not need to change — it has no path references.
@@ -115,10 +115,10 @@ management page to reflect when the plugin was last changed:
## ━━━ ADDING A NEW OS ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ ## ━━━ ADDING A NEW OS ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
`dev_install.sh` is built to support multiple OS targets. To add one: `plugin_setup.sh` is built to support multiple OS targets. To add one:
1. Create `Plugin/<os>/` with the OS-appropriate web app files 1. Create `Plugin/<os>/` with the OS-appropriate web app files
2. Add the OS marker to `detect_os()` in `dev_install.sh`: 2. Add the OS marker to `detect_os()` in `plugin_setup.sh`:
```bash ```bash
elif [[ -f /etc/<os>-marker ]]; then echo "<os>" elif [[ -f /etc/<os>-marker ]]; then echo "<os>"
``` ```
Plugin/README-Plugin.md
+2 -2
View File
@@ -31,7 +31,7 @@ no separate monitoring stack, no third-party dashboards.
``` ```
Plugin/ Plugin/
├── dev_install.sh # One-time developer setup: symlinks plugin into web server ├── plugin_setup.sh # One-time setup: symlinks plugin into web server
├── Icons/ # Source icon assets (1024px master files) ├── Icons/ # Source icon assets (1024px master files)
└── unraid/ # The Unraid platform adapter + plugin application └── unraid/ # The Unraid platform adapter + plugin application
├── adapter.sh # Platform adapter — provides platform_*() API to all scripts ├── adapter.sh # Platform adapter — provides platform_*() API to all scripts
@@ -75,7 +75,7 @@ so behaviour stays consistent without a shell dependency.
| Script | Role | When It Runs | | Script | Role | When It Runs |
|--------|------|--------------| |--------|------|--------------|
| `dev_install.sh` | Symlinks `Plugin/unraid/` into Unraid's web server | Once, manually, after cloning or moving the repo | | `plugin_setup.sh` | Symlinks `Plugin/unraid/` into Unraid's web server | Once, manually, after cloning or moving the repo |
| `build.sh` | Packages the plugin as a `.txz` for release | Before publishing a new plugin version | | `build.sh` | Packages the plugin as a `.txz` for release | Before publishing a new plugin version |
--- ---
Plugin/build.sh
+1 -1
View File
@@ -7,7 +7,7 @@
# ───────────────────────────────────────────────────────────────────────────── # ─────────────────────────────────────────────────────────────────────────────
# Packages the plugin web files (Plugin/unraid/) into a Slackware .txz, the # Packages the plugin web files (Plugin/unraid/) into a Slackware .txz, the
# format unRAID re-installs from flash on every boot. This is the RELEASE path — # format unRAID re-installs from flash on every boot. This is the RELEASE path —
# for day-to-day development use dev_install.sh (symlink, instant edits). # for day-to-day development use plugin_setup.sh (symlink, instant edits).
# #
# What it produces (in Plugin/dist/): # What it produces (in Plugin/dist/):
# varaverk-<version>-noarch-1.txz the package unRAID installs to # varaverk-<version>-noarch-1.txz the package unRAID installs to
Plugin/dev_install.sh → Plugin/plugin_setup.sh
+5 -5
View File
@@ -1,6 +1,6 @@
#!/bin/bash #!/bin/bash
# ============================================================================================== # ==============================================================================================
# ============================= dev_install.sh ================================================= # ============================= plugin_setup.sh =================================================
# ============================================================================================== # ==============================================================================================
# #
# PURPOSE # PURPOSE
@@ -47,10 +47,10 @@
# RUNTIME MODES # RUNTIME MODES
# ============================================================================================== # ==============================================================================================
# #
# ./dev_install.sh # ./plugin_setup.sh
# Auto-detects the running OS and installs. # Auto-detects the running OS and installs.
# #
# ./dev_install.sh <os> # ./plugin_setup.sh <os>
# Overrides OS detection. Valid values: unraid, debian, arch. # Overrides OS detection. Valid values: unraid, debian, arch.
# Useful when testing on a machine where the marker files differ. # Useful when testing on a machine where the marker files differ.
# #
@@ -70,7 +70,7 @@ detect_os() {
else echo "unknown"; fi else echo "unknown"; fi
} }
OS="${1:-$(detect_os)}" # accept override: ./dev_install.sh unraid OS="${1:-$(detect_os)}" # accept override: ./plugin_setup.sh unraid
# ── OS-specific install target ──────────────────────────────────────────────── # ── OS-specific install target ────────────────────────────────────────────────
@@ -81,7 +81,7 @@ case "$OS" in
;; ;;
debian|arch) debian|arch)
echo "OS '$OS' detected but plugin target path not yet defined." echo "OS '$OS' detected but plugin target path not yet defined."
echo "Add the TARGET= line for this OS in dev_install.sh." echo "Add the TARGET= line for this OS in plugin_setup.sh."
exit 1 exit 1
;; ;;
unknown|*) unknown|*)