Compare commits

...
5 Commits
60 changed files with 149 additions and 1755 deletions
-15
View File
@@ -1,15 +0,0 @@
{
"hooks": {
"PostToolUse": [
{
"matcher": "Edit|Write",
"hooks": [
{
"type": "command",
"command": "bash '/mnt/cloud-storage/Important Shit/Git/Development/Varaverk/Deployment/deploy.sh'"
}
]
}
]
}
}
+4
View File
@@ -27,6 +27,10 @@ varaverk-*.txz
# .txz packages are attached to GitHub releases, not committed to the repo.
Plugin/dist/
# ── Claude Code installation (lives alongside repo on flash, not source) ──────
claude-bin/
claude-data/
# ── OS / editor ───────────────────────────────────────────────────────────────
.DS_Store
*.swp
-10
View File
@@ -1,10 +0,0 @@
{
"files.exclude": {
"**/.cache/**": true,
"**/.next/**": true,
"**/build/**": true,
"**/coverage/**": true,
"**/dist/**": true,
"**/node_modules/**": true
}
}
+1 -1
View File
@@ -86,7 +86,7 @@
# false = USB flash boot — /mnt/user/appdata/Varaverk (preserves flash lifetime)
# Auto-detected from boot device transport on first setup.
# To change: Settings → Storage → Migrate.
HOSTN_STORAGE_MODE_INTERNAL=true
HOSTN_STORAGE_MODE_INTERNAL=false
# ==============================================================================================
# ── IDENTITY & CONNECTIVITY ───────────────────────────────────────────────────────────────────
-80
View File
@@ -1,80 +0,0 @@
#!/bin/bash
# ==============================================================================================
# deploy.sh — Sync dev working tree → prod runtime for Plugin PHP and conf files.
# ==============================================================================================
#
# Direct call (interactive): ! bash Deployment/deploy.sh
# Deploys both Plugin and Configurations with full output.
#
# Hook call (Claude Code PostToolUse):
# Reads the tool JSON from stdin, deploys only the component that was actually changed.
# Plugin/ → rsync --delete into prod Plugin/unraid/
# Configurations/ → conf_upgrade.sh (adds new keys, preserves prod values)
#
# ==============================================================================================
DEV_ROOT="/mnt/cloud-storage/Important Shit/Git/Development/Varaverk"
PROD_ROOT="/boot/config/plugins/varaverk"
UPGRADE="$PROD_ROOT/Deployment/conf_upgrade.sh"
LOG="/tmp/vv_deploy.log"
# ── Deploy functions ──────────────────────────────────────────────────────────
deploy_plugin() {
rsync -a --delete --exclude='.git' \
"$DEV_ROOT/Plugin/unraid/" \
"$PROD_ROOT/Plugin/unraid/"
}
deploy_conf() {
for conf in master.conf host1.conf host2.conf; do
tmpl="$DEV_ROOT/Configurations/$conf"
target="$PROD_ROOT/Configurations/$conf"
[[ -f "$tmpl" && -f "$target" ]] && \
"$UPGRADE" --template "$tmpl" --target "$target"
done
}
# ── Entry point ───────────────────────────────────────────────────────────────
# Read stdin (non-blocking — empty when called directly or via '!')
STDIN=$(cat 2>/dev/null || true)
# Extract file_path from hook JSON if present
FILE=$(python3 -c "
import json, sys
try:
d = json.loads(sys.stdin.read())
print(d.get('tool_input', {}).get('file_path', ''))
except:
print('')
" <<< "$STDIN" 2>/dev/null || echo "")
if [[ -z "$FILE" ]]; then
# Direct call (interactive or via !) — deploy everything with output
echo "=== deploy: dev → prod ==="
echo ""
echo "── Plugin ──"
deploy_plugin && echo " rsync done"
echo ""
echo "── Configurations ──"
deploy_conf
echo ""
echo "=== done ==="
exit 0
fi
# Hook call — deploy only the affected component, log silently
{
echo "[$(date '+%H:%M:%S')] hook: $FILE"
if [[ "$FILE" == "$DEV_ROOT/Plugin/"* ]]; then
deploy_plugin && echo " plugin synced"
fi
if [[ "$FILE" == "$DEV_ROOT/Configurations/"* ]]; then
deploy_conf && echo " conf upgraded"
fi
} >> "$LOG" 2>&1
exit 0
+1 -1
View File
@@ -211,7 +211,7 @@ for container in "${RUNNING[@]}"; do
-f '{{.State.Running}}' "$container" 2>/dev/null || echo "unknown")
if [[ "$STATE" != "true" ]]; then
log "$ICON_DONE $container stopped in $(format_duration $(( $(date +%s) - c_start )))"
echo "$ICON_DONE $container stopped in $(format_duration $(( $(date +%s) - c_start )))"
STOPPED+=("$container")
success=true
break
+1 -1
View File
@@ -273,7 +273,7 @@ for container in "${ORDERED_RESTART[@]}"; do
if retry_docker docker restart "$container"; then
[[ "${RESTART_VERIFY_WAIT:-3}" -gt 0 ]] && sleep "${RESTART_VERIFY_WAIT:-3}"
if verify_running "$container"; then
log "$ICON_STARTED $container restarted and running in $(format_duration $(( $(date +%s) - c_start )))"
echo "$ICON_STARTED $container restarted and running in $(format_duration $(( $(date +%s) - c_start )))"
RESTARTED+=("$container")
LAST_RESTARTED="$container"
else
+1 -1
View File
@@ -394,7 +394,7 @@ if [[ ${#UPDATED[@]} -gt 0 ]]; then
fi
log "$ICON_SYNC Rebuilding $container from template on new image..."
if platform_rebuild_container "$container"; then
log "$ICON_DONE $container rebuilt ✅"
echo "$ICON_DONE $container rebuilt ✅"
REBUILT+=("$container")
else
error "Failed to rebuild $container — will be picked up by docker_daily_restart.sh"
+1 -1
View File
@@ -238,7 +238,7 @@ for container in "${ORDERED_RESTART[@]}"; do
else
if retry_docker docker restart "$container"; then
if verify_running "$container"; then
log "$ICON_STARTED $container restarted and running in $(format_duration $(( $(date +%s) - c_start )))"
echo "$ICON_STARTED $container restarted and running in $(format_duration $(( $(date +%s) - c_start )))"
RESTARTED+=("$container")
LAST_RESTARTED="$container"
else
+1 -1
View File
@@ -218,7 +218,7 @@ if [[ -n "$SLSKD_URL" ]] && [[ -n "$SLSKD_API_KEY" ]]; then
sleep 10
_ELAPSED=$(( _ELAPSED + 10 ))
if _slskd_is_connected; then
log "slskd reconnected after ${_ELAPSED}s ✅"
echo "slskd reconnected after ${_ELAPSED}s ✅"
SLSKD_CONNECTED=true
break
fi
+4 -4
View File
@@ -395,7 +395,7 @@ local_start() {
post_status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \
"$container" 2>/dev/null)
if [[ "$post_status" == "true" ]]; then
log "$ICON_STARTED $container started and running ✅"
echo "$ICON_STARTED $container started and running ✅"
return 0
else
error "$container started but crashed immediately"
@@ -428,7 +428,7 @@ local_stop() {
return 0
fi
timeout "$DOCKER_TIMEOUT" docker stop "$container" >/dev/null 2>&1 && \
log "$ICON_STOPPED $container stopped" || \
echo "$ICON_STOPPED $container stopped" || \
error "Failed to stop $container locally"
}
@@ -460,7 +460,7 @@ remote_start() {
"timeout $DOCKER_TIMEOUT docker inspect -f '{{.State.Running}}' \
$container 2>/dev/null" 2>/dev/null)
if [[ "$post_status" == "true" ]]; then
log "$ICON_STARTED $container started on $REMOTE_SERVER_NAME"
echo "$ICON_STARTED $container started on $REMOTE_SERVER_NAME"
return 0
else
error "$container started on $REMOTE_SERVER_NAME but crashed immediately"
@@ -497,7 +497,7 @@ remote_stop() {
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \
root@"$REMOTE_SERVER" \
"timeout $DOCKER_TIMEOUT docker stop $container" >/dev/null 2>&1 && \
log "$ICON_STOPPED $container stopped on $REMOTE_SERVER_NAME" || \
echo "$ICON_STOPPED $container stopped on $REMOTE_SERVER_NAME" || \
error "Failed to stop $container on $REMOTE_SERVER_NAME"
}
+5 -5
View File
@@ -334,7 +334,7 @@ if [[ "$DRY_RUN" == false ]]; then
if [[ -f "$FALLBACK_STATE_FILE" ]]; then
NEW_STATE=$(grep "^state=" "$FALLBACK_STATE_FILE" 2>/dev/null | cut -d= -f2)
if [[ "$NEW_STATE" == "FALLBACK" ]]; then
log "State changed to FALLBACK — outage detected correctly ✅"
echo "State changed to FALLBACK — outage detected correctly ✅"
phase_pass "Fallback Detection"
else
error "State is $NEW_STATE — expected FALLBACK after ${FALLBACK_TEST_BLOCK_WAIT}s"
@@ -364,7 +364,7 @@ if [[ "$DRY_RUN" == false ]]; then
STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \
"$container" 2>/dev/null)
if [[ "$STATUS" == "true" ]]; then
log "$ICON_RUNNING $container is running locally ✅"
echo "$ICON_RUNNING $container is running locally ✅"
else
error "$ICON_NOT_RUNNING $container is NOT running locally"
CONTAINERS_OK=false
@@ -394,7 +394,7 @@ if [[ "$DRY_RUN" == false ]]; then
sleep 3
if ping_remote; then
log "$REMOTE_SERVER_NAME is reachable again ✅"
echo "$REMOTE_SERVER_NAME is reachable again ✅"
phase_pass "Restore Connectivity"
else
error "$REMOTE_SERVER_NAME still unreachable after removing iptables rule"
@@ -420,7 +420,7 @@ if [[ "$DRY_RUN" == false ]]; then
if [[ -f "$FALLBACK_STATE_FILE" ]]; then
FINAL_STATE=$(grep "^state=" "$FALLBACK_STATE_FILE" 2>/dev/null | cut -d= -f2)
if [[ "$FINAL_STATE" == "NORMAL" ]]; then
log "State returned to NORMAL — handback completed ✅"
echo "State returned to NORMAL — handback completed ✅"
phase_pass "Handback"
else
error "State is $FINAL_STATE — expected NORMAL after ${FALLBACK_TEST_HANDBACK_WAIT}s"
@@ -450,7 +450,7 @@ if [[ "$DRY_RUN" == false ]]; then
STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \
"$container" 2>/dev/null)
if [[ "$STATUS" != "true" ]]; then
log "$ICON_NOT_RUNNING $container stopped locally — handed back ✅"
echo "$ICON_NOT_RUNNING $container stopped locally — handed back ✅"
else
error "$ICON_RUNNING $container still running locally — handback may have failed"
HANDBACK_OK=false
+1 -1
View File
@@ -739,7 +739,7 @@ _enforce_monitored() {
"${url}/api/${api_ver}/${bulk_endpoint}" 2>/dev/null)
if [[ "$http_code" == "200" || "$http_code" == "202" ]]; then
log "${arr_type^}: re-monitored $count items ✅"
echo "${arr_type^}: re-monitored $count items ✅"
else
warn "${arr_type^}: bulk re-monitor failed (HTTP $http_code)"
fi
+1 -1
View File
@@ -296,7 +296,7 @@ process_arr() {
' 2>/dev/null)
if [[ -z "$problem_items" ]]; then
log "$arr_name — clean ✅ no failed imports or stalled downloads"
echo "$arr_name — clean ✅ no failed imports or stalled downloads"
ARR_SUMMARIES+=("$arr_name: clean ✅")
return
fi
+2 -2
View File
@@ -208,7 +208,7 @@ for FOLDER in "${CLEAN_FOLDERS[@]}"; do
FILE_COUNT=$("${CMD[@]}" 2>/dev/null | wc -l)
if [[ "$FILE_COUNT" -eq 0 ]]; then
log "$FOLDER_NAME — clean ✅"
echo "$FOLDER_NAME — clean ✅"
echo ""
continue
fi
@@ -223,7 +223,7 @@ for FOLDER in "${CLEAN_FOLDERS[@]}"; do
else
CLEAN_CMD=("${CMD[@]}" -exec rm -f {} +)
if "${CLEAN_CMD[@]}" 2>/dev/null; then
log "$FOLDER_NAME$FILE_COUNT file(s) removed"
echo "$FOLDER_NAME$FILE_COUNT file(s) removed"
TOTAL_REMOVED=$(( TOTAL_REMOVED + FILE_COUNT ))
else
error "$FOLDER_NAME — cleanup failed"
+2 -2
View File
@@ -147,7 +147,7 @@ DROPPED=$(echo "$MOVIES" | jq '[.[] | select(.status == "deleted")] | length')
echo " $TOTAL movies total — $DROPPED dropped from TMDb"
if [[ "$DROPPED" -eq 0 ]]; then
log "No TMDb-removed movies found — nothing to do"
echo "No TMDb-removed movies found — nothing to do"
echo ""
echo "━━━━━ $ICON_SUMMARY RADARR TMDB REMOVED SUMMARY ━━━━━"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
@@ -198,7 +198,7 @@ while IFS=$'\t' read -r id title year tmdb_id has_file file_size; do
CURL_EXIT=$?
if [[ "$CURL_EXIT" -eq 0 ]]; then
log " Removed from Radarr ✅"
echo " Removed from Radarr ✅"
REMOVED+=("$title")
if [[ "$DELETE_PARAM" == "true" ]]; then
(( FILES_DELETED++ ))
+2 -2
View File
@@ -146,7 +146,7 @@ DROPPED=$(echo "$SERIES" | jq '[.[] | select(.status == "deleted")] | length')
echo " $TOTAL series total — $DROPPED dropped from TVDB"
if [[ "$DROPPED" -eq 0 ]]; then
log "No TVDB-removed series found — nothing to do"
echo "No TVDB-removed series found — nothing to do"
echo ""
echo "━━━━━ $ICON_SUMMARY SONARR TVDB REMOVED SUMMARY ━━━━━"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
@@ -200,7 +200,7 @@ while IFS=$'\t' read -r id title year tvdb_id episode_file_count size_on_disk; d
CURL_EXIT=$?
if [[ "$CURL_EXIT" -eq 0 ]]; then
log " Removed from Sonarr ✅"
echo " Removed from Sonarr ✅"
REMOVED+=("$title")
if [[ "$DELETE_PARAM" == "true" ]]; then
(( FILES_DELETED++ ))
+4 -4
View File
@@ -177,14 +177,14 @@ resolve_remote_ip
# Connectivity — no point making 100+ SSH calls if remote is unreachable
check_connectivity
log "Connectivity to $REMOTE_SERVER_NAME"
echo "Connectivity to $REMOTE_SERVER_NAME"
# Version parity — mismatched unRAID could cause md5sum path differences
check_os_version_parity || {
warn "Version parity check failed — proceeding with caution"
warn "Checksum results may be unreliable if md5sum path changed between versions"
}
log "Version parity with $REMOTE_SERVER_NAME"
echo "Version parity with $REMOTE_SERVER_NAME"
# Remote array — if array is down all files appear "missing" = false alarm
if ! check_remote_array; then
@@ -194,7 +194,7 @@ if ! check_remote_array; then
"Backup Verify" "warning"
exit 1
fi
log "Remote array mounted on $REMOTE_SERVER_NAME"
echo "Remote array mounted on $REMOTE_SERVER_NAME"
echo "Pre-flight passed ✅"
@@ -291,7 +291,7 @@ for share in "${VERIFY_SHARES[@]}"; do
warn "$SHARE_NAME — match: $SHARE_MATCH missing: $SHARE_MISSING mismatch: $SHARE_MISMATCH"
SHARES_WITH_ISSUES+=("$SHARE_NAME")
else
log "$SHARE_NAME — all $SHARE_MATCH files match ✅"
echo "$SHARE_NAME — all $SHARE_MATCH files match ✅"
fi
echo ""
+1 -1
View File
@@ -186,7 +186,7 @@ SYSTEM_INFO=$(emby_api "System/Info" 2>/dev/null) || {
SERVER_NAME=$(echo "$SYSTEM_INFO" | jq -r '.ServerName // "Unknown"' 2>/dev/null)
SERVER_VERSION=$(echo "$SYSTEM_INFO" | jq -r '.Version // "Unknown"' 2>/dev/null)
log "$ICON_EMBY Connected to: $SERVER_NAME (v$SERVER_VERSION) ✅"
echo "$ICON_EMBY Connected to: $SERVER_NAME (v$SERVER_VERSION) ✅"
# ── Active Sessions ───────────────────────────────────────────────────────────────────────────
echo "━━━ $ICON_EMBY Active Sessions ━━━"
-872
View File
@@ -1,872 +0,0 @@
#!/usr/bin/env python3
# ==============================================================================================
# ================================= Arr Cleanup ================================================
# ==============================================================================================
#
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# Delete orphaned media files not tracked by an arr (Lidarr, Radarr, Sonarr,
# or any future arr). Queries the API for all tracked file paths, walks the
# library on disk, and removes anything untracked that is old enough to be
# past the import window. Triggers an Emby library clean after each deletion
# run so ghost entries disappear immediately.
#
# Called exclusively by arr_cleanup.sh, which sources shell config and exports
# all configuration as environment variables before exec'ing this script.
#
# ==============================================================================================
# OPERATIONAL MODEL
# ==============================================================================================
#
# Every file encountered on disk is classified into one of five categories:
#
# TRACKED — arr API knows this exact path → leave it alone
# PROTECTED — matches {ARR}_PROTECTED_PATTERNS → never delete
# ORPHAN — media file, not tracked, older than {ARR}_ORPHAN_AGE → delete
# JUNK — not a tracked extension, not protected → delete regardless of age
# RECENT — not tracked, under {ARR}_ORPHAN_AGE → skip (may be mid-import)
#
# Arr apps generate cover art (*.jpg), metadata (*.nfo), lyrics (*.lrc), and
# subtitles (*.srt) but do NOT include these in their tracked file API response.
# Without PROTECTED classification these would be deleted — breaking the arr
# app and Emby display.
#
# ==============================================================================================
# OPERATIONAL SAFEGUARDS
# ==============================================================================================
#
# Seven gates — ALL must pass before any file is touched:
# 1. Container running and not starting/unhealthy
# 2. API reachable
# 3. API version matches {ARR}_VERSION_MAJOR in master.conf
# 4. Parent count > 0 (artists / movies / series)
# 5. Tracked file count > 0
# 6. Tracked count >= {ARR}_MIN_TRACKED_PCT % of last known (if configured)
# 7. Deletion size < {ARR}_MAX_DELETE_GB — or --i-know-what-im-doing required
#
# ==============================================================================================
# ADDING A NEW ARR
# ==============================================================================================
#
# 1. Add an entry to ARR_PROFILES below (6 values — API endpoint pattern only)
# 2. Add HOST*_<ARR>_URL, API_KEY, MEDIA_ROOT, PATH_MAP to master_host*.conf
# 3. Add <ARR>_ORPHAN_AGE, MAX_DELETE_GB, EXTENSIONS, etc. to master.conf
# 4. Add an export block to arr_cleanup.sh (copy existing block, change prefix)
#
# ==============================================================================================
# CONFIGURATION
# ==============================================================================================
#
# master_host*.conf (host-specific, aliased by detect_hosts in arr_cleanup.sh)
#
# HOST*_{ARR}_URL — arr base URL
# HOST*_{ARR}_API_KEY — arr API key
# HOST*_{ARR}_MEDIA_ROOT — host-side library root (MUSIC_ROOT / MOVIES_ROOT / TV_ROOT)
# HOST*_{ARR}_PATH_MAP — container path → host path translation (assoc array)
#
# master.conf (shared thresholds)
#
# {ARR}_ORPHAN_AGE — days before untracked file is eligible for deletion
# {ARR}_MAX_DELETE_GB — require --i-know-what-im-doing above this
# {ARR}_MIN_TRACKED_PCT — abort if tracked count drops below this % of last run
# {ARR}_TRACKED_COUNT_FILE — persistent baseline file path (enables gate 6)
# {ARR}_EXTENSIONS — media file extensions for orphan classification
# {ARR}_PROTECTED_PATTERNS — glob patterns never deleted (cover art, metadata, etc.)
# {ARR}_VERSION_MAJOR — expected arr major version for API safety check
# {ARR}_IMPORT_SCAN_TIMEOUT — seconds to wait for pre-flight import scan (default 600)
# {ARR}_LOCK_WARN_AGE — override default lock warning age (large libraries)
# ARR_CLEANUP_STATS — stats file path (read by sunday_morning_coffee_report)
#
# ==============================================================================================
# RUNTIME MODES
# ==============================================================================================
#
# arr_cleanup.sh --arr lidarr — normal run
# arr_cleanup.sh --arr lidarr --dry-run — preview, no deletions
# arr_cleanup.sh --arr lidarr --log — verbose output
# arr_cleanup.sh --arr lidarr --status — show config and exit
# arr_cleanup.sh --arr lidarr --i-know-what-im-doing — bypass size threshold
# arr_cleanup.sh --arr lidarr --i-know-what-im-doing --skip-strike-list — NUCLEAR MODE
#
# NUCLEAR MODE: both flags bypass age check AND size threshold. Use when the arr
# has filled gaps and you want a clean one-pass wipe. Flag name is long and
# annoying by design.
#
# ==============================================================================================
import argparse
import atexit
import datetime
import fnmatch
import json
import os
import subprocess
import sys
import time
import urllib.error
import urllib.request
from pathlib import Path
# ── Arr profiles — only what cannot come from env vars ────────────────────────
# API endpoint patterns and labels. Everything else is config in master.conf.
ARR_PROFILES = {
"lidarr": {
"api_version": "v1",
"parent_endpoint": "artist",
"parent_id_param": "artistId",
"file_endpoint": "trackFile",
"import_scan_cmd": "DownloadedAlbumsScan",
"parent_label": "artists",
},
"radarr": {
"api_version": "v3",
"parent_endpoint": "movie",
"parent_id_param": "movieId",
"file_endpoint": "moviefile",
"import_scan_cmd": "DownloadedMoviesScan",
"parent_label": "movies",
},
"sonarr": {
"api_version": "v3",
"parent_endpoint": "series",
"parent_id_param": "seriesId",
"file_endpoint": "episodefile",
"import_scan_cmd": "DownloadedEpisodesScan",
"parent_label": "series",
},
# Add new arrs here. 6 values — everything else goes in master.conf.
# "readarr": {
# "api_version": "v1",
# "parent_endpoint": "author",
# "parent_id_param": "authorId",
# "file_endpoint": "bookfile",
# "import_scan_cmd": "DownloadedBooksScan",
# "parent_label": "authors",
# },
}
# ── Output helpers ─────────────────────────────────────────────────────────────
VERBOSE = False
def log(msg):
if VERBOSE:
print(f" {msg}")
def warn(msg):
print(f" ⚠️ {msg}")
def error(msg):
print(f"{msg}", file=sys.stderr)
def success(msg):
print(f"{msg}")
def die(msg, notify_fn=None):
error(msg)
if notify_fn:
notify_fn(msg)
sys.exit(1)
# ── Env var readers ────────────────────────────────────────────────────────────
def _env(key, default=""):
return os.environ.get(key, default)
def _env_int(key, default=0):
try:
return int(os.environ.get(key, str(default)))
except ValueError:
return default
def _env_float(key, default=0.0):
try:
return float(os.environ.get(key, str(default)))
except ValueError:
return default
def _env_list(key):
val = os.environ.get(key, "")
return val.split() if val else []
def _env_bool(key):
return os.environ.get(key, "false").lower() == "true"
# ── Lock ───────────────────────────────────────────────────────────────────────
LOCK_DIR = _env("LOCK_DIR", "/tmp/unraid_locks")
LOCK_TIMEOUT = _env_int("LOCK_WAIT_TIMEOUT", 30)
SCRIPT_NAME = "arr_cleanup"
def acquire_lock(warn_age=3600):
os.makedirs(LOCK_DIR, exist_ok=True)
lockfile = Path(LOCK_DIR) / f"{SCRIPT_NAME}.lock"
if lockfile.exists():
try:
content = lockfile.read_text().strip()
pid_str, locked_name = content.split(":", 1)
pid = int(pid_str)
try:
os.kill(pid, 0)
pid_alive = True
except (ProcessLookupError, PermissionError):
pid_alive = False
if not pid_alive or locked_name != SCRIPT_NAME:
warn(f"Stale lock (PID {pid} gone) — clearing")
lockfile.unlink(missing_ok=True)
else:
age = time.time() - lockfile.stat().st_mtime
if age > warn_age:
warn(f"{SCRIPT_NAME} has been running for {int(age)}s — may be stuck (PID {pid})")
print(f" Another instance of {SCRIPT_NAME} is running — waiting up to {LOCK_TIMEOUT}s...")
waited = 0
while lockfile.exists() and waited < LOCK_TIMEOUT:
time.sleep(1)
waited += 1
if lockfile.exists():
die(f"{SCRIPT_NAME} still locked after {LOCK_TIMEOUT}s — exiting")
except (ValueError, OSError):
lockfile.unlink(missing_ok=True)
lockfile.write_text(f"{os.getpid()}:{SCRIPT_NAME}")
atexit.register(lambda: lockfile.unlink(missing_ok=True))
log(f"🔏 Lock acquired: {SCRIPT_NAME} (PID {os.getpid()})")
# ── HTTP helper ────────────────────────────────────────────────────────────────
def _http_get(url, api_key, timeout=30):
req = urllib.request.Request(url, headers={"X-Api-Key": api_key})
try:
with urllib.request.urlopen(req, timeout=timeout) as resp:
return resp.status, resp.read().decode()
except urllib.error.HTTPError as e:
return e.code, ""
except Exception:
return 0, ""
def _http_post(url, api_key, payload=None, timeout=30):
data = json.dumps(payload or {}).encode()
req = urllib.request.Request(
url, data=data, method="POST",
headers={"X-Api-Key": api_key, "Content-Type": "application/json"},
)
try:
with urllib.request.urlopen(req, timeout=timeout) as resp:
return resp.status, resp.read().decode()
except urllib.error.HTTPError as e:
return e.code, ""
except Exception:
return 0, ""
# ── Arr API ────────────────────────────────────────────────────────────────────
def arr_get(base_url, api_key, api_version, endpoint):
url = f"{base_url}/api/{api_version}/{endpoint}"
status, body = _http_get(url, api_key)
if status != 200:
error(f"API HTTP {status} for: {endpoint}")
return None
try:
return json.loads(body)
except json.JSONDecodeError:
error(f"Failed to parse JSON for: {endpoint}")
return None
# ── Path translation ───────────────────────────────────────────────────────────
# Replicates common.sh translate_path() — longest-prefix match wins.
def translate_path(api_path, path_map):
best_match = ""
best_len = 0
for container_path, host_path in path_map.items():
if api_path.startswith(container_path) and len(container_path) > best_len:
best_match = container_path
best_len = len(container_path)
if best_match:
return path_map[best_match] + api_path[best_len:]
return api_path
# ── Container health check ─────────────────────────────────────────────────────
def check_container(name, timeout=15):
def _inspect(fmt):
try:
r = subprocess.run(
["timeout", str(timeout), "docker", "inspect", "-f", fmt, name],
capture_output=True, text=True,
)
return r.stdout.strip()
except Exception:
return ""
if _inspect("{{.State.Running}}") != "true":
return False, f"{name} is not running"
health = _inspect("{{.State.Health.Status}}")
if health == "healthy":
log(f"{name} is healthy")
elif health == "":
log(f"{name} has no health check — proceeding")
elif health == "starting":
return False, f"{name} is still starting"
elif health == "unhealthy":
return False, f"{name} is unhealthy"
else:
warn(f"{name} health: {health} — proceeding with caution")
return True, ""
# ── API reachability ───────────────────────────────────────────────────────────
def check_api(url, label, timeout=10):
req = urllib.request.Request(url)
try:
urllib.request.urlopen(req, timeout=timeout)
log(f"{label} API reachable: {url}")
return True
except Exception:
error(f"{label} API not reachable: {url}")
return False
# ── API version check ──────────────────────────────────────────────────────────
def check_arr_version(base_url, api_key, api_version, expected_major, label):
url = f"{base_url}/api/{api_version}/system/status"
status, body = _http_get(url, api_key, timeout=10)
if status != 200 or not body:
warn(f"{label} version check failed — proceeding without verification")
return True
try:
data = json.loads(body)
version = data.get("version", "")
major = version.split(".")[0]
if major == str(expected_major):
success(f"{label} version: {version} (major {major} — tested ✅)")
return True
else:
error(f"{label} version mismatch — running v{major}, tested against v{expected_major}")
error(f"API structure may have changed — update {label.upper()}_VERSION_MAJOR in master.conf after verifying")
return False
except Exception:
warn(f"{label} version check failed — could not parse response")
return True
# ── Import scan (pre-flight) ───────────────────────────────────────────────────
def run_import_scan(base_url, api_key, api_version, scan_cmd, media_root, path_map, timeout=600, label="Arr"):
container_root = next(
(cp for cp, hp in path_map.items() if hp == media_root),
None,
)
if container_root:
log(f"Triggering {scan_cmd} on: {container_root}")
payload = {"name": scan_cmd, "path": container_root}
else:
log(f"No path map match — triggering {scan_cmd} (all root folders)")
payload = {"name": scan_cmd}
url = f"{base_url}/api/{api_version}/command"
status, body = _http_post(url, api_key, payload)
if status not in (200, 201):
warn(f"Could not trigger import scan (HTTP {status}) — proceeding without pre-flight")
return
try:
cmd_id = json.loads(body).get("id")
except Exception:
cmd_id = None
if not cmd_id:
warn("Could not get scan command ID — proceeding without pre-flight")
return
print(f" Import scan queued (command ID: {cmd_id}) — waiting for completion...")
polled = 0
poll_url = f"{base_url}/api/{api_version}/command/{cmd_id}"
while polled < timeout:
_, body = _http_get(poll_url, api_key, timeout=10)
try:
state = json.loads(body).get("status", "")
except Exception:
state = ""
if state == "completed":
log("Import scan complete ✅")
return
if state == "failed":
warn("Import scan reported failed — proceeding anyway")
return
time.sleep(10)
polled += 10
if polled % 60 == 0:
log(f" Still scanning... ({polled}s elapsed)")
warn(f"Import scan timed out after {timeout}s — proceeding anyway")
# ── Fetch tracked paths ────────────────────────────────────────────────────────
def fetch_tracked_paths(base_url, api_key, api_version, profile, path_map, label):
parent_endpoint = profile["parent_endpoint"]
parent_id_param = profile["parent_id_param"]
file_endpoint = profile["file_endpoint"]
parent_label = profile["parent_label"]
parents = arr_get(base_url, api_key, api_version, parent_endpoint)
if parents is None:
return None, 0, 0
parent_ids = [p["id"] for p in parents]
parent_count = len(parent_ids)
if parent_count == 0:
return None, 0, 0
log(f"Found {parent_count} {parent_label} — fetching tracked files...")
tracked = set()
for i, pid in enumerate(parent_ids):
if i > 0 and i % 100 == 0:
log(f"Fetching files: {i}/{parent_count} {parent_label}...")
files = arr_get(base_url, api_key, api_version, f"{file_endpoint}?{parent_id_param}={pid}")
if not files:
continue
if isinstance(files, dict):
files = [files]
for f in files:
api_path = f.get("path", "")
if api_path:
tracked.add(translate_path(api_path, path_map))
return tracked, parent_count, len(tracked)
# ── File classification helpers ────────────────────────────────────────────────
def is_media_file(path, extensions):
ext = Path(path).suffix.lstrip(".").lower()
return ext in extensions
def is_protected(path, patterns):
name = Path(path).name
return any(fnmatch.fnmatch(name, p) for p in patterns)
# ── Notify Emby ───────────────────────────────────────────────────────────────
def notify_emby_scan(emby_url, emby_api_key, my_id):
if not emby_url or not emby_api_key:
log(f"Emby not configured on {my_id} — skipping library scan notification")
return
log("Notifying Emby to clean missing files...")
status, body = _http_get(f"{emby_url}/ScheduledTasks", emby_api_key, timeout=15)
if status != 200 or not body:
warn(f"Could not reach Emby scheduled tasks API — skipping scan")
return
try:
tasks = json.loads(body)
except Exception:
warn("Could not parse Emby tasks response")
return
task_id = None
for task in tasks:
if "Clean Missing" in task.get("Name", ""):
task_id = task.get("Id")
break
if not task_id:
for task in tasks:
if "Scan Media Library" in task.get("Name", ""):
task_id = task.get("Id")
log("Clean Missing Files not found — using Scan Media Library")
break
if not task_id:
warn("Could not find Emby Clean Missing Files or Scan Media Library task")
warn("Ghost entries will persist until next Emby scan")
return
status, _ = _http_post(f"{emby_url}/ScheduledTasks/Running/{task_id}", emby_api_key)
if status in (200, 204):
warn("🎬 Emby Clean Missing Files triggered — ghost entries will be removed")
else:
warn(f"Emby task trigger returned HTTP {status} — ghost entries may persist")
# ── Unraid + Discord notifications ────────────────────────────────────────────
def notify(msg, subject, notify_unraid, hostname, discord_webhook):
log(f"🔔 Notification: {subject}{msg}")
if notify_unraid:
notify_script = "/usr/local/emhttp/plugins/dynamix/scripts/notify"
if os.path.isfile(notify_script) and os.access(notify_script, os.X_OK):
subprocess.run([notify_script, "-s", subject, "-d", msg, "-i", "warning"],
capture_output=True)
if discord_webhook:
payload = json.dumps({"content": f"🔔 **{subject}**\n{msg}"}).encode()
req = urllib.request.Request(
discord_webhook, data=payload, method="POST",
headers={"Content-Type": "application/json"},
)
try:
urllib.request.urlopen(req, timeout=10)
except Exception:
pass
# ── Format helpers ─────────────────────────────────────────────────────────────
def format_bytes(b):
if b > 1_073_741_824:
return f"{b / 1_073_741_824:.1f}GB"
if b > 1_048_576:
return f"{b / 1_048_576:.1f}MB"
return f"{b}B"
def format_duration(secs):
if secs >= 3600:
return f"{secs // 3600}h{(secs % 3600) // 60}m{secs % 60}s"
if secs >= 60:
return f"{secs // 60}m{secs % 60}s"
return f"{secs}s"
# ══════════════════════════════════════════════════════════════════════════════
# MAIN
# ══════════════════════════════════════════════════════════════════════════════
def main():
global VERBOSE
parser = argparse.ArgumentParser(prog="arr_cleanup.py")
parser.add_argument("--arr", required=True, choices=list(ARR_PROFILES.keys()))
parser.add_argument("--dry-run", action="store_true")
parser.add_argument("--log", action="store_true")
parser.add_argument("--status", action="store_true")
parser.add_argument("--i-know-what-im-doing", action="store_true", dest="i_know")
parser.add_argument("--skip-strike-list", action="store_true", dest="skip_strikes")
args = parser.parse_args()
VERBOSE = args.log
ARR = args.arr
PREFIX = ARR.upper()
profile = ARR_PROFILES[ARR]
arr_label = ARR.capitalize()
# ── Read config from env ───────────────────────────────────────────────────
url = _env(f"{PREFIX}_URL")
api_key = _env(f"{PREFIX}_API_KEY")
media_root = _env(f"{PREFIX}_MEDIA_ROOT")
extensions = set(_env_list(f"{PREFIX}_EXTENSIONS"))
protected = _env_list(f"{PREFIX}_PROTECTED_PATTERNS")
orphan_age = _env_int(f"{PREFIX}_ORPHAN_AGE", 3)
max_del_gb = _env_float(f"{PREFIX}_MAX_DELETE_GB", 10.0)
min_pct = _env_int(f"{PREFIX}_MIN_TRACKED_PCT", 0)
count_file = _env(f"{PREFIX}_TRACKED_COUNT_FILE")
ver_major = _env(f"{PREFIX}_VERSION_MAJOR", "0")
scan_tmout = _env_int(f"{PREFIX}_IMPORT_SCAN_TIMEOUT", 600)
lock_warn = _env_int(f"{PREFIX}_LOCK_WARN_AGE", 3600)
container = _env(f"{PREFIX}_CONTAINER") or arr_label
path_map_json = _env(f"{PREFIX}_PATH_MAP_JSON", "{}")
try:
path_map = json.loads(path_map_json)
except json.JSONDecodeError:
path_map = {}
my_id = _env("MY_ID", "HOST1")
server_name = _env("LOCAL_SERVER_NAME")
notify_unraid = _env_bool("NOTIFY_UNRAID")
emby_url = _env("EMBY_URL")
emby_api_key = _env("EMBY_API_KEY")
discord = _env("MY_DISCORD_WEBHOOK")
stats_file = _env("ARR_CLEANUP_STATS")
hostname = _env("ARR_HOSTNAME")
api_version = profile["api_version"]
def _notify(msg, subject=f"{arr_label} Cleanup"):
notify(msg, subject, notify_unraid, hostname, discord)
# ── Nuclear mode warning ───────────────────────────────────────────────────
if args.i_know and args.skip_strikes and not args.dry_run:
print()
print("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━")
print("⚠️ WARNING — NUCLEAR MODE ACTIVE")
print("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━")
print(" Flags: --i-know-what-im-doing --skip-strike-list")
print(" Strike system: BYPASSED — deletes on first pass")
print(" Size threshold: BYPASSED — no GB limit")
print(" Data recovery: NOT POSSIBLE after deletion")
print()
print(" Review --dry-run output before proceeding.")
print(" You have 10 seconds to cancel (Ctrl+C)...")
print("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━")
time.sleep(10)
print(" Proceeding...")
print()
# ── Setup ──────────────────────────────────────────────────────────────────
print()
print("━━━ ⚙️ Setup ━━━")
if not url or not api_key:
log(f"{arr_label} not configured on {my_id} ({server_name}) — skipping")
sys.exit(0)
if not media_root:
die(f"{PREFIX}_MEDIA_ROOT not set — check master_host*.conf", _notify)
if not Path(media_root).is_dir():
die(f"Media root not found: {media_root}", _notify)
acquire_lock(warn_age=lock_warn)
print(f" {my_id} ({server_name}) — {url}")
if args.dry_run: warn("DRY RUN — no files will be deleted")
if args.i_know: warn("OVERRIDE — --i-know-what-im-doing active")
if args.skip_strikes: warn("OVERRIDE — --skip-strike-list active — age check bypassed")
# ── Status ─────────────────────────────────────────────────────────────────
if args.status:
print()
print(f"━━━━━ 📋 STATUS ━━━━━")
print(f"⚙️ Identity: {my_id} ({server_name})")
print(f"⚙️ {arr_label} URL: {url}")
print(f"⚙️ Media root: {media_root}")
print(f"⏱️ Orphan age: {orphan_age} days")
print(f"⚙️ Max delete: {max_del_gb}GB (requires --i-know-what-im-doing)")
if min_pct:
print(f"⚙️ Min tracked %: {min_pct}%")
print(f"⚙️ {arr_label} ver: v{ver_major} expected")
print(f"⚙️ Extensions: {' '.join(sorted(extensions))}")
print(f"⚙️ Protected patterns: {' '.join(protected)}")
print(f"⚙️ Dry Run: {args.dry_run}")
print(f"⚙️ I know: {args.i_know}")
print(f"⚙️ Skip strikes: {args.skip_strikes}")
print("━━━━━━━━━━━━━━━━━━━━━━━")
sys.exit(0)
# ── Safety gate 1 — container health ──────────────────────────────────────
print()
print("━━━ 🛡️ Safety Checks ━━━")
ok, reason = check_container(container)
if not ok:
_notify(f"{arr_label} cleanup aborted on {hostname}{reason}")
die(f"{reason} — aborting")
log("Safety gate 1 passed — container healthy")
# ── Pre-flight import scan ─────────────────────────────────────────────────
print()
print(f"━━━ 🔄 Pre-flight: {arr_label} Import Scan ━━━")
run_import_scan(url, api_key, api_version,
profile["import_scan_cmd"], media_root, path_map,
timeout=scan_tmout, label=arr_label)
# ── Safety gate 2 — API reachability ──────────────────────────────────────
print()
print(f"━━━ 🔄 Fetching {arr_label} Tracked Files ━━━")
if not check_api(url, arr_label):
_notify(f"{arr_label} cleanup aborted on {hostname} — API unreachable")
sys.exit(1)
# ── Safety gate 3 — API version ───────────────────────────────────────────
if not check_arr_version(url, api_key, api_version, ver_major, arr_label):
_notify(f"{arr_label} version mismatch on {hostname} — check master.conf")
sys.exit(1)
# ── Fetch tracked paths ────────────────────────────────────────────────────
print(f" Querying {arr_label} API...")
tracked, parent_count, tracked_count = fetch_tracked_paths(
url, api_key, api_version, profile, path_map, arr_label,
)
# ── Safety gate 4 — parent count > 0 ──────────────────────────────────────
if tracked is None or parent_count == 0:
msg = f"API returned 0 {profile['parent_label']} — aborting to prevent mass deletion"
_notify(f"{arr_label} cleanup aborted on {hostname} — 0 {profile['parent_label']} returned")
die(msg)
# ── Safety gate 5 — tracked count > 0 ────────────────────────────────────
if tracked_count == 0:
_notify(f"{arr_label} cleanup aborted on {hostname} — 0 tracked files returned")
die("API returned 0 tracked files — aborting to prevent mass deletion")
print(f" {parent_count} {profile['parent_label']} | {tracked_count} tracked files")
log(f"Built in-memory lookup set: {tracked_count} tracked paths")
# ── Safety gate 6 — tracked % drop (only if configured) ──────────────────
if count_file and min_pct > 0:
count_path = Path(count_file)
if count_path.exists():
try:
last = int(count_path.read_text().strip())
if last > 0:
pct = int((tracked_count / last) * 100)
if pct < min_pct:
error(f"Tracked count dropped to {pct}% of last run ({tracked_count} vs {last})")
error(f"Suggests API issue — aborting to prevent mass deletion")
error(f"If expected (large removal) delete: {count_file}")
_notify(f"{arr_label} cleanup aborted on {hostname} — tracked count dropped to {pct}%")
sys.exit(1)
log(f"Tracked count: {pct}% of last run ({tracked_count} vs {last}) ✅")
except (ValueError, OSError):
log("Could not read previous count — skipping % check")
else:
log("No previous count on record — first run, saving baseline")
try:
count_path.write_text(str(tracked_count))
except OSError as e:
warn(f"Could not write tracked count file: {e}")
# ── Scan media root ────────────────────────────────────────────────────────
print()
print(f"━━━ 🧹 Scanning Media Root ━━━")
print(f" Root: {media_root} | Orphan age: {orphan_age} days")
print()
start = time.time()
orphan_count = junk_count = recent_count = protected_count = 0
orphan_bytes = junk_bytes = 0
age_threshold = orphan_age * 86400
now = time.time()
max_del_bytes = int(max_del_gb * 1_073_741_824)
scan_roots = set(path_map.values()) | {media_root}
all_files = []
for root in scan_roots:
if Path(root).is_dir():
for fp in Path(root).rglob("*"):
if fp.is_file():
all_files.append(str(fp))
all_files = sorted(set(all_files))
for filepath in all_files:
if filepath in tracked:
log(f"TRACKED: {filepath}")
continue
if is_protected(filepath, protected):
log(f"🔰 PROTECTED: {filepath}")
protected_count += 1
continue
try:
st = Path(filepath).stat()
except OSError:
continue
file_size = st.st_size
if is_media_file(filepath, extensions):
file_age = now - st.st_mtime
if file_age < age_threshold and not args.skip_strikes:
log(f"RECENT (skipping): {filepath}")
recent_count += 1
continue
warn(f"🗑️ ORPHAN: {filepath}")
orphan_count += 1
orphan_bytes += file_size
else:
log(f"JUNK: {filepath}")
junk_count += 1
junk_bytes += file_size
total_del_bytes = orphan_bytes + junk_bytes
total_removed = orphan_count + junk_count
# ── Safety gate 7 — deletion size threshold ───────────────────────────────
if total_del_bytes > max_del_bytes:
total_human = format_bytes(total_del_bytes)
if not args.i_know:
print()
error(f"Deletion would exceed {max_del_gb}GB — {total_human} would be deleted")
error("Review ORPHAN lines above carefully before proceeding")
error("Rerun with: --i-know-what-im-doing")
error("To also bypass age check: add --skip-strike-list")
_notify(f"{arr_label} cleanup halted on {hostname}{total_human} requires --i-know-what-im-doing")
sys.exit(1)
else:
warn(f"OVERRIDE — deletion is {total_human} — proceeding with --i-know-what-im-doing")
# ── Execute deletions ──────────────────────────────────────────────────────
if not args.dry_run:
for filepath in all_files:
if filepath in tracked:
continue
if is_protected(filepath, protected):
continue
try:
st = Path(filepath).stat()
file_age = now - st.st_mtime
except OSError:
continue
if is_media_file(filepath, extensions):
if file_age < age_threshold and not args.skip_strikes:
continue
try:
Path(filepath).unlink()
except OSError as e:
error(f"Failed to delete: {filepath}{e}")
log("Cleaning up empty folders...")
for root in scan_roots:
if Path(root).is_dir():
for d in sorted(Path(root).rglob("*"), key=lambda p: len(p.parts), reverse=True):
if d.is_dir():
try:
d.rmdir()
except OSError:
pass
log("Empty folders removed")
elapsed = int(time.time() - start)
# ── Summary ────────────────────────────────────────────────────────────────
orphan_human = format_bytes(orphan_bytes)
junk_human = format_bytes(junk_bytes)
print()
print(f"━━━━━ 📋 {arr_label.upper()} CLEANUP SUMMARY ━━━━━")
print(f"🖥️ Identity: {my_id} ({server_name})")
print(f"🔄 Tracked: {tracked_count} files ({parent_count} {profile['parent_label']})")
print(f"🛡️ Protected: {protected_count} files (cover art, metadata)")
print(f"🗑️ Orphans: {orphan_count} files ({orphan_human})")
print(f"🗑️ Junk: {junk_count} files ({junk_human})")
print(f"⏭️ Recent skipped: {recent_count} files (under {orphan_age} days)")
print(f"⏱️ Duration: {format_duration(elapsed)}")
print()
if args.dry_run:
warn("DRY RUN — no files deleted")
elif total_removed == 0:
success("Clean — nothing to remove")
else:
warn(f"🏁 Removed {total_removed} files (orphans: {orphan_human} junk: {junk_human})")
_notify(
f"{arr_label} cleanup on {hostname} — removed {total_removed} files "
f"(orphans: {orphan_human} junk: {junk_human})"
)
notify_emby_scan(emby_url, emby_api_key, my_id)
print("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━")
# ── Write stats for coffee report ─────────────────────────────────────────
if not args.dry_run and stats_file:
today = datetime.date.today().strftime("%Y-%m-%d")
line = f"{today}|{ARR}|{orphan_count}|{orphan_bytes}|{junk_count}|{junk_bytes}|{recent_count}|{tracked_count}\n"
try:
with open(stats_file, "a") as f:
f.write(line)
except OSError:
pass
if __name__ == "__main__":
main()
-116
View File
@@ -1,116 +0,0 @@
#!/bin/bash
# ==============================================================================================
# ================================= Arr Cleanup Launcher =======================================
# ==============================================================================================
#
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# Thin bash launcher for arr_cleanup.py. Handles everything bash is uniquely
# suited for: sourcing shell config, detect_hosts(), exporting env vars.
# All logic lives in Python.
#
# USAGE
# ─────────────────────────────────────────────────────────────────────────────
# arr_cleanup.sh --arr lidarr [--dry-run] [--log] [--status]
# arr_cleanup.sh --arr radarr [--i-know-what-im-doing] [--skip-strike-list]
# arr_cleanup.sh --arr sonarr
#
# ADDING A NEW ARR
# ─────────────────────────────────────────────────────────────────────────────
# 1. Add HOST*_<ARR>_URL, API_KEY, MEDIA_ROOT, PATH_MAP to host*.conf
# 2. Add <ARR>_ORPHAN_AGE, MAX_DELETE_GB, EXTENSIONS, etc. to master.conf
# 3. Add a profile entry to ARR_PROFILES in arr_cleanup.py (6 values)
# 4. Add an export block for the new arr below (copy Sonarr block, change prefix)
#
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../load_config.sh"
if [[ "$EUID" -ne 0 ]]; then
echo "ERROR: Must be run as root" >&2
exit 1
fi
acquire_lock
if ! command -v python3 >/dev/null 2>&1; then
echo "ERROR: python3 not found — required for arr_cleanup" >&2
exit 1
fi
detect_hosts
# ── Helper: serialize host-specific associative array to JSON ──────────────────
# Reads ${MY_ID}_${ARR_UPPER}_PATH_MAP and emits {"container_path":"host_path",...}
# Paths with double-quotes in names are not supported (not a real-world constraint).
_path_map_json() {
local arr_upper="$1"
local map_var="${MY_ID}_${arr_upper}_PATH_MAP"
local json="{"
local sep="" keys k v
if ! declare -p "$map_var" 2>/dev/null | grep -q "declare -A"; then
echo "{}"
return
fi
eval "keys=(\"\${!${map_var}[@]}\")"
for k in "${keys[@]}"; do
eval "v=\"\${${map_var}[\$k]}\""
json+="${sep}\"${k}\":\"${v}\""
sep=","
done
json+="}"
echo "$json"
}
# ── Host identity ──────────────────────────────────────────────────────────────
export MY_ID LOCAL_SERVER_NAME
export ARR_HOSTNAME
ARR_HOSTNAME=$(hostname)
# ── Notifications ──────────────────────────────────────────────────────────────
export NOTIFY_UNRAID
export EMBY_URL EMBY_API_KEY
export MY_DISCORD_WEBHOOK
# ── Shared ─────────────────────────────────────────────────────────────────────
export ARR_CLEANUP_STATS
export LOCK_DIR LOCK_WAIT_TIMEOUT
# ── Lidarr ─────────────────────────────────────────────────────────────────────
export LIDARR_URL LIDARR_API_KEY
export LIDARR_MEDIA_ROOT="${LIDARR_MUSIC_ROOT:-}"
export LIDARR_EXTENSIONS="${LIDARR_EXTENSIONS[*]:-}"
export LIDARR_PROTECTED_PATTERNS="${LIDARR_PROTECTED_PATTERNS[*]:-}"
export LIDARR_ORPHAN_AGE LIDARR_MAX_DELETE_GB
export LIDARR_MIN_TRACKED_PCT LIDARR_TRACKED_COUNT_FILE
export LIDARR_VERSION_MAJOR LIDARR_IMPORT_SCAN_TIMEOUT LIDARR_LOCK_WARN_AGE
export LIDARR_PATH_MAP_JSON
LIDARR_PATH_MAP_JSON=$(_path_map_json "LIDARR")
# ── Radarr ─────────────────────────────────────────────────────────────────────
export RADARR_URL RADARR_API_KEY
export RADARR_MEDIA_ROOT="${RADARR_MOVIES_ROOT:-}"
export RADARR_EXTENSIONS="${RADARR_EXTENSIONS[*]:-}"
export RADARR_PROTECTED_PATTERNS="${RADARR_PROTECTED_PATTERNS[*]:-}"
export RADARR_ORPHAN_AGE RADARR_MAX_DELETE_GB
export RADARR_MIN_TRACKED_PCT RADARR_TRACKED_COUNT_FILE
export RADARR_VERSION_MAJOR RADARR_IMPORT_SCAN_TIMEOUT RADARR_LOCK_WARN_AGE
export RADARR_PATH_MAP_JSON
RADARR_PATH_MAP_JSON=$(_path_map_json "RADARR")
# ── Sonarr ─────────────────────────────────────────────────────────────────────
export SONARR_URL SONARR_API_KEY
export SONARR_MEDIA_ROOT="${SONARR_TV_ROOT:-}"
export SONARR_EXTENSIONS="${SONARR_EXTENSIONS[*]:-}"
export SONARR_PROTECTED_PATTERNS="${SONARR_PROTECTED_PATTERNS[*]:-}"
export SONARR_ORPHAN_AGE SONARR_MAX_DELETE_GB
export SONARR_MIN_TRACKED_PCT SONARR_TRACKED_COUNT_FILE
export SONARR_VERSION_MAJOR SONARR_IMPORT_SCAN_TIMEOUT SONARR_LOCK_WARN_AGE
export SONARR_PATH_MAP_JSON
SONARR_PATH_MAP_JSON=$(_path_map_json "SONARR")
exec python3 "$SCRIPT_DIR/arr_cleanup.py" "$@"
@@ -1,517 +0,0 @@
#!/bin/bash
# ==============================================================================================
# ========================= Continuous Scripts Status ==========================================
# ==============================================================================================
#
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# Live status dashboard for all continuously running scripts in the ecosystem.
# Run manually at any time — no schedule, no cron.
#
# For each script shows: running state, PID, uptime, approximate cycle count,
# active strikes, skip list, recent restart history, and a live health snapshot.
#
# system_watchdog — rootfs, RAM, ZFS ARC, load, zombie count, CPU temp
# docker_watchdog — running/stopped/unhealthy containers, required containers,
# memory-monitored containers, recent restart history
# failover — current state, tier status, remote Tailscale visibility
#
# If a script is mid-cycle, state files are read as-is — reflects last completed cycle.
#
# ==============================================================================================
# OPERATIONAL SAFEGUARDS
# ==============================================================================================
#
# Host-Aware Output
# detect_hosts() sets MY_ID and aliases all HOST*_WATCHDOG_* arrays.
# Required containers and tier delays are shown for the correct host.
#
# Read-Only
# Reads state files and docker inspect output only — makes no changes to any
# running script, container, or state file.
#
# ==============================================================================================
# RUNTIME MODES
# ==============================================================================================
#
# continuous_scripts_status.sh
# Show the full dashboard for all continuous scripts.
#
# continuous_scripts_status.sh --log
# Verbose output with additional detail per script section.
#
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../load_config.sh"
# Dashboard script — output is the point
SILENT_MODE=false
parse_args "$@"
DOCKER_TIMEOUT=15
# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
error "Must be run as root"
exit 1
fi
acquire_lock
# detect_hosts() sets MY_ID and aliases all HOST*_WATCHDOG_* arrays
detect_hosts
# ==============================================================================================
# ── HELPER FUNCTIONS ──────────────────────────────────────────────────────────────────────────
# ==============================================================================================
get_lock_pid() {
local script_name="$1"
local lockfile="$LOCK_DIR/${script_name}.lock"
if [[ -f "$lockfile" ]]; then
local content
content=$(cat "$lockfile" 2>/dev/null)
echo "${content%%:*}"
fi
}
get_lock_name() {
local script_name="$1"
local lockfile="$LOCK_DIR/${script_name}.lock"
if [[ -f "$lockfile" ]]; then
local content
content=$(cat "$lockfile" 2>/dev/null)
echo "${content##*:}"
fi
}
is_script_running() {
local script_name="$1"
local pid locked_name
pid=$(get_lock_pid "$script_name")
locked_name=$(get_lock_name "$script_name")
[[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null && [[ "$locked_name" == "$script_name" ]]
}
get_lock_age() {
local script_name="$1"
local lockfile="$LOCK_DIR/${script_name}.lock"
if [[ -f "$lockfile" ]]; then
local mtime now
mtime=$(stat -c %Y "$lockfile" 2>/dev/null || echo 0)
now=$(date +%s)
echo $(( now - mtime ))
else
echo 0
fi
}
# Human readable uptime — days/hours/mins
format_uptime() {
local seconds=$1
local days=$(( seconds / 86400 ))
local hours=$(( (seconds % 86400) / 3600 ))
local mins=$(( (seconds % 3600) / 60 ))
if (( days > 0 )); then
echo "${days}d ${hours}h ${mins}m"
elif (( hours > 0 )); then
echo "${hours}h ${mins}m"
else
echo "${mins}m"
fi
}
divider() { printf '%.0s─' {1..57}; echo; }
section() { echo ""; echo " $1"; divider; }
# ==============================================================================================
# ━━━ Header ━━━
# ==============================================================================================
clear
echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo " 🛡️ WATCHDOG STATUS — $(date '+%A, %B %-d at %-I:%M%p')"
echo " $ICON_HOST $MY_ID$LOCAL_SERVER_NAME"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
# ==============================================================================================
# ━━━ System Watchdog ━━━
# ==============================================================================================
section "⚙️ SYSTEM WATCHDOG"
SYS_PID=$(get_lock_pid "system_watchdog")
SYS_RUNNING=false
if is_script_running "system_watchdog"; then
SYS_RUNNING=true
SYS_AGE=$(get_lock_age "system_watchdog")
SYS_UPTIME=$(format_uptime "$SYS_AGE")
SYS_CYCLE=$(( SYS_AGE / SYSTEM_WATCHDOG_INTERVAL ))
echo " ✅ Running │ PID: $SYS_PID │ Uptime: $SYS_UPTIME │ ~Cycle: $SYS_CYCLE"
echo " ⏱️ Interval: ${SYSTEM_WATCHDOG_INTERVAL}s │ Heartbeat every: ${SYSTEM_WATCHDOG_HEARTBEAT_HOURS}hr"
else
echo " ❌ NOT RUNNING — system_watchdog.sh is not active"
echo " Start via: bash Orchestrators/array_started.sh"
fi
echo ""
# System strikes
if [[ -f "$SYS_WATCHDOG_STATE_FILE" ]]; then
ACTIVE_STRIKES=$(grep -v ":0$" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$")
if [[ -n "$ACTIVE_STRIKES" ]]; then
echo " ⚠️ Active strikes:"
while IFS=: read -r key count; do
[[ -z "$key" ]] && continue
echo "$key: $count/$SYS_WATCHDOG_STRIKE_LIMIT"
done <<< "$ACTIVE_STRIKES"
else
echo " ✅ Strikes: none"
fi
else
echo " ️ Strike state file not found (watchdog may not have run yet)"
fi
# Reboot log
if [[ -f "$SYS_WATCHDOG_REBOOT_LOG" ]]; then
TOTAL_REBOOTS=$(grep -c "." "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null || echo 0)
TOTAL_REBOOTS="${TOTAL_REBOOTS//[^0-9]/}"
TOTAL_REBOOTS="${TOTAL_REBOOTS:-0}"
WEEK_CUTOFF=$(date -d "7 days ago" '+%Y-%m-%d %H:%M:%S')
WEEK_REBOOTS=$(awk -v cutoff="$WEEK_CUTOFF" '$0 >= cutoff' \
"$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null | wc -l)
echo " 🔄 Watchdog reboots: $WEEK_REBOOTS this week / $TOTAL_REBOOTS total"
fi
# Container skip list
if [[ -f "$DOCKER_WATCHDOG_FAILED_FILE" ]] && [[ -s "$DOCKER_WATCHDOG_FAILED_FILE" ]]; then
SKIP_COUNT=$(wc -l < "$DOCKER_WATCHDOG_FAILED_FILE")
echo ""
echo " ⛔ Skip list ($SKIP_COUNT — manual intervention needed):"
while IFS= read -r container; do
[[ -z "$container" ]] && continue
echo "$container"
done < "$DOCKER_WATCHDOG_FAILED_FILE"
else
echo " ✅ Skip list: empty"
fi
# Live system health snapshot
echo ""
echo " 📊 Current system state:"
ROOTFS_PCT=$(df / --output=pcent 2>/dev/null | tail -1 | tr -d ' %')
[[ "${ROOTFS_PCT:-0}" -ge "${SYS_WATCHDOG_ROOTFS_PCT:-95}" ]] && \
ROOTFS_ICON="⚠️ " || ROOTFS_ICON="✅"
echo " ${ROOTFS_ICON} rootfs: ${ROOTFS_PCT}% (threshold: ${SYS_WATCHDOG_ROOTFS_PCT}%)"
MEM_AVAIL_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo)
MEM_FREE_GB=$(awk "BEGIN {printf \"%.1f\", $MEM_AVAIL_KB / 1048576}")
MEM_TOTAL_GB=$(awk '/MemTotal/ {printf "%.0f", $2/1048576}' /proc/meminfo)
[[ $(printf "%.0f" "$MEM_FREE_GB") -lt "${SYS_WATCHDOG_MEM_GB:-4}" ]] && \
MEM_ICON="⚠️ " || MEM_ICON="✅"
echo " ${MEM_ICON} RAM: ${MEM_FREE_GB}GB free / ${MEM_TOTAL_GB}GB total (threshold: ${SYS_WATCHDOG_MEM_GB}GB free)"
if [[ -f /proc/spl/kstat/zfs/arcstats ]]; then
ARC_SIZE=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats)
ARC_MAX=$(awk '/^c_max / {print $3}' /proc/spl/kstat/zfs/arcstats)
ARC_PCT=$(( ARC_SIZE * 100 / ARC_MAX ))
ARC_GB=$(awk "BEGIN {printf \"%.1f\", $ARC_SIZE / 1073741824}")
[[ "$ARC_PCT" -ge "${SYS_WATCHDOG_ARC_PINNED_PCT:-98}" ]] && \
ARC_ICON="⚠️ " || ARC_ICON="✅"
echo " ${ARC_ICON} ZFS ARC: ${ARC_GB}GB (${ARC_PCT}% of max, threshold: ${SYS_WATCHDOG_ARC_PINNED_PCT}%)"
fi
LOAD=$(awk '{print $1}' /proc/loadavg)
CORES=$(nproc)
LOAD_THRESH=$(( CORES * ${SYS_WATCHDOG_LOAD_MULTIPLIER:-3} ))
LOAD_INT=$(printf "%.0f" "$LOAD")
[[ "$LOAD_INT" -ge "$LOAD_THRESH" ]] && LOAD_ICON="⚠️ " || LOAD_ICON="✅"
echo " ${LOAD_ICON} Load avg: $LOAD (threshold: ${LOAD_THRESH} = ${SYS_WATCHDOG_LOAD_MULTIPLIER}x ${CORES} cores)"
ZOMBIE_COUNT=$(ps aux | awk '{print $8}' | grep -c "^Z$" 2>/dev/null || echo 0)
ZOMBIE_COUNT="${ZOMBIE_COUNT//[^0-9]/}"
ZOMBIE_COUNT="${ZOMBIE_COUNT:-0}"
[[ "$ZOMBIE_COUNT" -ge "${SYS_WATCHDOG_ZOMBIE_LIMIT:-50}" ]] && \
ZOMBIE_ICON="⚠️ " || ZOMBIE_ICON="✅"
echo " ${ZOMBIE_ICON} Zombies: $ZOMBIE_COUNT (threshold: ${SYS_WATCHDOG_ZOMBIE_LIMIT})"
if command -v sensors >/dev/null 2>&1; then
CPU_TEMP=$(sensors 2>/dev/null | \
grep -i "Package id 0\|Tctl\|CPU Temp" | \
awk '{print $NF}' | tr -d '+°C' | head -1)
if [[ -n "$CPU_TEMP" ]]; then
CPU_TEMP_INT=$(printf "%.0f" "$CPU_TEMP")
[[ "$CPU_TEMP_INT" -ge "${SYS_WATCHDOG_CPU_TEMP_MAX:-95}" ]] && \
TEMP_ICON="⚠️ " || TEMP_ICON="✅"
echo " ${TEMP_ICON} CPU temp: ${CPU_TEMP_INT}°C (threshold: ${SYS_WATCHDOG_CPU_TEMP_MAX}°C)"
fi
fi
# ==============================================================================================
# ━━━ Docker Watchdog ━━━
# ==============================================================================================
section "🐳 DOCKER WATCHDOG"
DOCKER_PID=$(get_lock_pid "docker_watchdog")
DOCKER_RUNNING=false
if is_script_running "docker_watchdog"; then
DOCKER_RUNNING=true
DOCKER_AGE=$(get_lock_age "docker_watchdog")
DOCKER_UPTIME=$(format_uptime "$DOCKER_AGE")
DOCKER_CYCLE=$(( DOCKER_AGE / DOCKER_WATCHDOG_INTERVAL ))
echo " ✅ Running │ PID: $DOCKER_PID │ Uptime: $DOCKER_UPTIME │ ~Cycle: $DOCKER_CYCLE"
echo " ⏱️ Interval: ${DOCKER_WATCHDOG_INTERVAL}s │ Heartbeat every: ${DOCKER_WATCHDOG_HEARTBEAT_HOURS}hr"
else
echo " ❌ NOT RUNNING — docker_watchdog.sh is not active"
echo " Start via: bash Orchestrators/array_started.sh"
fi
echo ""
# Container strikes
if [[ -f "$WATCHDOG_STATE_FILE" ]]; then
ACTIVE_CONTAINER_STRIKES=$(grep -v ":0$" "$WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$")
if [[ -n "$ACTIVE_CONTAINER_STRIKES" ]]; then
echo " ⚠️ Active container strikes:"
while IFS=: read -r key count; do
[[ -z "$key" ]] && continue
echo "$key: $count"
done <<< "$ACTIVE_CONTAINER_STRIKES"
else
echo " ✅ Container strikes: none"
fi
fi
# Container restart history
if [[ -f "$WATCHDOG_CONTAINER_RESTART_LOG" ]]; then
WEEK_CUTOFF=$(date -d "7 days ago" '+%Y-%m-%d %H:%M:%S')
WEEK_RESTARTS=$(awk -F'|' -v cutoff="$WEEK_CUTOFF" \
'$2 >= cutoff' "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | wc -l)
if [[ "${WEEK_RESTARTS:-0}" -gt 0 ]]; then
echo ""
echo " 🔄 Container restarts this week: $WEEK_RESTARTS"
awk -F'|' -v cutoff="$WEEK_CUTOFF" \
'$2 >= cutoff {print $1}' "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | \
sort | uniq -c | sort -rn | head -5 | \
while read -r count name; do
echo "$name: $count restart(s)"
done
else
echo " ✅ Container restarts this week: none"
fi
fi
# Container overview
echo ""
echo " 📦 Container overview:"
if command -v docker >/dev/null 2>&1; then
RUNNING=$(timeout "$DOCKER_TIMEOUT" docker ps -q 2>/dev/null | wc -l)
TOTAL=$(timeout "$DOCKER_TIMEOUT" docker ps -aq 2>/dev/null | wc -l)
UNHEALTHY=$(timeout "$DOCKER_TIMEOUT" docker ps \
--filter health=unhealthy -q 2>/dev/null | wc -l)
# Stopped containers — bucket into clean vs unexpected, skip SCAN_IGNORE entirely
CLEAN_STOPPED=()
UNEXPECTED_STOPPED=()
while IFS= read -r name; do
[[ -z "$name" ]] && continue
SKIP=false
for ignore in "${WATCHDOG_SCAN_IGNORE[@]}"; do
[[ "$name" == "$ignore" ]] && SKIP=true && break
done
[[ "$SKIP" == true ]] && continue
exit_code=$(docker inspect --format '{{.State.ExitCode}}' "$name" 2>/dev/null)
if [[ "$exit_code" == "0" || "$exit_code" == "143" ]]; then
CLEAN_STOPPED+=("$name")
else
UNEXPECTED_STOPPED+=("$name")
fi
done < <(timeout "$DOCKER_TIMEOUT" docker ps -af "status=exited" \
--format "{{.Names}}" 2>/dev/null)
echo " Running: $RUNNING / $TOTAL total"
[[ "$UNHEALTHY" -gt 0 ]] && echo " ⚠️ Unhealthy: $UNHEALTHY"
if [[ "${#UNEXPECTED_STOPPED[@]}" -gt 0 ]]; then
echo " ⚠️ Stopped (unexpected):"
for name in "${UNEXPECTED_STOPPED[@]}"; do
echo "$name"
done
fi
if [[ "${#CLEAN_STOPPED[@]}" -gt 0 ]]; then
echo " ⏸️ Stopped (clean):"
for name in "${CLEAN_STOPPED[@]}"; do
echo "$name"
done
fi
if [[ "${#UNEXPECTED_STOPPED[@]}" -eq 0 && "${#CLEAN_STOPPED[@]}" -eq 0 ]]; then
echo " ✅ All containers running"
fi
# Required containers — aliased by detect_hosts() → WATCHDOG_REQUIRED_CONTAINERS
REQUIRED_ISSUES=0
if [[ ${#WATCHDOG_REQUIRED_CONTAINERS[@]} -gt 0 ]]; then
echo ""
echo " 🔐 Required containers:"
for container in "${WATCHDOG_REQUIRED_CONTAINERS[@]}"; do
[[ -z "$container" ]] && continue
STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
'{{.State.Running}}' "$container" 2>/dev/null || echo "not found")
if [[ "$STATUS" == "true" ]]; then
echo "$container"
else
echo "$container$STATUS"
(( REQUIRED_ISSUES++ ))
fi
done
fi
# Memory-monitored containers — aliased by detect_hosts() → WATCHDOG_CONTAINERS
if [[ ${#WATCHDOG_CONTAINERS[@]} -gt 0 ]]; then
echo ""
echo " 📊 Monitored containers (memory):"
for container in "${!WATCHDOG_CONTAINERS[@]}"; do
LIMIT_MB="${WATCHDOG_CONTAINERS[$container]}"
LIMIT_GB=$(awk "BEGIN {printf \"%.0f\", $LIMIT_MB / 1024}")
USAGE=$(timeout "$DOCKER_TIMEOUT" docker stats --no-stream \
--format "{{.MemUsage}}" "$container" 2>/dev/null | awk '{print $1}')
STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
'{{.State.Running}}' "$container" 2>/dev/null || echo "not found")
if [[ "$STATUS" == "true" ]]; then
echo "$container: ${USAGE:-?} (limit: ${LIMIT_GB}GB)"
else
echo "$container: not running (limit: ${LIMIT_GB}GB)"
fi
done
fi
else
echo " Docker not available"
fi
# ==============================================================================================
# ━━━ Failover ━━━
# ==============================================================================================
section "🔀 FALLBACK"
FALLBACK_PID=$(get_lock_pid "fallback")
FALLBACK_RUNNING=false
if is_script_running "fallback"; then
FALLBACK_RUNNING=true
FALLBACK_AGE=$(get_lock_age "fallback")
FALLBACK_UPTIME=$(format_uptime "$FALLBACK_AGE")
echo " ✅ Running │ PID: $FALLBACK_PID │ Uptime: $FALLBACK_UPTIME"
else
if [[ "${FALLBACK_ENABLED:-true}" == false ]]; then
echo " ⏸️ Disabled — FALLBACK_ENABLED=false in master.conf"
else
echo " ❌ NOT RUNNING — fallback.sh is not active"
echo " Start via: bash Orchestrators/array_started.sh"
fi
fi
echo ""
# Fallback state
FALLBACK_STATE="UNKNOWN"
FALLBACK_STATE_SECONDS=0
if [[ -f "$FALLBACK_STATE_FILE" ]]; then
FALLBACK_STATE=$(grep "^state=" "$FALLBACK_STATE_FILE" 2>/dev/null | cut -d= -f2)
FALLBACK_LAST_EPOCH=$(grep "^fallback_start=" "$FALLBACK_STATE_FILE" \
2>/dev/null | cut -d= -f2)
if [[ -n "$FALLBACK_LAST_EPOCH" && "$FALLBACK_LAST_EPOCH" -gt 0 ]]; then
FALLBACK_STATE_SECONDS=$(( $(date +%s) - FALLBACK_LAST_EPOCH ))
fi
fi
STATE_DURATION=$(format_uptime "${FALLBACK_STATE_SECONDS:-0}")
# Tier delays via REMOTE_ID — same logic as fallback.sh
REMOTE_TIER2_VAR="${REMOTE_ID}_TIER2_DELAY"
REMOTE_TIER3_VAR="${REMOTE_ID}_TIER3_DELAY"
REMOTE_TIER4_VAR="${REMOTE_ID}_TIER4_DELAY"
TIER2_DELAY="${!REMOTE_TIER2_VAR:-240}"
TIER3_DELAY="${!REMOTE_TIER3_VAR:-720}"
TIER4_DELAY="${!REMOTE_TIER4_VAR:-1440}"
case "$FALLBACK_STATE" in
NORMAL)
echo " ✅ State: NORMAL"
;;
FALLBACK)
echo " ⚠️ State: FALLBACK — $REMOTE_SERVER_NAME is down"
echo " ⏱️ Duration: $STATE_DURATION"
FALLBACK_MINS=$(( FALLBACK_STATE_SECONDS / 60 ))
echo ""
echo " 🔄 Tier status:"
echo " Tier 1 (immediate): ✅ active"
if (( FALLBACK_MINS >= TIER2_DELAY )); then
echo " Tier 2 (${TIER2_DELAY}min): ✅ active"
else
REMAINING=$(( TIER2_DELAY - FALLBACK_MINS ))
echo " Tier 2 (${TIER2_DELAY}min): ⏳ in ${REMAINING}min"
fi
if (( FALLBACK_MINS >= TIER3_DELAY )); then
echo " Tier 3 (${TIER3_DELAY}min): ✅ active"
else
REMAINING=$(( TIER3_DELAY - FALLBACK_MINS ))
echo " Tier 3 (${TIER3_DELAY}min): ⏳ in ${REMAINING}min"
fi
if (( FALLBACK_MINS >= TIER4_DELAY )); then
echo " Tier 4 (${TIER4_DELAY}min): ✅ active"
else
REMAINING=$(( TIER4_DELAY - FALLBACK_MINS ))
echo " Tier 4 (${TIER4_DELAY}min): ⏳ in ${REMAINING}min"
fi
;;
NO_INTERNET)
echo " ❌ State: NO_INTERNET — DDNS stopped"
echo " ⏱️ Down for: $STATE_DURATION"
;;
DARK)
echo " ❌ State: DARK — $REMOTE_SERVER_NAME down AND no internet"
echo " ⏱️ Duration: $STATE_DURATION"
;;
*)
echo " ❓ State: ${FALLBACK_STATE:-unknown}"
;;
esac
echo " 📡 Check interval: ${FALLBACK_CHECK_INTERVAL}s │ Handback strikes: ${FALLBACK_HANDBACK_STRIKES}"
# ==============================================================================================
# ━━━ Footer ━━━
# ==============================================================================================
echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
ISSUES=0
[[ "$SYS_RUNNING" == false ]] && (( ISSUES++ ))
[[ "$DOCKER_RUNNING" == false ]] && (( ISSUES++ ))
[[ "$FALLBACK_RUNNING" == false && "${FALLBACK_ENABLED:-true}" != false ]] && (( ISSUES++ ))
[[ -n "$ACTIVE_STRIKES" ]] && (( ISSUES++ ))
[[ -n "$ACTIVE_CONTAINER_STRIKES" ]] && (( ISSUES++ ))
[[ "${REQUIRED_ISSUES:-0}" -gt 0 ]] && (( ISSUES++ ))
[[ "$FALLBACK_STATE" != "NORMAL" && "$FALLBACK_STATE" != "UNKNOWN" ]] && (( ISSUES++ ))
if [[ "$ISSUES" -eq 0 ]]; then
echo "$MY_ID — all continuous scripts healthy"
else
echo " ⚠️ $ISSUES issue(s) detected — review above"
fi
echo " 🕐 Checked: $(date '+%H:%M:%S')"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo ""
+1 -1
View File
@@ -185,7 +185,7 @@ for relative_path in "${ARRAY_START_SCRIPTS[@]}"; do
wait "$PID"
EXIT_CODE=$?
if [[ "$EXIT_CODE" -eq 0 ]]; then
log "$SCRIPT_NAME — completed (one-shot) ✅"
echo "$SCRIPT_NAME — completed (one-shot) ✅"
(( LAUNCHED++ ))
else
error "$SCRIPT_NAME — exited with code $EXIT_CODE"
+1 -1
View File
@@ -140,7 +140,7 @@ for entry in "${ARRAY_STOP_SCRIPTS[@]}"; do
fi
if bash "$script_path" "${extra_args[@]}"; then
log "$script_name — done ✅"
echo "$script_name — done ✅"
PASSED+=("$script_name")
else
warn "$script_name — failed (exit $?) — continuing to next step"
+1 -1
View File
@@ -153,7 +153,7 @@ else
if [[ "$RSYNC_EXIT" -eq 0 ]]; then
PASS+=("$SHARE_NAME")
log "$SHARE_NAME — done in $SHARE_DUR"
echo "$SHARE_NAME — done in $SHARE_DUR"
RSYNC_OK=true
else
FAIL+=("$SHARE_NAME")
+2 -2
View File
@@ -172,7 +172,7 @@ run_job() {
log "Running: $script_name ${extra_args[*]}"
# shellcheck disable=SC2086
if bash "$script_path" "${extra_args[@]}" $extra_dry; then
log "$script_name — done ✅"
echo "$script_name — done ✅"
JOB_PASS+=("$script_name ${extra_args[*]}")
else
error "$script_name — failed (exit $?)"
@@ -271,7 +271,7 @@ else
case "$RSYNC_EXIT" in
0)
PASS+=("$SHARE_NAME")
log "$SHARE_NAME — done ✅"
echo "$SHARE_NAME — done ✅"
;;
1)
FAIL+=("$SHARE_NAME:temp-warn")
@@ -100,7 +100,7 @@ run_job() {
log "Running: $script_name ${extra_args[*]}"
# shellcheck disable=SC2086
if bash "$script_path" "${extra_args[@]}" $extra_dry; then
log "$script_name — done ✅"
echo "$script_name — done ✅"
JOB_PASS+=("$script_name ${extra_args[*]}")
else
warn "$script_name — failed (exit $?)"
@@ -164,7 +164,7 @@ else
_conf_args=("--pull-only")
[[ "$DRY_RUN" == true ]] && _conf_args+=("--dry-run")
if bash "$CONF_SYNC_SCRIPT" "${_conf_args[@]}"; then
log "Partner conf cache refreshed ✅"
echo "Partner conf cache refreshed ✅"
JOB_PASS+=("conf_sync.sh --pull-only")
else
warn "Partner conf pull failed — cache may be stale"
@@ -244,7 +244,7 @@ else
case "$RSYNC_EXIT" in
0)
PASS+=("$SHARE_NAME")
log "$SHARE_NAME — done ✅"
echo "$SHARE_NAME — done ✅"
;;
1)
FAIL+=("$SHARE_NAME:temp-warn")
+1 -1
View File
@@ -233,7 +233,7 @@ for entry in "${MONTHLY_MAINTENANCE_SCRIPTS[@]}"; do
[[ "$VERBOSE" == true ]] && local_args+=("--log")
if bash "$script_path" "${extra_args[@]}" "${local_args[@]}"; then
log "$script_name — done ✅"
echo "$script_name — done ✅"
PASSED+=("$script_name")
else
warn "$script_name — failed (exit $?) — continuing to next step"
@@ -58,7 +58,7 @@ run_job() {
log "Running: $script_name ${extra_args[*]}"
if bash "$script_path" "${extra_args[@]}" $extra_dry $extra_log; then
log "$script_name — done ✅"
echo "$script_name — done ✅"
JOB_PASS+=("$script_name")
else
error "$script_name — failed (exit $?)"
+5 -5
View File
@@ -119,7 +119,7 @@ run_job() {
log "Running: $script_name ${extra_args[*]}"
# shellcheck disable=SC2086
if bash "$script_path" "${extra_args[@]}" $extra_dry; then
log "$script_name — done ✅"
echo "$script_name — done ✅"
JOB_PASS+=("$script_name ${extra_args[*]}")
else
error "$script_name — failed (exit $?)"
@@ -259,7 +259,7 @@ if [[ "$WEEKLY_SYNC_UPDATES_REMOTE" == true ]]; then
-o ConnectTimeout="$SSH_TIMEOUT" \
root@"$REMOTE_SERVER" \
"docker pull $IMAGE" >/dev/null 2>&1; then
log "$c — remote image updated ✅"
echo "$c — remote image updated ✅"
else
warn "$c — remote pull failed, will start on existing image"
fi
@@ -304,7 +304,7 @@ else
if [[ "$EXIT_CODE" -eq 0 ]]; then
PASS+=("$JOB_NAME")
log "$JOB_NAME — done in $JOB_DUR"
echo "$JOB_NAME — done in $JOB_DUR"
else
FAIL+=("$JOB_NAME")
error "$JOB_NAME — failed after $JOB_DUR (exit $EXIT_CODE)"
@@ -343,13 +343,13 @@ else
if [[ -n "${_weekly_needs_rebuild[$_c]:-}" ]]; then
log "Rebuilding $_c on new image..."
if platform_rebuild_container "$_c"; then
log "$_c rebuilt on new image ✅"
echo "$_c rebuilt on new image ✅"
else
warn "$_c rebuild failed — falling back to docker start"
docker start "$_c" >/dev/null 2>&1 || error "Failed to start $_c"
fi
else
docker start "$_c" >/dev/null 2>&1 && log "$_c started" || error "Failed to start $_c"
docker start "$_c" >/dev/null 2>&1 && echo "$_c started" || error "Failed to start $_c"
fi
done
unset _c _d _needs_delay
+2 -2
View File
@@ -205,7 +205,7 @@ HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" \
"$GITEA_API/user" 2>/dev/null)
if [[ "$HTTP_CODE" == "200" ]]; then
log "API token valid ✅"
echo "API token valid ✅"
elif [[ "$HTTP_CODE" == "401" ]]; then
error "API token rejected (HTTP 401) — token may be expired or have wrong scope"
error "Regenerate the token in Gitea: Settings → Applications → Generate Token → scope: write:user"
@@ -436,7 +436,7 @@ PYEOF
echo " IdentityFile $GITEA_SSH_KEY"
} >> "$SSH_CONFIG"
chmod 600 "$SSH_CONFIG"
log "SSH config entry added: Host gitea-${MY_ID,,}"
echo "SSH config entry added: Host gitea-${MY_ID,,}"
fi
END=$(date +%s)
+4 -4
View File
@@ -90,7 +90,7 @@ if [[ "$DIRECTION" == "h1" || "$DIRECTION" == "both" ]]; then
"sed -i \"|${KEY_BLOB}|d\" /root/.ssh/authorized_keys 2>/dev/null
sed -i \"/^${MIRROR_ID}_PHASE\|^${MIRROR_ID}_KEY_READY/d\" $(platform_setup_db_path) 2>/dev/null
echo ok" 2>/dev/null | grep -q ok && {
log "HOST1 key removed from $MIRROR authorized_keys ✅"
echo "HOST1 key removed from $MIRROR authorized_keys ✅"
H1_DONE=true
} || warn "Could not SSH to $MIRROR — remove HOST1 key there manually"
fi
@@ -102,7 +102,7 @@ if [[ "$DIRECTION" == "h1" || "$DIRECTION" == "both" ]]; then
elif [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would delete: $SSH_KEY and ${SSH_KEY}.pub"
else
rm -f "$SSH_KEY" "$SSH_KEY_PUB" && log "Local key pair deleted ✅" || \
rm -f "$SSH_KEY" "$SSH_KEY_PUB" && echo "Local key pair deleted ✅" || \
warn "Failed to delete local key — check permissions"
fi
@@ -110,7 +110,7 @@ if [[ "$DIRECTION" == "h1" || "$DIRECTION" == "both" ]]; then
if [[ -f "$STATE_FILE" ]]; then
if [[ "$DRY_RUN" == false ]]; then
sed -i "/^${MIRROR_ID}_PHASE/d; /^${MIRROR_ID}_KEY_READY/d" "$STATE_FILE"
log "Phase flags cleared from local setup.db ✅"
echo "Phase flags cleared from local setup.db ✅"
else
warn "DRY RUN — would clear ${MIRROR_ID}_PHASE* from setup.db"
fi
@@ -133,7 +133,7 @@ if [[ "$DIRECTION" == "h2" || "$DIRECTION" == "both" ]]; then
H2_DONE=true
else
sed -i "/${MIRROR_SHORT}/Id" "$AUTH_KEYS" && {
log "$MIRROR key removed from HOST1 authorized_keys ✅"
echo "$MIRROR key removed from HOST1 authorized_keys ✅"
H2_DONE=true
} || warn "Failed to remove $MIRROR key from HOST1 authorized_keys"
fi
+17 -17
View File
@@ -357,7 +357,7 @@ push_state_to_remote() {
fi
timeout "$SSH_TIMEOUT" scp -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" \
"$local_file" "root@${remote_ip}:${local_file}" 2>/dev/null && \
log "State file pushed to remote ✅" || \
echo "State file pushed to remote ✅" || \
warn "Could not push state file to remote — will propagate on next sync"
}
@@ -466,7 +466,7 @@ do_ssh_key_revocation() {
> /root/.ssh/authorized_keys.tmp 2>/dev/null \
&& mv /root/.ssh/authorized_keys.tmp /root/.ssh/authorized_keys \
&& echo removed" 2>/dev/null | grep -q removed; then
log "Our pubkey revoked from $REMOTE_SERVER_NAME"
echo "Our pubkey revoked from $REMOTE_SERVER_NAME"
SSH_REVOKE_REMOTE_OK=true
else
warn "Remote revocation failed — revoke manually on $REMOTE_SERVER_NAME:"
@@ -491,7 +491,7 @@ do_ssh_key_revocation() {
if grep -v "@${REMOTE_SERVER_NAME}" /root/.ssh/authorized_keys \
> /root/.ssh/authorized_keys.tmp 2>/dev/null && \
mv /root/.ssh/authorized_keys.tmp /root/.ssh/authorized_keys; then
log "$REMOTE_SERVER_NAME pubkey revoked locally ✅"
echo "$REMOTE_SERVER_NAME pubkey revoked locally ✅"
SSH_REVOKE_LOCAL_OK=true
else
warn "Failed to update local authorized_keys — remove @${REMOTE_SERVER_NAME} entry manually"
@@ -568,7 +568,7 @@ start_own_stack() {
continue
fi
if timeout "${DOCKER_TIMEOUT:-30}" docker start "$container" >/dev/null 2>&1; then
log "$container started ✅"
echo "$container started ✅"
else
warn "$container failed to start — check manually"
fi
@@ -610,7 +610,7 @@ cleanup_partner_containers() {
timeout "${DOCKER_TIMEOUT:-30}" docker stop "$container" >/dev/null 2>&1 || true
_PM_TRAP_STOPPED+=("$container")
timeout "${DOCKER_TIMEOUT:-30}" docker rm "$container" >/dev/null 2>&1 && \
log "$container removed ✅" || warn "$container rm failed"
echo "$container removed ✅" || warn "$container rm failed"
else
log "$container not found — skipping"
fi
@@ -623,7 +623,7 @@ cleanup_partner_containers() {
warn " DRY RUN — would rm -rf $path"
continue
fi
rm -rf "$path" && log " Appdata removed: $path" || warn " Failed to remove: $path"
rm -rf "$path" && echo " Appdata removed: $path" || warn " Failed to remove: $path"
done <<< "$all_appdata_paths"
}
@@ -667,7 +667,7 @@ cleanup_owner_containers_on_mirror() {
"docker stop '$container' >/dev/null 2>&1
docker rm '$container' >/dev/null 2>&1 && echo removed" 2>/dev/null | \
grep -q removed && \
log "$container removed from $MIRROR" || \
echo "$container removed from $MIRROR" || \
warn "Failed to remove $container from $MIRROR"
# Delete appdata on remote after container removal
@@ -676,7 +676,7 @@ cleanup_owner_containers_on_mirror() {
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
-o ConnectTimeout="$SSH_TIMEOUT" root@"$mirror_ip" \
"rm -rf '$path' && echo removed" 2>/dev/null | grep -q removed && \
log " Appdata removed on $MIRROR: $path" || \
echo " Appdata removed on $MIRROR: $path" || \
warn " Failed to remove appdata on $MIRROR: $path"
done <<< "$appdata_paths"
done <<< "$container_list"
@@ -710,7 +710,7 @@ start_mirror_own_stack() {
-o ConnectTimeout="$SSH_TIMEOUT" root@"$mirror_ip" \
"docker start '$container' >/dev/null 2>&1 && echo started" 2>/dev/null | \
grep -q started && \
log "$container started on $MIRROR" || \
echo "$container started on $MIRROR" || \
warn "$container failed to start on $MIRROR — check manually"
done
}
@@ -801,7 +801,7 @@ provision_emby_admin() {
"${emby_url}/Users/${user_id}/Password" 2>/dev/null)
if [[ "$pw_code" == "200" ]] || [[ "$pw_code" == "204" ]]; then
log "Emby admin '$username' created (id: $user_id) ✅"
echo "Emby admin '$username' created (id: $user_id) ✅"
else
warn "User created but password set failed (HTTP $pw_code) — set password manually"
fi
@@ -812,7 +812,7 @@ provision_emby_admin() {
-H "Content-Type: application/json" \
-d '{"IsAdministrator": true, "IsDisabled": false}' \
"${emby_url}/Users/${user_id}/Policy" 2>/dev/null && \
log "$username granted admin policy ✅" || \
echo "$username granted admin policy ✅" || \
warn "Could not set admin policy — grant manually in Emby dashboard"
}
@@ -867,7 +867,7 @@ revoke_emby_admin() {
"${emby_url}/Users/${user_id}" 2>/dev/null)
if [[ "$del_code" == "200" ]] || [[ "$del_code" == "204" ]] || [[ "$del_code" == "404" ]]; then
log "Emby admin '$username' removed ✅"
echo "Emby admin '$username' removed ✅"
else
warn "Failed to delete Emby user '$username' (HTTP $del_code) — remove manually"
fi
@@ -922,7 +922,7 @@ update_master_conf() {
return 1
fi
if sed -i "s|^[[:space:]]*${key}=.*| ${key}=${value}|" "$conf" 2>/dev/null; then
log "master.conf updated: ${key}=${value}"
echo "master.conf updated: ${key}=${value}"
return 0
else
error "Failed to update master.conf: ${key}=${value}"
@@ -1168,7 +1168,7 @@ if [[ "$MODE" == "onboard" ]]; then
else
echo "PARTNERSHIP_ENABLED=true" >> "$_master_conf"
fi
log "PARTNERSHIP_ENABLED=true in master.conf ✅"
echo "PARTNERSHIP_ENABLED=true in master.conf ✅"
platform_push_conf | while IFS= read -r line; do log "$line"; done
else
warn "DRY RUN — would set PARTNERSHIP_ENABLED=true in master.conf and push"
@@ -1184,7 +1184,7 @@ if [[ "$MODE" == "onboard" ]]; then
echo "${flag_key}=true" >> "$local_state_file"
fi
platform_push_setup_state
log "${MY_ID}_LOCAL_DONE=true written to setup.db ✅"
echo "${MY_ID}_LOCAL_DONE=true written to setup.db ✅"
else
warn "DRY RUN — would write ${MY_ID}_LOCAL_DONE=true"
fi
@@ -1259,7 +1259,7 @@ if [[ "$MODE" == "onboard" ]]; then
container="${entry%%|*}"
port="${entry##*|}"
if curl -sf --max-time 10 "http://${OWNER_IP}:${port}/" >/dev/null 2>&1; then
log "$container reachable at http://${OWNER_IP}:${port}/ ✅"
echo "$container reachable at http://${OWNER_IP}:${port}/ ✅"
else
warn "$container not reachable at http://${OWNER_IP}:${port}/ — may not be running"
fi
@@ -1438,7 +1438,7 @@ if false; then
-o ConnectTimeout="$SSH_TIMEOUT" root@"$NEW_MIRROR_IP" \
"sed -i 's|^[[:space:]]*PARTNERSHIP_OWNER_HOST=.*| PARTNERSHIP_OWNER_HOST=\"$NEW_OWNER_ID\"|' \
'$SCRIPT_DIR/../master.conf'" 2>/dev/null && \
log "Remote master.conf updated ✅" || \
echo "Remote master.conf updated ✅" || \
error "Failed to update remote master.conf — update manually"
else
warn "DRY RUN — would set PARTNERSHIP_OWNER_HOST=$NEW_OWNER_ID on both servers"
+4 -4
View File
@@ -205,7 +205,7 @@ revoke_local_emby_admin() {
"${emby_url}/Users/${user_id}" 2>/dev/null)
if [[ "$del_code" == "200" ]] || [[ "$del_code" == "204" ]] || [[ "$del_code" == "404" ]]; then
log "Emby admin '$username' removed ✅"
echo "Emby admin '$username' removed ✅"
else
warn "Failed to delete Emby user '$username' (HTTP $del_code) — remove manually"
fi
@@ -243,7 +243,7 @@ if [[ "$AM_MIRROR" == true ]]; then
if [[ "$DRY_RUN" == false ]]; then
bash "$SCRIPTS_ROOT/Rsync/rsync_stop.sh" --rsync-only 2>/dev/null || true
log "Rsync stopped ✅"
echo "Rsync stopped ✅"
else
warn "DRY RUN — would stop rsync"
fi
@@ -379,7 +379,7 @@ echo "━━━ $ICON_STOP Step 1/10 — Stop Rsync ━━━"
if [[ "$DRY_RUN" == false ]]; then
bash "$SCRIPTS_ROOT/Rsync/rsync_stop.sh" --rsync-only 2>/dev/null || STEP_STOP_OK=false
log "Rsync stopped ✅"
echo "Rsync stopped ✅"
else
warn "DRY RUN — would stop rsync"
fi
@@ -474,7 +474,7 @@ NOW=$(date '+%Y-%m-%d %H:%M:%S')
if [[ "$DRY_RUN" == false ]]; then
write_state_file "$LOCAL_STATE_FILE" \
"INACTIVE" "" "$NOW" "$LOCAL_SERVER_NAME" "$REASON"
log "Local state: INACTIVE ✅"
echo "Local state: INACTIVE ✅"
add_to_blocklist "$MIRROR" "$REASON"
[[ "$MIRROR_REACHABLE" == true ]] && \
push_state_to_remote "$LOCAL_STATE_FILE" "$MIRROR_IP" "$MIRROR_SSH_KEY"
+8 -8
View File
@@ -278,7 +278,7 @@ stop_mirror_stack() {
"docker stop '$container' 2>/dev/null
docker rm '$container' 2>/dev/null && echo removed" 2>/dev/null | \
grep -q removed && \
log " $container removed ✅" || \
echo " $container removed ✅" || \
log " $container not found on $MIRROR — skipping"
done
}
@@ -295,7 +295,7 @@ if [[ "$AM_MIRROR" == true ]]; then
if [[ "$SKIP_SSH" == true ]]; then
warn "Skipping SSH setup (--skip-ssh)"
elif bash "$SCRIPT_DIR/ssh_setup.sh" "${EXTRA_FLAGS[@]}"; then
log "SSH key ready ✅"
echo "SSH key ready ✅"
else
error "SSH key setup failed"
exit 1
@@ -323,7 +323,7 @@ if [[ "$AM_MIRROR" == true ]]; then
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$OWNER_IP" \
"nohup bash '${OWNER_SCRIPTS_DIR}/Partnership/partnership_onboard.sh' --phase2-only > /tmp/vv_phase2_onboard.log 2>&1 & echo triggered" \
2>/dev/null | grep -q triggered; then
log "Phase 2 triggered on $OWNER"
echo "Phase 2 triggered on $OWNER"
log "Watch progress on $OWNER: tail -f /tmp/vv_phase2_onboard.log"
PHASE2_TRIGGERED=true
else
@@ -387,12 +387,12 @@ elif [[ "$PHASE1_ONLY" == true ]]; then
# hanging for a password prompt with no TTY.
if timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$MIRROR_IP" exit 0 2>/dev/null; then
log "SSH to $MIRROR already works ✅ — skipping key install"
echo "SSH to $MIRROR already works ✅ — skipping key install"
STEP_SSH_OK=true
else
# Key not yet on HOST2 — try ssh_setup.sh (works interactively, may fail in background)
if bash "$SCRIPT_DIR/ssh_setup.sh" "${EXTRA_FLAGS[@]}"; then
log "SSH keys ready ✅"
echo "SSH keys ready ✅"
STEP_SSH_OK=true
else
# Soft-fail: generate key locally if not present, then tell user to install manually
@@ -423,7 +423,7 @@ elif [[ "$PHASE1_ONLY" == true ]]; then
fi
fi
elif bash "$SCRIPT_DIR/ssh_setup.sh" "${EXTRA_FLAGS[@]}"; then
log "SSH keys ready ✅"
echo "SSH keys ready ✅"
STEP_SSH_OK=true
else
error "SSH key setup failed — aborting"
@@ -472,7 +472,7 @@ if [[ "$PHASE1_ONLY" == true ]]; then
[[ -n "$push_output" ]] && echo "$push_output"
platform_push_setup_state
if [[ $push_rc -eq 0 ]]; then
log "Conf push complete ✅"
echo "Conf push complete ✅"
CONF_PUSH_OK=true
else
warn "Conf push had failures — retry via Scheduler → master.conf → Save Conf"
@@ -514,7 +514,7 @@ if [[ "$DRY_RUN" == true ]]; then
elif timeout 60 ssh -i "$MIRROR_SSH_KEY" \
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$MIRROR_IP" \
"bash '$_net_script'" 2>/dev/null; then
log "Docker network ready on $MIRROR"
echo "Docker network ready on $MIRROR"
STEP_NETWORK_OK=true
else
warn "docker_network_connect.sh failed on $MIRROR — containers may fail if network is missing"
+1 -1
View File
@@ -296,7 +296,7 @@ if [[ "$DRY_RUN" == false ]]; then
-o StrictHostKeyChecking=no \
"$SCRIPTS_ROOT/Configurations/master.conf" \
"root@${MIRROR_IP}:${_REMOTE_SD}/Configurations/master.conf" 2>/dev/null && \
log "master.conf pushed to $NEW_OWNER" || \
echo "master.conf pushed to $NEW_OWNER" || \
error "Failed to push master.conf to $NEW_OWNER — set PARTNERSHIP_OWNER_HOST=\"$NEW_OWNER_ID\" manually"
else
warn "DRY RUN — would set PARTNERSHIP_OWNER_HOST=$NEW_OWNER_ID on both servers"
+4 -4
View File
@@ -141,7 +141,7 @@ update_conf_key_path() {
if grep -q "^[[:space:]]*${KEY_CONF_VAR}=" "$HOST_CONF" 2>/dev/null; then
sed -i "s|^[[:space:]]*${KEY_CONF_VAR}=.*| ${KEY_CONF_VAR}=\"${SSH_KEY_PATH}\"|" "$HOST_CONF" && \
log "${KEY_CONF_VAR} updated in $(basename "$HOST_CONF")" || \
echo "${KEY_CONF_VAR} updated in $(basename "$HOST_CONF")" || \
warn "Failed to update ${KEY_CONF_VAR} in $(basename "$HOST_CONF") — update manually"
else
warn "${KEY_CONF_VAR} not found in $(basename "$HOST_CONF") — add manually:"
@@ -237,9 +237,9 @@ if [[ "$MODE" == "validate" ]]; then
STRIKES=$(read_strikes)
if [[ "$STRIKES" -gt 0 ]]; then
write_strike_file 0 "" "$NOW"
log "SSH validate — auth restored to $REMOTE_SERVER_NAME ✅ (strikes reset)"
echo "SSH validate — auth restored to $REMOTE_SERVER_NAME ✅ (strikes reset)"
else
log "SSH validate — $REMOTE_SERVER_NAME SSH auth OK ✅"
echo "SSH validate — $REMOTE_SERVER_NAME SSH auth OK ✅"
fi
exit 0
fi
@@ -301,7 +301,7 @@ else
warn "Generating ed25519 keypair: $SSH_KEY_PATH"
if [[ "$DRY_RUN" == false ]]; then
ssh-keygen -t ed25519 -N "" -f "$SSH_KEY_PATH" -C "${SSH_KEY_NAME}@${LOCAL_SERVER_NAME}" && \
log "Keypair generated ✅" || {
echo "Keypair generated ✅" || {
error "Failed to generate keypair"
exit 1
}
+4 -4
View File
@@ -18,7 +18,7 @@ For folder overview see `README-Plugin.md`. For web app logic see the headers in
```bash
cd Plugin/
./dev_install.sh
./plugin_setup.sh
```
This creates:
@@ -94,7 +94,7 @@ All other configuration lives in `Configurations/master.conf` and `Configuration
If the repo is cloned to a new path:
1. Re-run `dev_install.sh` — removes the stale symlink and creates a new one pointing at the new path
1. Re-run `plugin_setup.sh` — removes the stale symlink and creates a new one pointing at the new path
2. Update `SCRIPTS_DIR` in Settings → Other Settings → Varaverk (or edit `varaverk.cfg` directly on flash)
The `.plg` on flash does not need to change — it has no path references.
@@ -115,10 +115,10 @@ management page to reflect when the plugin was last changed:
## ━━━ ADDING A NEW OS ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
`dev_install.sh` is built to support multiple OS targets. To add one:
`plugin_setup.sh` is built to support multiple OS targets. To add one:
1. Create `Plugin/<os>/` with the OS-appropriate web app files
2. Add the OS marker to `detect_os()` in `dev_install.sh`:
2. Add the OS marker to `detect_os()` in `plugin_setup.sh`:
```bash
elif [[ -f /etc/<os>-marker ]]; then echo "<os>"
```
+2 -2
View File
@@ -31,7 +31,7 @@ no separate monitoring stack, no third-party dashboards.
```
Plugin/
├── dev_install.sh # One-time developer setup: symlinks plugin into web server
├── plugin_setup.sh # One-time setup: symlinks plugin into web server
├── Icons/ # Source icon assets (1024px master files)
└── unraid/ # The Unraid platform adapter + plugin application
├── adapter.sh # Platform adapter — provides platform_*() API to all scripts
@@ -75,7 +75,7 @@ so behaviour stays consistent without a shell dependency.
| Script | Role | When It Runs |
|--------|------|--------------|
| `dev_install.sh` | Symlinks `Plugin/unraid/` into Unraid's web server | Once, manually, after cloning or moving the repo |
| `plugin_setup.sh` | Symlinks `Plugin/unraid/` into Unraid's web server | Once, manually, after cloning or moving the repo |
| `build.sh` | Packages the plugin as a `.txz` for release | Before publishing a new plugin version |
---
+1 -1
View File
@@ -7,7 +7,7 @@
# ─────────────────────────────────────────────────────────────────────────────
# Packages the plugin web files (Plugin/unraid/) into a Slackware .txz, the
# format unRAID re-installs from flash on every boot. This is the RELEASE path —
# for day-to-day development use dev_install.sh (symlink, instant edits).
# for day-to-day development use plugin_setup.sh (symlink, instant edits).
#
# What it produces (in Plugin/dist/):
# varaverk-<version>-noarch-1.txz the package unRAID installs to
@@ -1,6 +1,6 @@
#!/bin/bash
# ==============================================================================================
# ============================= dev_install.sh =================================================
# ============================= plugin_setup.sh =================================================
# ==============================================================================================
#
# PURPOSE
@@ -47,10 +47,10 @@
# RUNTIME MODES
# ==============================================================================================
#
# ./dev_install.sh
# ./plugin_setup.sh
# Auto-detects the running OS and installs.
#
# ./dev_install.sh <os>
# ./plugin_setup.sh <os>
# Overrides OS detection. Valid values: unraid, debian, arch.
# Useful when testing on a machine where the marker files differ.
#
@@ -70,7 +70,7 @@ detect_os() {
else echo "unknown"; fi
}
OS="${1:-$(detect_os)}" # accept override: ./dev_install.sh unraid
OS="${1:-$(detect_os)}" # accept override: ./plugin_setup.sh unraid
# ── OS-specific install target ────────────────────────────────────────────────
@@ -81,7 +81,7 @@ case "$OS" in
;;
debian|arch)
echo "OS '$OS' detected but plugin target path not yet defined."
echo "Add the TARGET= line for this OS in dev_install.sh."
echo "Add the TARGET= line for this OS in plugin_setup.sh."
exit 1
;;
unknown|*)
+8 -8
View File
@@ -43,7 +43,7 @@ wait_for_container_healthy() {
case "$status" in
healthy|true)
log " $name ready ✅"
echo " $name ready ✅"
return 0
;;
*)
@@ -159,7 +159,7 @@ deploy_container_from_xml() {
timeout 120 ssh -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" root@"$remote_ip" \
"bash '$remote_script' 2>&1; rc=\$?; rm -f '$remote_script'; exit \$rc" 2>/dev/null | \
grep -q "deployed:${name}"; then
log " $name deployed ✅"
echo " $name deployed ✅"
rm -f "$tmp_script"
return 0
else
@@ -253,7 +253,7 @@ cleanup_deployed_stack_on_remote() {
"docker stop '$cname' >/dev/null 2>&1
docker rm '$cname' >/dev/null 2>&1 && echo removed" 2>/dev/null | \
grep -q removed && \
log " $cname removed from $MIRROR" || \
echo " $cname removed from $MIRROR" || \
log " $cname not found on $MIRROR — skipping"
while IFS= read -r path; do
@@ -261,7 +261,7 @@ cleanup_deployed_stack_on_remote() {
timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
-o ConnectTimeout="$SSH_TIMEOUT" root@"$remote_ip" \
"rm -rf '$path' && echo removed" 2>/dev/null | grep -q removed && \
log " Appdata removed on $MIRROR: $path" || \
echo " Appdata removed on $MIRROR: $path" || \
warn " Failed to remove appdata on $MIRROR: $path"
done <<< "$appdata_paths"
done
@@ -331,14 +331,14 @@ cleanup_deployed_stack_locally() {
timeout "${DOCKER_TIMEOUT:-30}" docker stop "$cname" >/dev/null 2>&1 || true
_PM_TRAP_STOPPED+=("$cname")
timeout "${DOCKER_TIMEOUT:-30}" docker rm "$cname" >/dev/null 2>&1 && \
log " $cname removed ✅" || warn " $cname rm failed"
echo " $cname removed ✅" || warn " $cname rm failed"
else
log " $cname not found locally — skipping"
fi
while IFS= read -r path; do
[[ -z "$path" ]] && continue
rm -rf "$path" && log " Appdata removed: $path" || warn " Failed to remove: $path"
rm -rf "$path" && echo " Appdata removed: $path" || warn " Failed to remove: $path"
done <<< "$appdata_paths"
done
}
@@ -372,7 +372,7 @@ reconfigure_webui() {
-o ConnectTimeout="$SSH_TIMEOUT" root@"$remote_ip" \
"sed -i 's|<WebUI>.*</WebUI>|<WebUI>http://${target_ip}:${port}/</WebUI>|g' '$template'" \
2>/dev/null && \
log "$container → http://${target_ip}:${port}/ ✅" || {
echo "$container → http://${target_ip}:${port}/ ✅" || {
error "Failed to reconfigure $container WebUI on $label"
return 1
}
@@ -408,7 +408,7 @@ reconfigure_local_webuis() {
sed -i "s|<WebUI>.*</WebUI>|<WebUI>http://${target_ip}:${port}/</WebUI>|g" \
"$template" 2>/dev/null && \
log "$container → http://${target_ip}:${port}/ ✅" || \
echo "$container → http://${target_ip}:${port}/ ✅" || \
{ error "Failed to reconfigure $container"; (( failures++ )); }
done
return $failures
@@ -135,7 +135,7 @@ MOVER_PID=$(platform_get_mover_pid)
MOVER_START=$(ps -o lstart= -p "$MOVER_PID" 2>/dev/null | xargs)
MOVER_ELAPSED=$(ps -o etimes= -p "$MOVER_PID" 2>/dev/null | tr -d ' ')
warn "Mover is running (PID $MOVER_PID) — stopping in ${MOVER_STOP_TIMEOUT}s"
log "$ICON_TIME Mover started: ${MOVER_START:-unknown} — running for $(format_duration "${MOVER_ELAPSED:-0}")"
echo "$ICON_TIME Mover started: ${MOVER_START:-unknown} — running for $(format_duration "${MOVER_ELAPSED:-0}")"
# ── Warn users via wall ───────────────────────────────────────────────────────────────────────
if [[ "$DRY_RUN" == false ]]; then
@@ -196,7 +196,7 @@ if ! sed -i "s/^pm\.max_children.*/pm.max_children = $PHP_MAX_CHILDREN/" "$PHP_C
exit 1
fi
log "Config updated"
echo "Config updated"
# ── Restart PHP-FPM ──────────────────────────────────────────────────────────────────────────
log "Restarting PHP-FPM..."
@@ -223,7 +223,7 @@ if [[ "${APPLIED_VAL:-0}" -ne "$PHP_MAX_CHILDREN" ]]; then
warn "Config reads pm.max_children = ${APPLIED_VAL:-unknown} — expected $PHP_MAX_CHILDREN"
warn "Check $PHP_CONF manually"
else
log "Verified: pm.max_children = $APPLIED_VAL"
echo "Verified: pm.max_children = $APPLIED_VAL"
fi
FPM_WORKERS=$(pgrep -fc "php-fpm" 2>/dev/null || echo "?")
@@ -155,7 +155,7 @@ PUSHSCRIPT
if timeout 10 ssh -i "$SSH_KEY" -o ConnectTimeout=10 -o BatchMode=yes \
-o StrictHostKeyChecking=no "root@${partner_ip}" \
"bash '${remote}'; rc=\$?; rm -f '${remote}'; exit \$rc" 2>/dev/null | grep -q ok; then
log "Key pushed to $partner_host"
echo "Key pushed to $partner_host"
else
warn "Key push to $partner_host failed — they can create their own copy"
fi
+3 -3
View File
@@ -154,7 +154,7 @@ if ! mountpoint -q /mnt/user; then
"Recreate Shares" "warning"
exit 1
fi
log "Array is started — /mnt/user is mounted ✅"
echo "Array is started — /mnt/user is mounted ✅"
# Check share cfg directory exists and has files
if [[ ! -d "$SHARE_CFG_DIR" ]]; then
@@ -222,7 +222,7 @@ for cfg in "${CFG_FILES[@]}"; do
warn "DRY RUN — would create: $DISK_PATH"
(( DIRS_CREATED++ ))
elif mkdir -p "$DISK_PATH"; then
log "Created: $DISK_PATH"
echo "Created: $DISK_PATH"
(( DIRS_CREATED++ ))
else
error "Failed to create: $DISK_PATH"
@@ -242,7 +242,7 @@ for cfg in "${CFG_FILES[@]}"; do
warn "DRY RUN — would place marker: $MARKER_PATH"
CREATED+=("$SHARE_NAME")
elif touch "$MARKER_PATH" 2>/dev/null; then
log "Marker placed: $MARKER_PATH"
echo "Marker placed: $MARKER_PATH"
CREATED+=("$SHARE_NAME")
else
warn "$SHARE_NAME — could not place .recovery marker"
+2 -2
View File
@@ -331,7 +331,7 @@ for (( ATTEMPT=1; ATTEMPT<=RETRY_COUNT; ATTEMPT++ )); do
awk '/Total transferred file size:/{gsub(/,/,"",$NF); gsub(/[^0-9]/,"",$NF); print $NF+0}')
BYTES_TRANSFERRED="${BYTES_TRANSFERRED:-0}"
log "$ICON_DONE Rsync complete — $BYTES_TRANSFERRED bytes transferred"
echo "$ICON_DONE Rsync complete — $BYTES_TRANSFERRED bytes transferred"
RSYNC_SUCCESS=true
break
else
@@ -402,7 +402,7 @@ if [[ ${#REMOTE_RESTART_CONTAINERS[@]} -gt 0 ]]; then
timeout 15 ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \
"docker restart $container" >/dev/null 2>&1 && \
log "$ICON_STARTED $container restarted on $REMOTE_SERVER_NAME" || \
echo "$ICON_STARTED $container restarted on $REMOTE_SERVER_NAME" || \
warn "Failed to restart $container on $REMOTE_SERVER_NAME"
done
fi
+3 -3
View File
@@ -72,7 +72,7 @@ if [[ "$PUSH_ONLY" == false ]] && [[ "$PULL_ONLY" == false ]]; then
warn "DRY RUN — would copy $(basename "$MY_CONF")$CACHE_DIR/"
else
cp "$MY_CONF" "$CACHE_DIR/${MY_ID,,}.conf" && \
log "Own conf cached ✅" || warn "Failed to cache own conf"
echo "Own conf cached ✅" || warn "Failed to cache own conf"
fi
else
warn "Own conf not found: $MY_CONF"
@@ -107,7 +107,7 @@ for host_var in $(compgen -v | grep -E '^HOST[0-9]+$' | sort); do
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \
"root@${partner_ip}:${remote_conf}" \
"$CACHE_DIR/${partner_slot}.conf" 2>/dev/null; then
log "Pulled ${partner_slot}.conf from $partner_host"
echo "Pulled ${partner_slot}.conf from $partner_host"
(( PULLED++ ))
else
warn "Could not pull ${partner_slot}.conf from $partner_host"
@@ -134,7 +134,7 @@ for host_var in $(compgen -v | grep -E '^HOST[0-9]+$' | sort); do
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \
"$MY_CONF" \
"root@${partner_ip}:${CACHE_DIR}/${MY_ID,,}.conf" 2>/dev/null; then
log "Pushed ${MY_ID,,}.conf to $partner_host"
echo "Pushed ${MY_ID,,}.conf to $partner_host"
(( PUSHED++ ))
else
warn "Could not push to $partner_host"
+1 -1
View File
@@ -131,7 +131,7 @@ if [[ -z "$REMOTE_SERVER" ]]; then
warn "$REMOTE_SERVER_NAME — cannot resolve Tailscale IP, remote operations will be skipped"
elif timeout "$SSH_TIMEOUT" ping -c1 -W3 "$REMOTE_SERVER" &>/dev/null; then
REMOTE_REACHABLE=true
log "$REMOTE_SERVER_NAME reachable ✅"
echo "$REMOTE_SERVER_NAME reachable ✅"
else
warn "$REMOTE_SERVER_NAME unreachable — remote operations will be skipped"
fi
+2 -2
View File
@@ -247,7 +247,7 @@ elif [[ "$DRY_RUN" == true ]]; then
bash "$ARRAY_STOP_SCRIPT" --dry-run
else
if bash "$ARRAY_STOP_SCRIPT"; then
log "Array stop complete ✅"
echo "Array stop complete ✅"
else
warn "array_stopping.sh reported failures — proceeding with reboot"
fi
@@ -309,7 +309,7 @@ if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would sync filesystem buffers"
else
sync
log "Filesystem buffers flushed ✅"
echo "Filesystem buffers flushed ✅"
fi
# ==============================================================================================
+1 -1
View File
@@ -184,7 +184,7 @@ for share_path in "${PARSED_ARGS[@]}"; do
-exec chmod "${PERMISSIONS_FILE_MODE:-664}" {} + 2>/dev/null || CHMOD_FILE_OK=false
if [[ "$CHOWN_OK" == true && "$CHMOD_DIR_OK" == true && "$CHMOD_FILE_OK" == true ]]; then
log "$ICON_UNLOCKED $(basename "$share_path") — permissions applied ✅"
echo "$ICON_UNLOCKED $(basename "$share_path") — permissions applied ✅"
PASS+=("$(basename "$share_path")")
else
error "$(basename "$share_path") — repair failed"
+5 -5
View File
@@ -175,7 +175,7 @@ case "$STATUS" in
log "Stopping $CONTAINER_NAME for clean export..."
if [[ "$DRY_RUN" == false ]]; then
if timeout "$DOCKER_TIMEOUT" docker stop "$CONTAINER_NAME" >/dev/null 2>&1; then
log "$CONTAINER_NAME stopped ✅"
echo "$CONTAINER_NAME stopped ✅"
else
error "Failed to stop $CONTAINER_NAME — aborting export"
exit 1
@@ -185,7 +185,7 @@ case "$STATUS" in
fi
;;
false)
log "$CONTAINER_NAME is not running — archiving as-is (was stopped state respected)"
echo "$CONTAINER_NAME is not running — archiving as-is (was stopped state respected)"
;;
"")
error "$CONTAINER_NAME not found — check container name"
@@ -224,7 +224,7 @@ if [[ "$DRY_RUN" == false ]]; then
log "Verifying archive..."
if tar --test-label -f "$ARCHIVE_PATH" 2>/dev/null || \
tar -tzf "$ARCHIVE_PATH" >/dev/null 2>&1; then
log "Archive verified ✅"
echo "Archive verified ✅"
ARCHIVE_VERIFIED=true
else
error "Archive verification FAILED — archive may be corrupt"
@@ -260,7 +260,7 @@ if [[ "$CONTAINER_WAS_RUNNING" == true ]]; then
POST_STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
'{{.State.Running}}' "$CONTAINER_NAME" 2>/dev/null)
if [[ "$POST_STATUS" == "true" ]]; then
log "$CONTAINER_NAME restarted and running ✅"
echo "$CONTAINER_NAME restarted and running ✅"
RESTART_OK=true
else
error "$CONTAINER_NAME started but crashed immediately — check container logs"
@@ -277,7 +277,7 @@ if [[ "$CONTAINER_WAS_RUNNING" == true ]]; then
RESTART_OK=true
fi
else
log "$CONTAINER_NAME was not running — leaving stopped (state respected) ✅"
echo "$CONTAINER_NAME was not running — leaving stopped (state respected) ✅"
RESTART_OK=true
fi
+2 -2
View File
@@ -110,13 +110,13 @@ if [[ "$ALL_MODE" == true ]]; then
STOPPED_COUNT=$(echo "$STOPPED_IDS" | grep -c . || echo 0)
if [[ "$STOPPED_COUNT" -eq 0 ]]; then
log "No stopped containers"
echo "No stopped containers"
elif [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would remove $STOPPED_COUNT stopped container(s):"
docker ps -a --filter "status=exited" --filter "status=created" \
--format " {{.Names}} {{.Image}} {{.Status}}" 2>/dev/null
else
log "Removing $STOPPED_COUNT stopped container(s)..."
echo "Removing $STOPPED_COUNT stopped container(s)..."
docker container prune -f 2>&1 | grep -v "^Total\|^$" || true
success "Removed $STOPPED_COUNT stopped container(s)"
fi
+6 -6
View File
@@ -200,7 +200,7 @@ case "$STATUS" in
warn "Stopping $EMBY_CONTAINER — active sessions will be interrupted"
if [[ "$DRY_RUN" == false ]]; then
if timeout "$DOCKER_TIMEOUT" docker stop "$EMBY_CONTAINER" >/dev/null 2>&1; then
log "$EMBY_CONTAINER stopped ✅"
echo "$EMBY_CONTAINER stopped ✅"
sleep 3 # let file handles release
else
error "Failed to stop $EMBY_CONTAINER — aborting"
@@ -263,7 +263,7 @@ for db_rel in "${DB_FILES[@]}"; do
warn "Uncommitted WAL data — will be merged when Emby next starts cleanly"
PASS_DBS+=("$db_name (WAL — see warning)")
else
log "$db_name exists but is empty — no pending transactions ✅"
echo "$db_name exists but is empty — no pending transactions ✅"
PASS_DBS+=("$db_name")
fi
continue
@@ -282,7 +282,7 @@ for db_rel in "${DB_FILES[@]}"; do
error "$db_name — sqlite3 could not open database (locked or corrupt)"
FAIL_DBS+=("$db_name")
elif [[ "$RESULT" == "ok" ]]; then
log "$db_name — integrity check passed ✅"
echo "$db_name — integrity check passed ✅"
PASS_DBS+=("$db_name")
else
error "$db_name — CORRUPTION DETECTED"
@@ -311,7 +311,7 @@ if [[ "$EMBY_WAS_RUNNING" == true ]]; then
POST_STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
'{{.State.Running}}' "$EMBY_CONTAINER" 2>/dev/null)
if [[ "$POST_STATUS" == "true" ]]; then
log "$EMBY_CONTAINER restarted and running ✅"
echo "$EMBY_CONTAINER restarted and running ✅"
RESTART_OK=true
else
error "$EMBY_CONTAINER started but crashed — database may be corrupt"
@@ -329,7 +329,7 @@ if [[ "$EMBY_WAS_RUNNING" == true ]]; then
RESTART_OK=true
fi
else
log "$EMBY_CONTAINER was not running — leaving stopped (state respected) ✅"
echo "$EMBY_CONTAINER was not running — leaving stopped (state respected) ✅"
RESTART_OK=true
fi
@@ -351,7 +351,7 @@ echo " $ICON_SUCCESS Passed: ${#PASS_DBS[@]}"
[[ ${#MISSING_DBS[@]} -gt 0 ]] && echo " Skipped: ${#MISSING_DBS[@]} (not found)"
echo ""
[[ ${#PASS_DBS[@]} -gt 0 ]] && for db in "${PASS_DBS[@]}"; do log " $ICON_SUCCESS $db"; done
[[ ${#PASS_DBS[@]} -gt 0 ]] && for db in "${PASS_DBS[@]}"; do echo " $ICON_SUCCESS $db"; done
[[ ${#FAIL_DBS[@]} -gt 0 ]] && for db in "${FAIL_DBS[@]}"; do echo " $ICON_ERROR $db"; done
echo ""
+1 -1
View File
@@ -219,7 +219,7 @@ fi
echo ""
echo "━━━ $ICON_SUMMARY Artists to add (${#MISSING[@]}) ━━━"
for a in "${MISSING[@]}"; do log " $a"; done
for a in "${MISSING[@]}"; do echo " $a"; done
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN complete — run without --dry-run to add these artists"
+5 -5
View File
@@ -177,7 +177,7 @@ if ! mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
exit 0
fi
log "Ramdisk is mounted ✅"
echo "Ramdisk is mounted ✅"
# Warn if transcode_manager is running — it may flip symlink back on next cycle
if pgrep -f "transcode_manager.sh" >/dev/null 2>&1; then
@@ -199,10 +199,10 @@ if [[ ! -d "$TRANSCODE_SSD" ]]; then
error "Cannot safely redirect symlink — aborting"
exit 1
}
log "SSD fallback created ✅"
echo "SSD fallback created ✅"
fi
else
log "SSD fallback exists: $TRANSCODE_SSD"
echo "SSD fallback exists: $TRANSCODE_SSD"
fi
START=$(date +%s)
@@ -253,7 +253,7 @@ if [[ "$FILE_COUNT" -gt 0 ]]; then
done
fi
else
log "No active files on ramdisk ✅"
echo "No active files on ramdisk ✅"
fi
# ==============================================================================================
@@ -303,7 +303,7 @@ last_flip_time=$NOW
flip_count_hour=0
flip_hour_start=$NOW
EOF
log "State file updated: current_target=$TRANSCODE_SSD"
echo "State file updated: current_target=$TRANSCODE_SSD"
else
warn "Skipping state file update — stop had errors"
fi
+2 -2
View File
@@ -203,7 +203,7 @@ for pool in "${POOLS_TO_SCRUB[@]}"; do
warn "DRY RUN — would scrub: $pool"
STARTED+=("$pool")
elif zpool scrub "$pool" 2>/dev/null; then
log "$pool scrub started ✅"
echo "$pool scrub started ✅"
STARTED+=("$pool")
else
error "Failed to start scrub on $pool"
@@ -231,7 +231,7 @@ fi
echo ""
echo "━━━ $ICON_TIME Waiting for Scrubs to Complete ━━━"
log "Polling every 60 seconds — scrubs may take hours on large pools"
log "Safe to interrupt — scrubs continue in background if script is stopped"
echo "Safe to interrupt — scrubs continue in background if script is stopped"
while [[ "$SCRUB_RUNNING" == true ]]; do
sleep 60
+3 -3
View File
@@ -226,7 +226,7 @@ else
else
log "Creating SSD fallback directory: $TRANSCODE_SSD"
if mkdir -p "$TRANSCODE_SSD"; then
log "SSD fallback created: $TRANSCODE_SSD"
echo "SSD fallback created: $TRANSCODE_SSD"
else
error "Failed to create SSD fallback: $TRANSCODE_SSD"
SETUP_SUCCESS=false
@@ -270,7 +270,7 @@ else
}
fi
[[ "$SETUP_SUCCESS" == true ]] && log "Symlink: $TRANSCODE_LINK$RAMDISK_PATH"
[[ "$SETUP_SUCCESS" == true ]] && echo "Symlink: $TRANSCODE_LINK$RAMDISK_PATH"
fi
# ==============================================================================================
@@ -291,7 +291,7 @@ else
log "transcoding-temp already exists on ramdisk"
else
if mkdir -p "$TRANSCODE_TEMP_DIR"; then
log "Created transcoding-temp on ramdisk ✅"
echo "Created transcoding-temp on ramdisk ✅"
else
error "Failed to create transcoding-temp on ramdisk"
SETUP_SUCCESS=false
+1 -1
View File
@@ -272,7 +272,7 @@ if [[ -n "$NPM_URL" ]]; then
if curl -sf --max-time "${NETWORK_WATCHDOG_NPM_TIMEOUT:-10}" "$NPM_URL" >/dev/null 2>&1; then
log "$ICON_SUCCESS NPM proxy reachable — $NPM_URL"
if [[ "$NPM_STRIKES" -gt 0 ]]; then
log "NPM strikes cleared (was $NPM_STRIKES)"
echo "NPM strikes cleared (was $NPM_STRIKES)"
set_strikes "npm" 0 "${NETWORK_WATCHDOG_NPM_STATE_FILE}"
fi
else
+1 -1
View File
@@ -561,7 +561,7 @@ elif [[ "$TARGET_LEVEL" -lt "$CURRENT_LEVEL" ]]; then
if [[ "$NEW_LEVEL" -gt 0 ]]; then
warn "De-escalated to level $NEW_LEVEL (${LEVEL_NAMES[$NEW_LEVEL]}) — ${RW_RECOVER_CYCLES:-3} more cycles to fully clear"
else
log "All pressure cleared — system at normal operation ✅"
echo "All pressure cleared — system at normal operation ✅"
notify "Resource Manager: pressure resolved on $(hostname) ($MY_ID) — system back to normal" \
"Resource Manager" "normal"
fi