FailedProxy/Varaverk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: FailedProxy/Varaverk:260f0a61ce315b8170f9d7fba20a581e338dc5ce
Branches Tags
FailedProxy/Varaverk:main
..
compare: FailedProxy/Varaverk:6623d1e776c7abe4730c32e6174095b6907e9709
Branches Tags
FailedProxy/Varaverk:main
2 Commits
260f0a61ce .. 6623d1e776
Author SHA1 Message Date
Gmer4Lfe 6623d1e776 Fix dead/incorrect vars and consolidate duplicated logic into common.sh 
Codebase-wide audit pass: fixed real bugs (SSH hangs missing BatchMode,
local-outside-function no-ops, variable name collisions, a truncated
ratio calc, wrong state-dir path, DARK vs NO_INTERNET drift, and more),
then pulled logic that was duplicated across multiple scripts — arr
cleanup safety gates, docker restart ordering, container maintenance
stop/restart, watchdog state-file helpers, partnership role resolution,
cert expiry checks, remote node discovery, and TMDB discovery scoring —
into common.sh so each now has a single implementation.
 2026-07-03 23:52:33 -04:00
Gmer4Lfe ef3980cf07 Stop tracking Notes_To-Do.md — personal scratch notes, dev-only 
Keeps the file locally for day-to-day use but out of the shared repo.
 2026-07-03 23:52:12 -04:00
48 changed files with 924 additions and 1472 deletions
Expand all files Collapse all files
.gitignore
+3
View File
@@ -6,6 +6,9 @@ Configurations/master.conf
Configurations/*.bak
.vscode
# ── Personal scratch notes — dev-only, never pushed ───────────────────────────
Notes_To-Do.md
# ── Runtime state, data, logs ─────────────────────────────────────────────────
data/
State_Files/
Arrs_Stack/arr_sync.sh
+4 -10
View File
@@ -206,13 +206,7 @@ declare -A _ID_TYPE=([lidarr]="string" [sonarr]="int" [radarr]="int")
ARR_TYPES=(lidarr sonarr radarr)
# ── Remote node discovery ──────────────────────────────────────────────────────────────────────
REMOTE_NODES=()
for _hv in HOST1 HOST2 HOST3 HOST4 HOST5 HOST6 HOST7 HOST8; do
[[ "$_hv" == "$MY_ID" ]] && continue
[[ -z "${!_hv:-}" ]] && continue
REMOTE_NODES+=("$_hv")
done
unset _hv
discover_remote_nodes
if [[ "${#REMOTE_NODES[@]}" -eq 0 ]]; then
warn "No remote nodes defined in master.conf — nothing to sync"
@@ -478,7 +472,7 @@ if [[ "$SHOW_STATUS" == true ]]; then
echo ""
echo " Arr Port URL"
for _arr in "${ARR_TYPES[@]}"; do
local _url="" _key=""
_url=""
case "$_arr" in
lidarr) _url="${LIDARR_URL:-not configured}" ;;
sonarr) _url="${SONARR_URL:-not configured}" ;;
@@ -992,8 +986,8 @@ _sync_arr() {
fi
fi
local _n_local=${#to_add_local[@]:-}; _n_local=${_n_local:-0}
local _n_remote=${#to_add_remote[@]:-}; _n_remote=${_n_remote:-0}
local _n_local=${#to_add_local[@]}
local _n_remote=${#to_add_remote[@]}
log " $node_name: +${_n_local} local | +${_n_remote} remote | $total_skipped blocklisted"
unset remote_ids to_add_local to_add_remote _n_local _n_remote
Arrs_Stack/lidarr_cleanup.sh
+17 -187
View File
@@ -60,7 +60,7 @@
#
# acquire_lock "wait" — large scans take time, wait for previous run to finish
# jq + curl validation — exits if either tool missing
# DOCKER_TIMEOUT — container checks protected against daemon hangs
# ARR_DOCKER_TIMEOUT — container checks protected against daemon hangs (script-local, not common.sh's DOCKER_TIMEOUT)
# Duplicate detection — temp file of tracked paths, grep before delete
# platform_require_cmd — notify script validated before use
# Silent by default — orphans/junk warn(), clean library logs silently
@@ -120,37 +120,12 @@ source "$SCRIPT_DIR/../load_config.sh"
# ── Special flag pre-processing ───────────────────────────────────────────────────────────────
# Filter --i-know-what-im-doing and --skip-age-check before parse_args
# to avoid unknown flag errors — both are handled separately below.
I_KNOW=false
SKIP_AGE_CHECK=false
FILTERED_ARGS=()
for arg in "$@"; do
case "$arg" in
--i-know-what-im-doing) I_KNOW=true ;;
--skip-age-check) SKIP_AGE_CHECK=true ;;
*) FILTERED_ARGS+=("$arg") ;;
esac
done
parse_destructive_flags "$@"
parse_args "${FILTERED_ARGS[@]}"
# ── Nuclear mode warning ──────────────────────────────────────────────────────────────────────
if [[ "$I_KNOW" == true ]] && [[ "$SKIP_AGE_CHECK" == true ]] && [[ "$DRY_RUN" != true ]]; then
echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "⚠️ WARNING — NUCLEAR MODE ACTIVE"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo " Flags: --i-know-what-im-doing --skip-age-check"
echo " Age check: BYPASSED — deletes on first pass"
echo " Size threshold: BYPASSED — no GB limit"
echo " Data recovery: NOT POSSIBLE after deletion"
echo ""
echo " Review --dry-run output before proceeding."
echo " You have 10 seconds to cancel (Ctrl+C)..."
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
sleep 10
echo " Proceeding..."
echo ""
fi
nuclear_mode_warning
# ==============================================================================================
# ━━━ Setup ━━━
@@ -194,15 +169,11 @@ if [[ -z "${LIDARR_URL:-}" ]] || [[ -z "${LIDARR_API_KEY:-}" ]]; then
exit 0
fi
DOCKER_TIMEOUT=15
ARR_DOCKER_TIMEOUT=15
LIDARR_CONTAINER="Lidarr" # container name on HOST1
# Build path map from MY_ID's Lidarr path map
declare -A ARR_PATH_MAP
local_path_map_var="${MY_ID}_LIDARR_PATH_MAP"
eval "for key in \"\${!${local_path_map_var}[@]}\"; do
ARR_PATH_MAP[\"\$key\"]=\"\${${local_path_map_var}[\$key]}\"
done"
build_arr_path_map "LIDARR"
# Validate required vars — detect_hosts() should have set these
require_var LIDARR_URL
@@ -251,83 +222,13 @@ fi
echo ""
echo "━━━ $ICON_SHIELD Safety Checks ━━━"
CONTAINER_RUNNING=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
'{{.State.Running}}' "$LIDARR_CONTAINER" 2>/dev/null)
if [[ "$CONTAINER_RUNNING" != "true" ]]; then
error "$LIDARR_CONTAINER is not running — aborting"
notify "Lidarr cleanup aborted on $(hostname) — container not running" \
"Lidarr Cleanup" "warning"
exit 1
fi
CONTAINER_HEALTH=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
'{{.State.Health.Status}}' "$LIDARR_CONTAINER" 2>/dev/null)
case "$CONTAINER_HEALTH" in
healthy) info "$LIDARR_CONTAINER is healthy" ;;
"") info "$LIDARR_CONTAINER has no health check — proceeding" ;;
starting)
error "$LIDARR_CONTAINER is still starting — aborting"
notify "Lidarr cleanup aborted on $(hostname) — container still starting" \
"Lidarr Cleanup" "warning"
exit 1 ;;
unhealthy)
error "$LIDARR_CONTAINER is unhealthy — aborting"
notify "Lidarr cleanup aborted on $(hostname) — container unhealthy" \
"Lidarr Cleanup" "warning"
exit 1 ;;
*) warn "$LIDARR_CONTAINER health: $CONTAINER_HEALTH — proceeding with caution" ;;
esac
info "Safety layer 1 passed — container healthy"
check_container_health "$LIDARR_CONTAINER" "$ARR_DOCKER_TIMEOUT" "Lidarr Cleanup"
# ==============================================================================================
# ── HELPER FUNCTIONS ──────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Lidarr API call with HTTP status check
# Usage: lidarr_api "artist" | lidarr_api "trackFile?artistId=123"
lidarr_api() {
local endpoint="$1"
local response http_code body
response=$(curl -sf \
--max-time 30 \
-H "X-Api-Key: $LIDARR_API_KEY" \
-w "\n%{http_code}" \
"${LIDARR_URL}/api/v1/${endpoint}" 2>/dev/null)
http_code=$(echo "$response" | tail -1)
body=$(echo "$response" | head -n -1)
if [[ "$http_code" != "200" ]]; then
error "Lidarr API HTTP $http_code for: $endpoint"
return 1
fi
echo "$body"
}
# Check if a file extension is a tracked music format
is_music_file() {
local ext="${1##*.}"
ext="${ext,,}"
for valid_ext in "${LIDARR_EXTENSIONS[@]}"; do
[[ "$ext" == "$valid_ext" ]] && return 0
done
return 1
}
# Check if a file matches any protected pattern
is_protected_file() {
local filename
filename=$(basename "$1")
for pattern in "${LIDARR_PROTECTED_PATTERNS[@]}"; do
# shellcheck disable=SC2254
case "$filename" in
$pattern) return 0 ;;
esac
done
return 1
}
# check_container_health(), arr_api(), has_extension(), matches_pattern_list() — common.sh
# ==============================================================================================
# ━━━ Pre-flight: Lidarr Import Scan ━━━
@@ -353,36 +254,7 @@ else
SCAN_PAYLOAD='{"name": "DownloadedAlbumsScan"}'
fi
SCAN_RESPONSE=$(curl -sf --max-time 30 -X POST \
-H "X-Api-Key: $LIDARR_API_KEY" \
-H "Content-Type: application/json" \
-d "$SCAN_PAYLOAD" \
"${LIDARR_URL}/api/v1/command" 2>/dev/null)
SCAN_CMD_ID=$(echo "$SCAN_RESPONSE" | jq -r '.id // empty' 2>/dev/null)
if [[ -z "$SCAN_CMD_ID" ]]; then
warn "Could not trigger import scan — proceeding without pre-flight"
else
info "Import scan queued (command ID: $SCAN_CMD_ID) — waiting for completion..."
POLL_TIMEOUT=${LIDARR_IMPORT_SCAN_TIMEOUT:-600}
POLLED=0
while [[ "$POLLED" -lt "$POLL_TIMEOUT" ]]; do
SCAN_STATUS=$(curl -sf --max-time 10 \
-H "X-Api-Key: $LIDARR_API_KEY" \
"${LIDARR_URL}/api/v1/command/${SCAN_CMD_ID}" 2>/dev/null | \
jq -r '.status // empty' 2>/dev/null)
case "$SCAN_STATUS" in
completed) info "Import scan complete ✅"; break ;;
failed) warn "Import scan reported failed — proceeding anyway"; break ;;
esac
sleep 10
(( POLLED += 10 ))
[[ $(( POLLED % 60 )) -eq 0 ]] && log " Still scanning... (${POLLED}s elapsed)"
done
[[ "$POLLED" -ge "$POLL_TIMEOUT" ]] && \
warn "Import scan timed out after ${POLL_TIMEOUT}s — proceeding anyway"
fi
trigger_and_await_command "$LIDARR_URL" "$LIDARR_API_KEY" "v1" "$SCAN_PAYLOAD" "${LIDARR_IMPORT_SCAN_TIMEOUT:-600}"
# ==============================================================================================
# ━━━ Fetch Lidarr Tracked Files ━━━
@@ -402,7 +274,7 @@ check_arr_version "$LIDARR_URL" "$LIDARR_API_KEY" "v1" "$LIDARR_VERSION_MAJOR" "
info "Querying Lidarr API..."
# Fetch all artists
ARTIST_RESPONSE=$(lidarr_api "artist") || {
ARTIST_RESPONSE=$(arr_api "$LIDARR_URL" "$LIDARR_API_KEY" "v1" "artist" "Lidarr") || {
error "Failed to fetch artists from Lidarr"
notify "Lidarr cleanup failed on $(hostname) — could not fetch artists" \
"Lidarr Cleanup" "warning"
@@ -427,7 +299,7 @@ TRACKED_FILE="$TMP_DIR/tracked_paths.txt"
while IFS= read -r artist_id; do
[[ -z "$artist_id" ]] && continue
ARTIST_TRACKS=$(lidarr_api "trackFile?artistId=${artist_id}" 2>/dev/null)
ARTIST_TRACKS=$(arr_api "$LIDARR_URL" "$LIDARR_API_KEY" "v1" "trackFile?artistId=${artist_id}" "Lidarr" 2>/dev/null)
if [[ -n "$ARTIST_TRACKS" ]]; then
while IFS= read -r api_path; do
[[ -z "$api_path" ]] && continue
@@ -459,25 +331,7 @@ fi
info "$ARTIST_COUNT artists | $TRACKED_COUNT tracked files"
# Safety Layer 6 — percentage drop vs last known count
if [[ -f "$LIDARR_TRACKED_COUNT_FILE" ]]; then
LAST_COUNT=$(cat "$LIDARR_TRACKED_COUNT_FILE" 2>/dev/null || echo 0)
if [[ "$LAST_COUNT" -gt 0 ]]; then
PCT=$(awk "BEGIN {printf \"%d\", ($TRACKED_COUNT / $LAST_COUNT) * 100}")
if [[ "$PCT" -lt "$LIDARR_MIN_TRACKED_PCT" ]]; then
error "Tracked count dropped to ${PCT}% of last run ($TRACKED_COUNT vs $LAST_COUNT)"
error "Suggests API issue — aborting to prevent mass deletion"
error "If expected (large library removal) delete: $LIDARR_TRACKED_COUNT_FILE"
notify "Lidarr cleanup aborted on $(hostname) — tracked count dropped to ${PCT}%" \
"Lidarr Cleanup" "warning"
exit 1
fi
info "Tracked count: ${PCT}% of last run ($TRACKED_COUNT vs $LAST_COUNT) ✅"
fi
else
info "No previous count on record — first run, saving baseline"
fi
echo "$TRACKED_COUNT" > "$LIDARR_TRACKED_COUNT_FILE"
check_tracked_count_floor "$TRACKED_COUNT" "$LIDARR_TRACKED_COUNT_FILE" "$LIDARR_MIN_TRACKED_PCT" "Lidarr Cleanup"
# ==============================================================================================
# ━━━ Scan Music Root ━━━
@@ -496,7 +350,6 @@ JUNK_BYTES=0
AGE_SECONDS=$(( LIDARR_ORPHAN_AGE * 86400 ))
NOW=$(date +%s)
MAX_DELETE_BYTES=$(awk "BEGIN {printf \"%d\", $LIDARR_MAX_DELETE_GB * 1073741824}")
while IFS= read -r filepath; do
[[ -z "$filepath" ]] && continue
@@ -508,7 +361,7 @@ while IFS= read -r filepath; do
fi
# Protected — never delete
if is_protected_file "$filepath"; then
if matches_pattern_list "$filepath" "${LIDARR_PROTECTED_PATTERNS[@]}"; then
log "$ICON_PROTECTED PROTECTED: $filepath"
(( PROTECTED_COUNT++ ))
continue
@@ -516,7 +369,7 @@ while IFS= read -r filepath; do
FILE_SIZE=$(stat -c%s "$filepath" 2>/dev/null || echo 0)
if is_music_file "$filepath"; then
if has_extension "$filepath" "${LIDARR_EXTENSIONS[@]}"; then
FILE_MTIME=$(stat -c %Y "$filepath" 2>/dev/null || echo 0)
FILE_AGE=$(( NOW - FILE_MTIME ))
@@ -543,21 +396,7 @@ TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT ))
# ==============================================================================================
# ━━━ Safety Layer 7 — Deletion Size Threshold ━━━
# ==============================================================================================
if [[ "$TOTAL_DELETE_BYTES" -gt "$MAX_DELETE_BYTES" ]]; then
TOTAL_HUMAN=$(awk "BEGIN {printf \"%.1fGB\", $TOTAL_DELETE_BYTES / 1073741824}")
if [[ "$I_KNOW" != true ]]; then
echo ""
error "Deletion would exceed ${LIDARR_MAX_DELETE_GB}GB — $TOTAL_HUMAN would be deleted"
error "Review ORPHAN lines above carefully before proceeding"
error "Rerun with: --i-know-what-im-doing"
error "To also bypass age check: add --skip-age-check"
notify "Lidarr cleanup halted on $(hostname) — ${TOTAL_HUMAN} requires --i-know-what-im-doing" \
"Lidarr Cleanup" "warning"
exit 1
else
warn "OVERRIDE — deletion is $TOTAL_HUMAN — proceeding with --i-know-what-im-doing"
fi
fi
check_delete_size_threshold "$TOTAL_DELETE_BYTES" "$LIDARR_MAX_DELETE_GB" "Lidarr Cleanup"
# ── Execute Deletions ─────────────────────────────────────────────────────────────────────────
# All safety layers passed — delete orphans and junk
@@ -565,12 +404,12 @@ if [[ "$DRY_RUN" == false ]]; then
while IFS= read -r filepath; do
[[ -z "$filepath" ]] && continue
[[ -n "${TRACKED_MAP[$filepath]:-}" ]] && continue
is_protected_file "$filepath" && continue
matches_pattern_list "$filepath" "${LIDARR_PROTECTED_PATTERNS[@]}" && continue
FILE_MTIME=$(stat -c %Y "$filepath" 2>/dev/null || echo 0)
FILE_AGE=$(( NOW - FILE_MTIME ))
if is_music_file "$filepath"; then
if has_extension "$filepath" "${LIDARR_EXTENSIONS[@]}"; then
[[ "$FILE_AGE" -lt "$AGE_SECONDS" ]] && \
[[ "$SKIP_AGE_CHECK" != true ]] && continue
fi
@@ -586,16 +425,7 @@ fi
END=$(date +%s)
format_bytes() {
local bytes=$1
if (( bytes > 1073741824 )); then
awk "BEGIN {printf \"%.1fGB\", $bytes / 1073741824}"
elif (( bytes > 1048576 )); then
awk "BEGIN {printf \"%.1fMB\", $bytes / 1048576}"
else
echo "${bytes}B"
fi
}
# format_bytes() — provided by common.sh
ORPHAN_HUMAN=$(format_bytes "$ORPHAN_BYTES")
JUNK_HUMAN=$(format_bytes "$JUNK_BYTES")
Arrs_Stack/lidarr_release_fixer.sh
+1 -12
View File
@@ -283,17 +283,6 @@ read_file_mbid() {
esac
}
translate_container_path() {
local cpath="$1"
for cp in "${!ARR_PATH_MAP[@]}"; do
if [[ "$cpath" == "${cp}"* ]]; then
echo "${ARR_PATH_MAP[$cp]}${cpath#$cp}"
return
fi
done
echo "$cpath"
}
# ── Safety checks ─────────────────────────────────────────────────────────────────────────────
echo ""
echo "━━━ $ICON_SHIELD Safety Checks ━━━"
@@ -322,7 +311,7 @@ while IFS= read -r artist; do
aid=$(echo "$artist" | jq -r '.id')
apath=$(echo "$artist" | jq -r '.path // empty')
aname=$(echo "$artist" | jq -r '.artistName // empty')
[[ -n "$apath" ]] && ARTIST_PATH_CACHE[$aid]=$(translate_container_path "$apath")
[[ -n "$apath" ]] && ARTIST_PATH_CACHE[$aid]=$(translate_path "$apath")
[[ -n "$aname" ]] && ARTIST_NAME_CACHE[$aid]="$aname"
done < <(echo "$ALL_ARTISTS" | jq -c '.[]' 2>/dev/null)
unset ALL_ARTISTS
Arrs_Stack/playback_aware_radarr_discovery.sh
+2 -31
View File
@@ -134,7 +134,6 @@
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../load_config.sh"
source "$SCRIPT_DIR/../Kernel/decision_engine.sh"
parse_args "$@"
@@ -185,7 +184,7 @@ log "$ICON_GEAR Config: threshold=${THRESHOLD} lookback=${LOOKBACK_DAYS}d max-se
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no movies will be added to Radarr"
_fmt_rating() { local v="${1:-0}"; echo "${v::-1}.${v: -1}" 2>/dev/null || echo "$v"; }
# _fmt_rating() — provided by common.sh
# ==============================================================================================
# ━━━ Status ━━━
@@ -275,35 +274,7 @@ _freq_score() {
fi
}
# vote_avg_int = vote_average × 10 as integer (e.g. 7.8 → 78)
_rating_score_s2() {
local v="$1"
if (( v >= 80 )); then echo 40
elif (( v >= 75 )); then echo 32
elif (( v >= 70 )); then echo 25
elif (( v >= 65 )); then echo 18
elif (( v >= 60 )); then echo 12
else echo 5
fi
}
_votes_score() {
local c="$1"
if (( c >= 10000 )); then echo 20
elif (( c >= 5000 )); then echo 15
elif (( c >= 1000 )); then echo 10
elif (( c >= 200 )); then echo 5
else echo 2
fi
}
_breadth_score() {
local seeds="$1"
if (( seeds >= 3 )); then echo 40
elif (( seeds == 2 )); then echo 25
else echo 10
fi
}
# _rating_score_s2(), _votes_score(), _breadth_score() — provided by common.sh
# ==============================================================================================
# ━━━ Fetch Emby Libraries + Recently Watched Movies ━━━
Arrs_Stack/playback_aware_sonarr_discovery.sh
+2 -36
View File
@@ -148,7 +148,6 @@
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../load_config.sh"
source "$SCRIPT_DIR/../Kernel/decision_engine.sh"
parse_args "$@"
@@ -205,7 +204,7 @@ log "$ICON_GEAR Config: threshold=${THRESHOLD} lookback=${LOOKBACK_DAYS}d max-se
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no shows will be added to Sonarr"
_fmt_rating() { local v="${1:-0}"; echo "${v::-1}.${v: -1}" 2>/dev/null || echo "$v"; }
# _fmt_rating() — provided by common.sh
# ==============================================================================================
# ━━━ Status ━━━
@@ -315,37 +314,7 @@ _volume_score() {
fi
}
# Stage 2: TMDB vote_average × 10 (0-40)
_rating_score_s2() {
local v="$1"
if (( v >= 80 )); then echo 40
elif (( v >= 75 )); then echo 32
elif (( v >= 70 )); then echo 25
elif (( v >= 65 )); then echo 18
elif (( v >= 60 )); then echo 12
else echo 5
fi
}
# Stage 2: vote count (0-20)
_votes_score() {
local c="$1"
if (( c >= 10000 )); then echo 20
elif (( c >= 5000 )); then echo 15
elif (( c >= 1000 )); then echo 10
elif (( c >= 200 )); then echo 5
else echo 2
fi
}
# Stage 2: seed breadth (0-40)
_breadth_score() {
local seeds="$1"
if (( seeds >= 3 )); then echo 40
elif (( seeds == 2 )); then echo 25
else echo 10
fi
}
# _rating_score_s2(), _votes_score(), _breadth_score() — provided by common.sh
# ==============================================================================================
# ━━━ Fetch Emby Series Library ━━━
@@ -636,9 +605,6 @@ done < <(echo "$SONARR_SERIES_JSON" | jq -r '.[] |
log "${#SONARR_TVDB[@]} series in Sonarr | ${#EMBY_TMDB_IDS[@]} series in Emby"
_in_sonarr() { [[ "${SONARR_TVDB["$1"]+x}" || "${SONARR_TMDB["$2"]+x}" ]]; }
_in_emby() { [[ "${EMBY_TMDB_IDS["$1"]+x}" || "${EMBY_TVDB_IDS["$2"]+x}" ]]; }
# ==============================================================================================
# ━━━ Score Stage 2 Candidates ━━━
# ==============================================================================================
Arrs_Stack/radarr_cleanup.sh
+17 -185
View File
@@ -64,7 +64,7 @@
#
# acquire_lock "wait" — large scans take time, wait for previous run to finish
# jq + curl validation — exits if either tool missing
# DOCKER_TIMEOUT — container checks protected against daemon hangs
# ARR_DOCKER_TIMEOUT — container checks protected against daemon hangs (script-local, not common.sh's DOCKER_TIMEOUT)
# notify_emby_scan() — triggers Emby clean after deletion
# platform_require_cmd — notify script validated before use
# Silent by default — orphans/junk warn(), clean library logs silently
@@ -120,37 +120,12 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../load_config.sh"
# ── Special flag pre-processing ───────────────────────────────────────────────────────────────
I_KNOW=false
SKIP_AGE_CHECK=false
FILTERED_ARGS=()
for arg in "$@"; do
case "$arg" in
--i-know-what-im-doing) I_KNOW=true ;;
--skip-age-check) SKIP_AGE_CHECK=true ;;
*) FILTERED_ARGS+=("$arg") ;;
esac
done
parse_destructive_flags "$@"
parse_args "${FILTERED_ARGS[@]}"
# ── Nuclear mode warning ──────────────────────────────────────────────────────────────────────
if [[ "$I_KNOW" == true ]] && [[ "$SKIP_AGE_CHECK" == true ]] && [[ "$DRY_RUN" != true ]]; then
echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "⚠️ WARNING — NUCLEAR MODE ACTIVE"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo " Flags: --i-know-what-im-doing --skip-age-check"
echo " Age check: BYPASSED — deletes on first pass"
echo " Size threshold: BYPASSED — no GB limit"
echo " Data recovery: NOT POSSIBLE after deletion"
echo ""
echo " Review --dry-run output before proceeding."
echo " You have 10 seconds to cancel (Ctrl+C)..."
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
sleep 10
echo " Proceeding..."
echo ""
fi
nuclear_mode_warning
# ==============================================================================================
# ━━━ Setup ━━━
@@ -191,15 +166,11 @@ if [[ -z "${RADARR_URL:-}" ]] || [[ -z "${RADARR_API_KEY:-}" ]]; then
exit 0
fi
DOCKER_TIMEOUT=15
ARR_DOCKER_TIMEOUT=15
RADARR_CONTAINER="Radarr"
# Build path map from MY_ID's Radarr path map
declare -A ARR_PATH_MAP
local_path_map_var="${MY_ID}_RADARR_PATH_MAP"
eval "for key in \"\${!${local_path_map_var}[@]}\"; do
ARR_PATH_MAP[\"\$key\"]=\"\${${local_path_map_var}[\$key]}\"
done"
build_arr_path_map "RADARR"
require_var RADARR_URL
require_var RADARR_API_KEY
@@ -247,90 +218,13 @@ fi
echo ""
echo "━━━ $ICON_SHIELD Safety Checks ━━━"
CONTAINER_RUNNING=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
'{{.State.Running}}' "$RADARR_CONTAINER" 2>/dev/null)
if [[ "$CONTAINER_RUNNING" != "true" ]]; then
error "$RADARR_CONTAINER is not running — aborting"
notify "Radarr cleanup aborted on $(hostname) — container not running" \
"Radarr Cleanup" "warning"
exit 1
fi
CONTAINER_HEALTH=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
'{{.State.Health.Status}}' "$RADARR_CONTAINER" 2>/dev/null)
case "$CONTAINER_HEALTH" in
healthy) info "$RADARR_CONTAINER is healthy" ;;
"") info "$RADARR_CONTAINER has no health check — proceeding" ;;
starting)
error "$RADARR_CONTAINER is still starting — aborting"
notify "Radarr cleanup aborted on $(hostname) — container still starting" \
"Radarr Cleanup" "warning"
exit 1 ;;
unhealthy)
error "$RADARR_CONTAINER is unhealthy — aborting"
notify "Radarr cleanup aborted on $(hostname) — container unhealthy" \
"Radarr Cleanup" "warning"
exit 1 ;;
*) warn "$RADARR_CONTAINER health: $CONTAINER_HEALTH — proceeding with caution" ;;
esac
info "Safety layer 1 passed — container healthy"
check_container_health "$RADARR_CONTAINER" "$ARR_DOCKER_TIMEOUT" "Radarr Cleanup"
# ==============================================================================================
# ── HELPER FUNCTIONS ──────────────────────────────────────────────────────────────────────────
# ==============================================================================================
radarr_api() {
local endpoint="$1"
local response http_code body
response=$(curl -sf \
--max-time 30 \
-H "X-Api-Key: $RADARR_API_KEY" \
-w "\n%{http_code}" \
"${RADARR_URL}/api/v3/${endpoint}" 2>/dev/null)
http_code=$(echo "$response" | tail -1)
body=$(echo "$response" | head -n -1)
if [[ "$http_code" != "200" ]]; then
error "Radarr API HTTP $http_code for: $endpoint"
return 1
fi
echo "$body"
}
is_video_file() {
local ext="${1##*.}"
ext="${ext,,}"
for valid_ext in "${RADARR_EXTENSIONS[@]}"; do
[[ "$ext" == "$valid_ext" ]] && return 0
done
return 1
}
is_protected_file() {
local filename
filename=$(basename "$1")
for pattern in "${RADARR_PROTECTED_PATTERNS[@]}"; do
# shellcheck disable=SC2254
case "$filename" in
$pattern) return 0 ;;
esac
done
return 1
}
format_bytes() {
local bytes=$1
if (( bytes > 1073741824 )); then
awk "BEGIN {printf \"%.1fGB\", $bytes / 1073741824}"
elif (( bytes > 1048576 )); then
awk "BEGIN {printf \"%.1fMB\", $bytes / 1048576}"
else
echo "${bytes}B"
fi
}
# check_container_health(), arr_api(), has_extension(), matches_pattern_list(), format_bytes() — common.sh
# ==============================================================================================
# ━━━ Pre-flight: Radarr Import Scan ━━━
@@ -340,7 +234,7 @@ echo "━━━ $ICON_SYNC Pre-flight: Radarr Import Scan ━━━"
# Fetch root folders from Radarr API and translate container paths to host paths
mapfile -t SCAN_ROOTS < <(
radarr_api "rootfolder" | \
arr_api "$RADARR_URL" "$RADARR_API_KEY" "v3" "rootfolder" "Radarr" | \
jq -r '.[].path' 2>/dev/null | \
while IFS= read -r cp; do translate_path "$cp"; done
)
@@ -357,36 +251,7 @@ info "Scan targets (${#SCAN_ROOTS[@]}): ${SCAN_ROOTS[*]}"
info "Triggering ProcessMonitoredDownloads pre-flight"
SCAN_PAYLOAD='{"name": "ProcessMonitoredDownloads"}'
SCAN_RESPONSE=$(curl -sf --max-time 30 -X POST \
-H "X-Api-Key: $RADARR_API_KEY" \
-H "Content-Type: application/json" \
-d "$SCAN_PAYLOAD" \
"${RADARR_URL}/api/v3/command" 2>/dev/null)
SCAN_CMD_ID=$(echo "$SCAN_RESPONSE" | jq -r '.id // empty' 2>/dev/null)
if [[ -z "$SCAN_CMD_ID" ]]; then
warn "Could not trigger import scan — proceeding without pre-flight"
else
info "Import scan queued (command ID: $SCAN_CMD_ID) — waiting for completion..."
POLL_TIMEOUT=${RADARR_IMPORT_SCAN_TIMEOUT:-600}
POLLED=0
while [[ "$POLLED" -lt "$POLL_TIMEOUT" ]]; do
SCAN_STATUS=$(curl -sf --max-time 10 \
-H "X-Api-Key: $RADARR_API_KEY" \
"${RADARR_URL}/api/v3/command/${SCAN_CMD_ID}" 2>/dev/null | \
jq -r '.status // empty' 2>/dev/null)
case "$SCAN_STATUS" in
completed) info "Import scan complete ✅"; break ;;
failed) warn "Import scan reported failed — proceeding anyway"; break ;;
esac
sleep 10
(( POLLED += 10 ))
[[ $(( POLLED % 60 )) -eq 0 ]] && log " Still scanning... (${POLLED}s elapsed)"
done
[[ "$POLLED" -ge "$POLL_TIMEOUT" ]] && \
warn "Import scan timed out after ${POLL_TIMEOUT}s — proceeding anyway"
fi
trigger_and_await_command "$RADARR_URL" "$RADARR_API_KEY" "v3" "$SCAN_PAYLOAD" "${RADARR_IMPORT_SCAN_TIMEOUT:-600}"
# ==============================================================================================
# ━━━ Fetch Radarr Tracked Files ━━━
@@ -406,7 +271,7 @@ check_arr_version "$RADARR_URL" "$RADARR_API_KEY" "v3" "$RADARR_VERSION_MAJOR" "
info "Querying Radarr API..."
# Fetch all movies
MOVIES_RESPONSE=$(radarr_api "movie") || {
MOVIES_RESPONSE=$(arr_api "$RADARR_URL" "$RADARR_API_KEY" "v3" "movie" "Radarr") || {
error "Failed to fetch movies from Radarr"
notify "Radarr cleanup failed on $(hostname) — could not fetch movies" \
"Radarr Cleanup" "warning"
@@ -435,7 +300,7 @@ while IFS= read -r movie_id; do
(( MOVIE_INDEX++ ))
[[ $(( MOVIE_INDEX % 100 )) -eq 0 ]] && \
log "Fetching files: $MOVIE_INDEX/$MOVIE_COUNT movies..."
MOVIE_FILES=$(radarr_api "moviefile?movieId=${movie_id}" 2>/dev/null)
MOVIE_FILES=$(arr_api "$RADARR_URL" "$RADARR_API_KEY" "v3" "moviefile?movieId=${movie_id}" "Radarr" 2>/dev/null)
if [[ -n "$MOVIE_FILES" ]]; then
while IFS= read -r api_path; do
[[ -z "$api_path" ]] && continue
@@ -467,25 +332,7 @@ fi
info "$MOVIE_COUNT movies | $TRACKED_COUNT tracked movie files"
# Safety Layer 6 — percentage drop vs last known count
if [[ -f "$RADARR_TRACKED_COUNT_FILE" ]]; then
LAST_COUNT=$(cat "$RADARR_TRACKED_COUNT_FILE" 2>/dev/null || echo 0)
if [[ "$LAST_COUNT" -gt 0 ]]; then
PCT=$(awk "BEGIN {printf \"%d\", ($TRACKED_COUNT / $LAST_COUNT) * 100}")
if [[ "$PCT" -lt "$RADARR_MIN_TRACKED_PCT" ]]; then
error "Tracked count dropped to ${PCT}% of last run ($TRACKED_COUNT vs $LAST_COUNT)"
error "Suggests API issue — aborting to prevent mass deletion"
error "If expected (large library removal) delete: $RADARR_TRACKED_COUNT_FILE"
notify "Radarr cleanup aborted on $(hostname) — tracked count dropped to ${PCT}%" \
"Radarr Cleanup" "warning"
exit 1
fi
info "Tracked count: ${PCT}% of last run ($TRACKED_COUNT vs $LAST_COUNT) ✅"
fi
else
info "No previous count on record — first run, saving baseline"
fi
echo "$TRACKED_COUNT" > "$RADARR_TRACKED_COUNT_FILE"
check_tracked_count_floor "$TRACKED_COUNT" "$RADARR_TRACKED_COUNT_FILE" "$RADARR_MIN_TRACKED_PCT" "Radarr Cleanup"
# ==============================================================================================
# ━━━ Scan Movies Root ━━━
@@ -504,7 +351,6 @@ JUNK_BYTES=0
AGE_SECONDS=$(( RADARR_ORPHAN_AGE * 86400 ))
NOW=$(date +%s)
MAX_DELETE_BYTES=$(awk "BEGIN {printf \"%d\", $RADARR_MAX_DELETE_GB * 1073741824}")
while IFS= read -r filepath; do
[[ -z "$filepath" ]] && continue
@@ -514,7 +360,7 @@ while IFS= read -r filepath; do
continue
fi
if is_protected_file "$filepath"; then
if matches_pattern_list "$filepath" "${RADARR_PROTECTED_PATTERNS[@]}"; then
log "$ICON_PROTECTED PROTECTED: $filepath"
(( PROTECTED_COUNT++ ))
continue
@@ -522,7 +368,7 @@ while IFS= read -r filepath; do
FILE_SIZE=$(stat -c%s "$filepath" 2>/dev/null || echo 0)
if is_video_file "$filepath"; then
if has_extension "$filepath" "${RADARR_EXTENSIONS[@]}"; then
FILE_MTIME=$(stat -c %Y "$filepath" 2>/dev/null || echo 0)
FILE_AGE=$(( NOW - FILE_MTIME ))
@@ -553,33 +399,19 @@ TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT ))
# ==============================================================================================
# ━━━ Safety Layer 7 — Deletion Size Threshold ━━━
# ==============================================================================================
if [[ "$TOTAL_DELETE_BYTES" -gt "$MAX_DELETE_BYTES" ]]; then
TOTAL_HUMAN=$(awk "BEGIN {printf \"%.1fGB\", $TOTAL_DELETE_BYTES / 1073741824}")
if [[ "$I_KNOW" != true ]]; then
echo ""
error "Deletion would exceed ${RADARR_MAX_DELETE_GB}GB — $TOTAL_HUMAN would be deleted"
error "Review ORPHAN lines above carefully before proceeding"
error "Rerun with: --i-know-what-im-doing"
error "To also bypass age check: add --skip-age-check"
notify "Radarr cleanup halted on $(hostname) — ${TOTAL_HUMAN} requires --i-know-what-im-doing" \
"Radarr Cleanup" "warning"
exit 1
else
warn "OVERRIDE — deletion is $TOTAL_HUMAN — proceeding with --i-know-what-im-doing"
fi
fi
check_delete_size_threshold "$TOTAL_DELETE_BYTES" "$RADARR_MAX_DELETE_GB" "Radarr Cleanup"
# ── Execute Deletions ─────────────────────────────────────────────────────────────────────────
if [[ "$DRY_RUN" == false ]]; then
while IFS= read -r filepath; do
[[ -z "$filepath" ]] && continue
[[ -n "${TRACKED_MAP[$filepath]:-}" ]] && continue
is_protected_file "$filepath" && continue
matches_pattern_list "$filepath" "${RADARR_PROTECTED_PATTERNS[@]}" && continue
FILE_MTIME=$(stat -c %Y "$filepath" 2>/dev/null || echo 0)
FILE_AGE=$(( NOW - FILE_MTIME ))
if is_video_file "$filepath"; then
if has_extension "$filepath" "${RADARR_EXTENSIONS[@]}"; then
[[ "$FILE_AGE" -lt "$AGE_SECONDS" ]] && \
[[ "$SKIP_AGE_CHECK" != true ]] && continue
fi
Arrs_Stack/sonarr_cleanup.sh
+17 -185
View File
@@ -64,7 +64,7 @@
#
# acquire_lock "wait" — large scans take time, wait for previous run to finish
# jq + curl validation — exits if either tool missing
# DOCKER_TIMEOUT — container checks protected against daemon hangs
# ARR_DOCKER_TIMEOUT — container checks protected against daemon hangs (script-local, not common.sh's DOCKER_TIMEOUT)
# notify_emby_scan() — triggers Emby clean after deletion
# platform_require_cmd — notify script validated before use
# Silent by default — orphans/junk warn(), clean library logs silently
@@ -120,37 +120,12 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../load_config.sh"
# ── Special flag pre-processing ───────────────────────────────────────────────────────────────
I_KNOW=false
SKIP_AGE_CHECK=false
FILTERED_ARGS=()
for arg in "$@"; do
case "$arg" in
--i-know-what-im-doing) I_KNOW=true ;;
--skip-age-check) SKIP_AGE_CHECK=true ;;
*) FILTERED_ARGS+=("$arg") ;;
esac
done
parse_destructive_flags "$@"
parse_args "${FILTERED_ARGS[@]}"
# ── Nuclear mode warning ──────────────────────────────────────────────────────────────────────
if [[ "$I_KNOW" == true ]] && [[ "$SKIP_AGE_CHECK" == true ]] && [[ "$DRY_RUN" != true ]]; then
echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "⚠️ WARNING — NUCLEAR MODE ACTIVE"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo " Flags: --i-know-what-im-doing --skip-age-check"
echo " Age check: BYPASSED — deletes on first pass"
echo " Size threshold: BYPASSED — no GB limit"
echo " Data recovery: NOT POSSIBLE after deletion"
echo ""
echo " Review --dry-run output before proceeding."
echo " You have 10 seconds to cancel (Ctrl+C)..."
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
sleep 10
echo " Proceeding..."
echo ""
fi
nuclear_mode_warning
# ==============================================================================================
# ━━━ Setup ━━━
@@ -191,15 +166,11 @@ if [[ -z "${SONARR_URL:-}" ]] || [[ -z "${SONARR_API_KEY:-}" ]]; then
exit 0
fi
DOCKER_TIMEOUT=15
ARR_DOCKER_TIMEOUT=15
SONARR_CONTAINER="Sonarr"
# Build path map from MY_ID's Sonarr path map
declare -A ARR_PATH_MAP
local_path_map_var="${MY_ID}_SONARR_PATH_MAP"
eval "for key in \"\${!${local_path_map_var}[@]}\"; do
ARR_PATH_MAP[\"\$key\"]=\"\${${local_path_map_var}[\$key]}\"
done"
build_arr_path_map "SONARR"
require_var SONARR_URL
require_var SONARR_API_KEY
@@ -247,90 +218,13 @@ fi
echo ""
echo "━━━ $ICON_SHIELD Safety Checks ━━━"
CONTAINER_RUNNING=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
'{{.State.Running}}' "$SONARR_CONTAINER" 2>/dev/null)
if [[ "$CONTAINER_RUNNING" != "true" ]]; then
error "$SONARR_CONTAINER is not running — aborting"
notify "Sonarr cleanup aborted on $(hostname) — container not running" \
"Sonarr Cleanup" "warning"
exit 1
fi
CONTAINER_HEALTH=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
'{{.State.Health.Status}}' "$SONARR_CONTAINER" 2>/dev/null)
case "$CONTAINER_HEALTH" in
healthy) info "$SONARR_CONTAINER is healthy" ;;
"") info "$SONARR_CONTAINER has no health check — proceeding" ;;
starting)
error "$SONARR_CONTAINER is still starting — aborting"
notify "Sonarr cleanup aborted on $(hostname) — container still starting" \
"Sonarr Cleanup" "warning"
exit 1 ;;
unhealthy)
error "$SONARR_CONTAINER is unhealthy — aborting"
notify "Sonarr cleanup aborted on $(hostname) — container unhealthy" \
"Sonarr Cleanup" "warning"
exit 1 ;;
*) warn "$SONARR_CONTAINER health: $CONTAINER_HEALTH — proceeding with caution" ;;
esac
info "Safety layer 1 passed — container healthy"
check_container_health "$SONARR_CONTAINER" "$ARR_DOCKER_TIMEOUT" "Sonarr Cleanup"
# ==============================================================================================
# ── HELPER FUNCTIONS ──────────────────────────────────────────────────────────────────────────
# ==============================================================================================
sonarr_api() {
local endpoint="$1"
local response http_code body
response=$(curl -sf \
--max-time 30 \
-H "X-Api-Key: $SONARR_API_KEY" \
-w "\n%{http_code}" \
"${SONARR_URL}/api/v3/${endpoint}" 2>/dev/null)
http_code=$(echo "$response" | tail -1)
body=$(echo "$response" | head -n -1)
if [[ "$http_code" != "200" ]]; then
error "Sonarr API HTTP $http_code for: $endpoint"
return 1
fi
echo "$body"
}
is_video_file() {
local ext="${1##*.}"
ext="${ext,,}"
for valid_ext in "${SONARR_EXTENSIONS[@]}"; do
[[ "$ext" == "$valid_ext" ]] && return 0
done
return 1
}
is_protected_file() {
local filename
filename=$(basename "$1")
for pattern in "${SONARR_PROTECTED_PATTERNS[@]}"; do
# shellcheck disable=SC2254
case "$filename" in
$pattern) return 0 ;;
esac
done
return 1
}
format_bytes() {
local bytes=$1
if (( bytes > 1073741824 )); then
awk "BEGIN {printf \"%.1fGB\", $bytes / 1073741824}"
elif (( bytes > 1048576 )); then
awk "BEGIN {printf \"%.1fMB\", $bytes / 1048576}"
else
echo "${bytes}B"
fi
}
# check_container_health(), arr_api(), has_extension(), matches_pattern_list(), format_bytes() — common.sh
# ==============================================================================================
# ━━━ Pre-flight: Sonarr Import Scan ━━━
@@ -340,7 +234,7 @@ echo "━━━ $ICON_SYNC Pre-flight: Sonarr Import Scan ━━━"
# Fetch root folders from Sonarr API and translate container paths to host paths
mapfile -t SCAN_ROOTS < <(
sonarr_api "rootfolder" | \
arr_api "$SONARR_URL" "$SONARR_API_KEY" "v3" "rootfolder" "Sonarr" | \
jq -r '.[].path' 2>/dev/null | \
while IFS= read -r cp; do translate_path "$cp"; done
)
@@ -357,36 +251,7 @@ info "Scan targets (${#SCAN_ROOTS[@]}): ${SCAN_ROOTS[*]}"
info "Triggering ProcessMonitoredDownloads pre-flight"
SCAN_PAYLOAD='{"name": "ProcessMonitoredDownloads"}'
SCAN_RESPONSE=$(curl -sf --max-time 30 -X POST \
-H "X-Api-Key: $SONARR_API_KEY" \
-H "Content-Type: application/json" \
-d "$SCAN_PAYLOAD" \
"${SONARR_URL}/api/v3/command" 2>/dev/null)
SCAN_CMD_ID=$(echo "$SCAN_RESPONSE" | jq -r '.id // empty' 2>/dev/null)
if [[ -z "$SCAN_CMD_ID" ]]; then
warn "Could not trigger import scan — proceeding without pre-flight"
else
info "Import scan queued (command ID: $SCAN_CMD_ID) — waiting for completion..."
POLL_TIMEOUT=${SONARR_IMPORT_SCAN_TIMEOUT:-600}
POLLED=0
while [[ "$POLLED" -lt "$POLL_TIMEOUT" ]]; do
SCAN_STATUS=$(curl -sf --max-time 10 \
-H "X-Api-Key: $SONARR_API_KEY" \
"${SONARR_URL}/api/v3/command/${SCAN_CMD_ID}" 2>/dev/null | \
jq -r '.status // empty' 2>/dev/null)
case "$SCAN_STATUS" in
completed) info "Import scan complete ✅"; break ;;
failed) warn "Import scan reported failed — proceeding anyway"; break ;;
esac
sleep 10
(( POLLED += 10 ))
[[ $(( POLLED % 60 )) -eq 0 ]] && log " Still scanning... (${POLLED}s elapsed)"
done
[[ "$POLLED" -ge "$POLL_TIMEOUT" ]] && \
warn "Import scan timed out after ${POLL_TIMEOUT}s — proceeding anyway"
fi
trigger_and_await_command "$SONARR_URL" "$SONARR_API_KEY" "v3" "$SCAN_PAYLOAD" "${SONARR_IMPORT_SCAN_TIMEOUT:-600}"
# ==============================================================================================
# ━━━ Fetch Sonarr Tracked Files ━━━
@@ -406,7 +271,7 @@ check_arr_version "$SONARR_URL" "$SONARR_API_KEY" "v3" "$SONARR_VERSION_MAJOR" "
info "Querying Sonarr API..."
# Fetch all series
SERIES_RESPONSE=$(sonarr_api "series") || {
SERIES_RESPONSE=$(arr_api "$SONARR_URL" "$SONARR_API_KEY" "v3" "series" "Sonarr") || {
error "Failed to fetch series from Sonarr"
notify "Sonarr cleanup failed on $(hostname) — could not fetch series" \
"Sonarr Cleanup" "warning"
@@ -435,7 +300,7 @@ while IFS= read -r series_id; do
(( SERIES_INDEX++ ))
[[ $(( SERIES_INDEX % 50 )) -eq 0 ]] && \
log "Fetching files: $SERIES_INDEX/$SERIES_COUNT series..."
SERIES_FILES=$(sonarr_api "episodefile?seriesId=${series_id}" 2>/dev/null)
SERIES_FILES=$(arr_api "$SONARR_URL" "$SONARR_API_KEY" "v3" "episodefile?seriesId=${series_id}" "Sonarr" 2>/dev/null)
if [[ -n "$SERIES_FILES" ]]; then
while IFS= read -r api_path; do
[[ -z "$api_path" ]] && continue
@@ -466,25 +331,7 @@ fi
info "$SERIES_COUNT series | $TRACKED_COUNT tracked episode files"
# Safety Layer 6 — percentage drop vs last known count
if [[ -f "$SONARR_TRACKED_COUNT_FILE" ]]; then
LAST_COUNT=$(cat "$SONARR_TRACKED_COUNT_FILE" 2>/dev/null || echo 0)
if [[ "$LAST_COUNT" -gt 0 ]]; then
PCT=$(awk "BEGIN {printf \"%d\", ($TRACKED_COUNT / $LAST_COUNT) * 100}")
if [[ "$PCT" -lt "$SONARR_MIN_TRACKED_PCT" ]]; then
error "Tracked count dropped to ${PCT}% of last run ($TRACKED_COUNT vs $LAST_COUNT)"
error "Suggests API issue — aborting to prevent mass deletion"
error "If expected (large library removal) delete: $SONARR_TRACKED_COUNT_FILE"
notify "Sonarr cleanup aborted on $(hostname) — tracked count dropped to ${PCT}%" \
"Sonarr Cleanup" "warning"
exit 1
fi
info "Tracked count: ${PCT}% of last run ($TRACKED_COUNT vs $LAST_COUNT) ✅"
fi
else
info "No previous count on record — first run, saving baseline"
fi
echo "$TRACKED_COUNT" > "$SONARR_TRACKED_COUNT_FILE"
check_tracked_count_floor "$TRACKED_COUNT" "$SONARR_TRACKED_COUNT_FILE" "$SONARR_MIN_TRACKED_PCT" "Sonarr Cleanup"
# ==============================================================================================
# ━━━ Scan TV Root ━━━
@@ -503,7 +350,6 @@ JUNK_BYTES=0
AGE_SECONDS=$(( SONARR_ORPHAN_AGE * 86400 ))
NOW=$(date +%s)
MAX_DELETE_BYTES=$(awk "BEGIN {printf \"%d\", $SONARR_MAX_DELETE_GB * 1073741824}")
while IFS= read -r filepath; do
[[ -z "$filepath" ]] && continue
@@ -513,7 +359,7 @@ while IFS= read -r filepath; do
continue
fi
if is_protected_file "$filepath"; then
if matches_pattern_list "$filepath" "${SONARR_PROTECTED_PATTERNS[@]}"; then
log "$ICON_PROTECTED PROTECTED: $filepath"
(( PROTECTED_COUNT++ ))
continue
@@ -521,7 +367,7 @@ while IFS= read -r filepath; do
FILE_SIZE=$(stat -c%s "$filepath" 2>/dev/null || echo 0)
if is_video_file "$filepath"; then
if has_extension "$filepath" "${SONARR_EXTENSIONS[@]}"; then
FILE_MTIME=$(stat -c %Y "$filepath" 2>/dev/null || echo 0)
FILE_AGE=$(( NOW - FILE_MTIME ))
@@ -552,33 +398,19 @@ TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT ))
# ==============================================================================================
# ━━━ Safety Layer 7 — Deletion Size Threshold ━━━
# ==============================================================================================
if [[ "$TOTAL_DELETE_BYTES" -gt "$MAX_DELETE_BYTES" ]]; then
TOTAL_HUMAN=$(awk "BEGIN {printf \"%.1fGB\", $TOTAL_DELETE_BYTES / 1073741824}")
if [[ "$I_KNOW" != true ]]; then
echo ""
error "Deletion would exceed ${SONARR_MAX_DELETE_GB}GB — $TOTAL_HUMAN would be deleted"
error "Review ORPHAN lines above carefully before proceeding"
error "Rerun with: --i-know-what-im-doing"
error "To also bypass age check: add --skip-age-check"
notify "Sonarr cleanup halted on $(hostname) — ${TOTAL_HUMAN} requires --i-know-what-im-doing" \
"Sonarr Cleanup" "warning"
exit 1
else
warn "OVERRIDE — deletion is $TOTAL_HUMAN — proceeding with --i-know-what-im-doing"
fi
fi
check_delete_size_threshold "$TOTAL_DELETE_BYTES" "$SONARR_MAX_DELETE_GB" "Sonarr Cleanup"
# ── Execute Deletions ─────────────────────────────────────────────────────────────────────────
if [[ "$DRY_RUN" == false ]]; then
while IFS= read -r filepath; do
[[ -z "$filepath" ]] && continue
[[ -n "${TRACKED_MAP[$filepath]:-}" ]] && continue
is_protected_file "$filepath" && continue
matches_pattern_list "$filepath" "${SONARR_PROTECTED_PATTERNS[@]}" && continue
FILE_MTIME=$(stat -c %Y "$filepath" 2>/dev/null || echo 0)
FILE_AGE=$(( NOW - FILE_MTIME ))
if is_video_file "$filepath"; then
if has_extension "$filepath" "${SONARR_EXTENSIONS[@]}"; then
[[ "$FILE_AGE" -lt "$AGE_SECONDS" ]] && \
[[ "$SKIP_AGE_CHECK" != true ]] && continue
fi
Arrs_Stack/upgrade_webhook_handler.sh
+1 -7
View File
@@ -103,13 +103,7 @@ case "$ARR_TYPE" in
esac
# ── Remote node list ──────────────────────────────────────────────────────────────────────────
declare -a REMOTE_NODES=()
for _hv in HOST1 HOST2 HOST3 HOST4 HOST5 HOST6 HOST7 HOST8; do
[[ "$_hv" == "$MY_ID" ]] && continue
[[ -z "${!_hv:-}" ]] && continue
REMOTE_NODES+=("$_hv")
done
unset _hv
discover_remote_nodes
if [[ "${#REMOTE_NODES[@]}" -eq 0 ]]; then
echo "No remote nodes configured — nothing to push"
Configurations/master.conf
+9
View File
@@ -802,6 +802,9 @@
# Crash loop threshold
WATCHDOG_CRASH_LIMIT=5 # RestartCount above this = critical crash loop
# Required-container strikes — consecutive down-checks before docker_watchdog.sh attempts a restart
WATCHDOG_REQUIRED_STRIKE_LIMIT=2
# Startup grace period — skip restarts while system is still booting
WATCHDOG_STARTUP_GRACE=600 # seconds after boot before watchdog acts on failures
@@ -891,6 +894,9 @@
# Seconds of warning broadcast to logged-in users before server_reboot.sh reboots.
# Gives users time to save work — 300s = 5 minutes.
REBOOT_SLEEP=300
# Seconds to wait for graceful VM shutdown (ACPI signal via virsh) before libvirt
# stops it anyway — reboot takes priority.
REBOOT_VM_WAIT=30
# ━━━ Mover ━━━
# Seconds to wait before mover_stop.sh sends SIGTERM to the mover process.
@@ -1085,6 +1091,7 @@
LIDARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run
# protects against API returning partial data on a bad day
LIDARR_TRACKED_COUNT_FILE="$DATA_DIR/lidarr_tracked.count"
LIDARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan
LIDARR_EXTENSIONS=("flac" "mp3" "m4a" "wav" "aac" "ogg" "opus" "wma")
LIDARR_PROTECTED_PATTERNS=(
# Metadata
@@ -1153,6 +1160,7 @@
SONARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run
# protects against API returning partial data on a bad day
SONARR_TRACKED_COUNT_FILE="$DATA_DIR/sonarr_tracked.count"
SONARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan
SONARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "ts" "wmv" "mov")
SONARR_PROTECTED_PATTERNS=(
# Subtitles
@@ -1179,6 +1187,7 @@
RADARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run
# protects against API returning partial data on a bad day
RADARR_TRACKED_COUNT_FILE="$DATA_DIR/radarr_tracked.count"
RADARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan
RADARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "wmv" "mov")
RADARR_PROTECTED_PATTERNS=(
# Subtitles
Deployment/master.conf.template
+9
View File
@@ -797,6 +797,9 @@
# Crash loop threshold
WATCHDOG_CRASH_LIMIT=5 # RestartCount above this = critical crash loop
# Required-container strikes — consecutive down-checks before docker_watchdog.sh attempts a restart
WATCHDOG_REQUIRED_STRIKE_LIMIT=2
# Startup grace period — skip restarts while system is still booting
WATCHDOG_STARTUP_GRACE=600 # seconds after boot before watchdog acts on failures
@@ -886,6 +889,9 @@
# Seconds of warning broadcast to logged-in users before server_reboot.sh reboots.
# Gives users time to save work — 300s = 5 minutes.
REBOOT_SLEEP=300
# Seconds to wait for graceful VM shutdown (ACPI signal via virsh) before libvirt
# stops it anyway — reboot takes priority.
REBOOT_VM_WAIT=30
# ━━━ Mover ━━━
# Seconds to wait before mover_stop.sh sends SIGTERM to the mover process.
@@ -1080,6 +1086,7 @@
LIDARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run
# protects against API returning partial data on a bad day
LIDARR_TRACKED_COUNT_FILE="$DATA_DIR/lidarr_tracked.count"
LIDARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan
LIDARR_EXTENSIONS=("flac" "mp3" "m4a" "wav" "aac" "ogg" "opus" "wma")
LIDARR_PROTECTED_PATTERNS=(
# Metadata
@@ -1144,6 +1151,7 @@
SONARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run
# protects against API returning partial data on a bad day
SONARR_TRACKED_COUNT_FILE="$DATA_DIR/sonarr_tracked.count"
SONARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan
SONARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "ts" "wmv" "mov")
SONARR_PROTECTED_PATTERNS=(
# Subtitles
@@ -1168,6 +1176,7 @@
RADARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run
# protects against API returning partial data on a bad day
RADARR_TRACKED_COUNT_FILE="$DATA_DIR/radarr_tracked.count"
RADARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan
RADARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "wmv" "mov")
RADARR_PROTECTED_PATTERNS=(
# Subtitles
Docker_Essentials/docker_container_stop.sh
-2
View File
@@ -186,9 +186,7 @@ FAILED=()
for container in "${RUNNING[@]}"; do
[[ -z "$container" ]] && continue
local c_start
c_start=$(date +%s)
local c_image
c_image=$(docker inspect --format '{{.Config.Image}}' "$container" 2>/dev/null || echo "unknown")
log "━━━ $ICON_CONTAINERS $container ($c_image) ━━━"
Docker_Essentials/docker_daily_restart.sh
+2 -67
View File
@@ -159,71 +159,7 @@ fi
# docker_cmd, verify_running, retry_docker — defined in common.sh
# Builds a dependency-safe restart order from DAILY_RESTART_CONTAINERS.
# Containers that are dependencies of others restart first.
# Returns ordered list in ORDERED_RESTART array.
build_restart_order() {
ORDERED_RESTART=()
local remaining=("${DAILY_RESTART_CONTAINERS[@]}")
local placed=()
# First pass — add dependency containers that appear in our list
for container in "${remaining[@]}"; do
[[ -z "$container" ]] && continue
local is_dependency=false
# Check if this container is a dependency of any other in our list
for dep_string in "${WATCHDOG_DEPENDENCIES[@]}"; do
if [[ "$dep_string" == *"$container"* ]]; then
is_dependency=true
break
fi
done
# Also check associative array format
for dependent in "${!WATCHDOG_DEPENDENCIES[@]}"; do
if [[ "${WATCHDOG_DEPENDENCIES[$dependent]}" == *"$container"* ]]; then
is_dependency=true
break
fi
done
if [[ "$is_dependency" == true ]]; then
# Check not already placed
local already=false
for p in "${placed[@]}"; do [[ "$p" == "$container" ]] && already=true && break; done
if [[ "$already" == false ]]; then
ORDERED_RESTART+=("$container")
placed+=("$container")
fi
fi
done
# Second pass — add remaining containers (dependents and independents)
for container in "${remaining[@]}"; do
[[ -z "$container" ]] && continue
local already=false
for p in "${placed[@]}"; do [[ "$p" == "$container" ]] && already=true && break; done
if [[ "$already" == false ]]; then
ORDERED_RESTART+=("$container")
placed+=("$container")
fi
done
}
# Checks if a container is a dependent of the previously restarted container.
# If so, waits CONTAINER_DELAY before restarting to allow dependency to settle.
# Usage: check_dependency_delay "$container" "$last_restarted"
check_dependency_delay() {
local container="$1"
local last="$2"
[[ -z "$last" ]] && return
local deps="${WATCHDOG_DEPENDENCIES[$container]:-}"
if [[ -n "$deps" ]] && [[ "$deps" == *"$last"* ]]; then
echo " Waiting ${CONTAINER_DELAY}s — $container depends on $last..."
sleep "$CONTAINER_DELAY"
fi
}
# build_restart_order() / check_dependency_delay() — provided by common.sh
# ==============================================================================================
# ━━━ Daily Restart ━━━
@@ -241,8 +177,7 @@ RESTARTED=()
SKIPPED=()
# Build dependency-safe restart order
build_restart_order
log "$ICON_GEAR Restart order: ${ORDERED_RESTART[*]}"
build_restart_order DAILY_RESTART_CONTAINERS
LAST_RESTARTED=""
Docker_Essentials/docker_weekly_restart.sh
+2 -55
View File
@@ -158,58 +158,7 @@ fi
# docker_cmd, retry_docker, verify_running — defined in common.sh
# Builds a dependency-safe restart order from WEEKLY_RESTART_CONTAINERS.
# Containers that are dependencies of others restart first.
build_restart_order() {
ORDERED_RESTART=()
local remaining=("${WEEKLY_RESTART_CONTAINERS[@]}")
local placed=()
# First pass — add dependency containers that appear in our list
for container in "${remaining[@]}"; do
[[ -z "$container" ]] && continue
local is_dependency=false
for dependent in "${!WATCHDOG_DEPENDENCIES[@]}"; do
if [[ "${WATCHDOG_DEPENDENCIES[$dependent]}" == *"$container"* ]]; then
is_dependency=true
break
fi
done
if [[ "$is_dependency" == true ]]; then
local already=false
for p in "${placed[@]}"; do [[ "$p" == "$container" ]] && already=true && break; done
if [[ "$already" == false ]]; then
ORDERED_RESTART+=("$container")
placed+=("$container")
fi
fi
done
# Second pass — add remaining containers (dependents and independents)
for container in "${remaining[@]}"; do
[[ -z "$container" ]] && continue
local already=false
for p in "${placed[@]}"; do [[ "$p" == "$container" ]] && already=true && break; done
if [[ "$already" == false ]]; then
ORDERED_RESTART+=("$container")
placed+=("$container")
fi
done
log "Restart order: ${ORDERED_RESTART[*]}"
}
# Waits CONTAINER_DELAY if this container depends on the last restarted one.
check_dependency_delay() {
local container="$1"
local last="$2"
[[ -z "$last" ]] && return
local deps="${WATCHDOG_DEPENDENCIES[$container]:-}"
if [[ -n "$deps" ]] && [[ "$deps" == *"$last"* ]]; then
log "Waiting ${CONTAINER_DELAY}s — $container depends on $last..."
sleep "$CONTAINER_DELAY"
fi
}
# build_restart_order() / check_dependency_delay() — provided by common.sh
# ==============================================================================================
# ━━━ Weekly Restart ━━━
@@ -226,15 +175,13 @@ RESTARTED=()
SKIPPED=()
# Build dependency-safe restart order
build_restart_order
build_restart_order WEEKLY_RESTART_CONTAINERS
LAST_RESTARTED=""
for container in "${ORDERED_RESTART[@]}"; do
[[ -z "$container" ]] && continue
local c_start
c_start=$(date +%s)
local c_image
c_image=$(docker inspect --format '{{.Config.Image}}' "$container" 2>/dev/null || echo "unknown")
log "━━━ $ICON_CONTAINERS $container ($c_image) ━━━"
Docker_Essentials/downloaders_reset.sh
+6 -4
View File
@@ -602,11 +602,13 @@ if [[ -n "$QBIT_URL" ]] && [[ -n "$QBIT_USERNAME" ]]; then
# Age check — must be old enough
[[ "$AGE_DAYS" -lt "$QBIT_FAILSAFE_MIN_DAYS" ]] && ((SKIPPED++)) && continue
# Ratio check — if configured
# Ratio check — if configured. awk handles the fractional comparison;
# bash's [[ -lt ]] only does integers and would treat e.g. 1.4 and 1.5
# as equal once truncated. Missing/empty ratio defaults to 0 (protected).
if [[ "$QBIT_FAILSAFE_MIN_RATIO" != "0" ]]; then
RATIO_INT="${RATIO%.*}"
MIN_RATIO_INT="${QBIT_FAILSAFE_MIN_RATIO%.*}"
[[ "$RATIO_INT" -lt "$MIN_RATIO_INT" ]] && ((SKIPPED++)) && continue
if (( $(awk "BEGIN {print (${RATIO:-0} < $QBIT_FAILSAFE_MIN_RATIO) ? 1 : 0}") )); then
((SKIPPED++)) && continue
fi
fi
if [[ "$DRY_RUN" == true ]]; then
Fallback/fallback.sh
+8 -1
View File
@@ -1000,12 +1000,19 @@ while [[ "$FALLBACK_RUNNING" == true ]]; do
fi
elif [[ "$INTERNET_UP" == false ]]; then
# Lost internet during fallback — enter DARK
# Lost internet during fallback — enter DARK — same actions as NO_INTERNET
echo ""
echo "━━━ $ICON_NET Entering DARK — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
warn "Lost internet during fallback — entering DARK state"
state_set state "DARK"
local_ddns_stop
# Stop containers configured to stop on internet loss
read -r -a stop_on_no_net <<< "$(get_stop_on_no_net)"
for container in "${stop_on_no_net[@]}"; do
[[ -n "$container" ]] && local_stop "$container"
done
notify "DARK state on $(hostname) — lost internet during fallback" \
"Fallback" "warning"
Kernel/decision_engine.sh
+1
View File
@@ -65,6 +65,7 @@ score_candidate() {
local popularity_score="${2:-0}"
local recency_score="${3:-0}"
local quality_score="${4:-0}"
local TOTAL_SCORE
TOTAL_SCORE=$(( \
user_score + \
Monitors/cert_monitor.sh
+12 -31
View File
@@ -159,43 +159,24 @@ fi
check_cert() {
local domain="$1"
local port="${2:-443}"
_CERT_DAYS=""
_CERT_EXPIRY=""
local expiry_str
expiry_str=$(echo | timeout "$CERT_TIMEOUT" openssl s_client \
-connect "${domain}:${port}" \
-servername "$domain" \
2>/dev/null | openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2)
if [[ -z "$expiry_str" ]]; then
error "$ICON_CERT $domain — could not retrieve certificate (unreachable or no TLS)"
if ! check_cert_expiry "$domain" "$port" "$CERT_TIMEOUT"; then
if [[ -n "$_CERT_EXPIRY_RAW" ]]; then
error "$ICON_CERT $domain — could not parse expiry date: $_CERT_EXPIRY_RAW"
else
error "$ICON_CERT $domain — could not retrieve certificate (unreachable or no TLS)"
fi
return 3
fi
local expiry_epoch
expiry_epoch=$(date -d "$expiry_str" +%s 2>/dev/null)
if [[ -z "$expiry_epoch" ]]; then
error "$ICON_CERT $domain — could not parse expiry date: $expiry_str"
return 3
fi
local now days_remaining expiry_display
now=$(date +%s)
days_remaining=$(( (expiry_epoch - now) / 86400 ))
expiry_display=$(date -d "$expiry_str" '+%Y-%m-%d' 2>/dev/null)
_CERT_DAYS=$days_remaining
_CERT_EXPIRY=$expiry_display
if [[ "$days_remaining" -le "$CERT_CRIT_DAYS" ]]; then
error "$ICON_CERT $domain — CRITICAL: ${days_remaining} days remaining (expires $expiry_display)"
if [[ "$_CERT_DAYS" -le "$CERT_CRIT_DAYS" ]]; then
error "$ICON_CERT $domain — CRITICAL: ${_CERT_DAYS} days remaining (expires $_CERT_EXPIRY)"
return 2
elif [[ "$days_remaining" -le "$CERT_WARN_DAYS" ]]; then
warn "$ICON_CERT $domain — WARNING: ${days_remaining} days remaining (expires $expiry_display)"
elif [[ "$_CERT_DAYS" -le "$CERT_WARN_DAYS" ]]; then
warn "$ICON_CERT $domain — WARNING: ${_CERT_DAYS} days remaining (expires $_CERT_EXPIRY)"
return 1
else
log "$ICON_CERT $domain — OK: ${days_remaining} days remaining (expires $expiry_display)"
log "$ICON_CERT $domain — OK: ${_CERT_DAYS} days remaining (expires $_CERT_EXPIRY)"
return 0
fi
}
@@ -285,7 +266,7 @@ fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
# ── Write JSON status cache ───────────────────────────────────────────────────
_CERT_CACHE_FILE="$SCRIPTS_DIR/State_Files/cert_status.json"
_CERT_CACHE_FILE="$STATE_DIR/cert_status.json"
{
printf '{"checked_at":%d,"host":"%s","warn_days":%d,"crit_days":%d,"dry_run":%s,"domains":[\n' \
"$(date +%s)" "$MY_ID" "$CERT_WARN_DAYS" "$CERT_CRIT_DAYS" \
Monitors/emby_session_report.sh
+1 -1
View File
@@ -316,7 +316,7 @@ echo ""
echo "━━━ $ICON_RAM Transcode Status ━━━"
if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
RAMDISK_USED_KB=$(df "$RAMDISK_PATH" --output=used 2>/dev/null | tail -1 | tr -d ' ')
RAMDISK_USED_GB=$(awk "BEGIN {printf \"%.2f\", ${RAMDISK_USED_KB:-0} / 1048576}")
RAMDISK_USED_GB=$(kb_to_gb "$RAMDISK_USED_KB")
SYMLINK=$(readlink "$TRANSCODE_LINK" 2>/dev/null || echo "unknown")
echo " $ICON_RAM Ramdisk usage: ${RAMDISK_USED_GB}GB / ${RAMDISK_SIZE:-8G}"
echo " $ICON_LINK Symlink target: $SYMLINK"
Monitors/smart_health.sh
+3 -23
View File
@@ -156,11 +156,7 @@ if [[ "$SHOW_STATUS" == true ]]; then
for drive in /dev/sd? /dev/nvme?; do
[[ ! -e "$drive" ]] && continue
drive_name=$(basename "$drive")
ignored=false
for ignore in "${SMART_IGNORE_DRIVES[@]}"; do
[[ "$drive_name" == "$ignore" ]] && ignored=true && break
done
if [[ "$ignored" == true ]]; then
if is_in_list "$drive_name" "${SMART_IGNORE_DRIVES[@]}"; then
echo " $ICON_WARN $drive — ignored"
else
echo " $ICON_SMART $drive — would check"
@@ -201,18 +197,7 @@ get_drive_temp() {
echo "${temp:-}"
}
# Detect if a drive is SSD/NVMe (rotational=0)
is_ssd() {
local drive="$1"
local dev_name
dev_name=$(basename "$drive" | sed 's/nvme[0-9]/nvme0/')
local rotational="/sys/block/$(basename "$drive")/queue/rotational"
[[ -f "$rotational" ]] && [[ "$(cat "$rotational" 2>/dev/null)" == "0" ]] && return 0
# NVMe is always SSD
[[ "$drive" == *nvme* ]] && return 0
return 1
}
# is_ssd() — provided by common.sh
# ==============================================================================================
# ━━━ SMART Health Check ━━━
# ==============================================================================================
@@ -232,12 +217,7 @@ for drive in /dev/sd? /dev/nvme?; do
drive_name=$(basename "$drive")
# Check ignore list
ignored=false
for ignore in "${SMART_IGNORE_DRIVES[@]}"; do
[[ "$drive_name" == "$ignore" ]] && ignored=true && break
done
if [[ "$ignored" == true ]]; then
if is_in_list "$drive_name" "${SMART_IGNORE_DRIVES[@]}"; then
log "$drive_name — ignored (SMART_IGNORE_DRIVES)"
DRIVES_SKIP+=("$drive_name")
continue
Monitors/weekly_health_digest.sh
+4 -8
View File
@@ -270,7 +270,7 @@ fi
# ── Transcode Ramdisk ─────────────────────────────────────────────────────────────────────────
if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
RAMDISK_USED_KB=$(df "$RAMDISK_PATH" --output=used 2>/dev/null | tail -1 | tr -d ' ')
RAMDISK_USED_GB=$(awk "BEGIN {printf \"%.2f\", ${RAMDISK_USED_KB:-0} / 1048576}")
RAMDISK_USED_GB=$(kb_to_gb "$RAMDISK_USED_KB")
SYMLINK_TARGET=$(readlink "$TRANSCODE_LINK" 2>/dev/null || echo "unknown")
DIGEST_LINES+=("$ICON_RAM Transcodes: ${RAMDISK_USED_GB}GB used → $SYMLINK_TARGET")
@@ -309,7 +309,7 @@ if [[ -f "${BANDWIDTH_LOG:-}" ]] && [[ -s "$BANDWIDTH_LOG" ]]; then
YESTERDAY=$(date -d "yesterday" '+%Y-%m-%d')
YESTERDAY_BYTES=$(awk -F'|' -v d="$YESTERDAY" '$1==d{sum+=$6} END{print sum+0}' \
"$BANDWIDTH_LOG")
YESTERDAY_GB=$(awk "BEGIN {printf \"%.2f\", ${YESTERDAY_BYTES:-0} / 1073741824}")
YESTERDAY_GB=$(bytes_to_gb "$YESTERDAY_BYTES")
YESTERDAY_LARGE=$(awk -F'|' -v d="$YESTERDAY" '$1==d && $7=="LARGE"' \
"$BANDWIDTH_LOG" | wc -l)
@@ -329,12 +329,8 @@ if [[ ${#CERT_MONITOR_DOMAINS[@]} -gt 0 ]] && command -v openssl >/dev/null 2>&1
CERT_ISSUES=()
for domain in "${CERT_MONITOR_DOMAINS[@]}"; do
[[ -z "$domain" ]] && continue
expiry_str=$(echo | timeout "${CERT_TIMEOUT:-10}" openssl s_client \
-connect "${domain}:443" -servername "$domain" \
2>/dev/null | openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2)
if [[ -n "$expiry_str" ]]; then
expiry_epoch=$(date -d "$expiry_str" +%s 2>/dev/null)
days_remaining=$(( (expiry_epoch - $(date +%s)) / 86400 ))
if check_cert_expiry "$domain" 443 "${CERT_TIMEOUT:-10}"; then
days_remaining="$_CERT_DAYS"
if [[ "$days_remaining" -le "${CERT_CRIT_DAYS:-7}" ]]; then
CERT_ISSUES+=("$domain: ${days_remaining}d CRITICAL")
SHOULD_SEND=true
Monitors/zfs_memory_snapshot.sh
+5 -5
View File
@@ -261,9 +261,9 @@ else
ARC_SIZE=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats)
ARC_META_USED=$(awk '/^arc_meta_used / {print $3}' /proc/spl/kstat/zfs/arcstats)
ARC_MAX_GB=$(awk "BEGIN {printf \"%.1f\", $ARC_MAX / 1073741824}")
ARC_CUR_GB=$(awk "BEGIN {printf \"%.1f\", $ARC_SIZE / 1073741824}")
ARC_META_GB=$(awk "BEGIN {printf \"%.1f\", $ARC_META_USED / 1073741824}")
ARC_MAX_GB=$(bytes_to_gb "$ARC_MAX" 1)
ARC_CUR_GB=$(bytes_to_gb "$ARC_SIZE" 1)
ARC_META_GB=$(bytes_to_gb "$ARC_META_USED" 1)
ARC_PCT=$(awk "BEGIN {printf \"%.1f\", $ARC_SIZE * 100 / $ARC_MAX}")
ARC_PCT_INT=$(printf "%.0f" "$ARC_PCT")
@@ -287,8 +287,8 @@ else
META_MISSES=$(awk '/^demand_metadata_misses / {print $3}' \
/proc/spl/kstat/zfs/arcstats 2>/dev/null || echo 0)
MRU_GB=$(awk "BEGIN {printf \"%.2f\", $META_MRU_GHOST / 1073741824}")
MFU_GB=$(awk "BEGIN {printf \"%.2f\", $META_MFU_GHOST / 1073741824}")
MRU_GB=$(bytes_to_gb "$META_MRU_GHOST")
MFU_GB=$(bytes_to_gb "$META_MFU_GHOST")
echo " $ICON_ZFS MRU Ghost: ${MRU_GB}GB"
echo " $ICON_ZFS MFU Ghost: ${MFU_GB}GB"
Notes_To-Do.md
-74
View File
@@ -1,74 +0,0 @@
need a script that check file locations. like if a kids movie is in movies. it needs to detect this and update the location in the arrs. so it would tell radarr to move shrek from movies to kids movies and trck the new file
❯ now we should be able to allow or have other host be able to start a shared srvice if desired using
shared auth stack, maybe a container array in host*.conf folder that can add the container to the other
servers, just thinkig of future design
array started go through logs.... array stopped didnt trigger when stopping the array, and this was before standardization
check all scripts for the dead verbose bug, like we did in orchs.... maybe even look for standardizing all scripts more like we did with orchs
pull to dev, dev is out of date
WAY LATER
now we should be able to allow or have other host be able to start a shared srvice if desired using
shared auth stack, maybe a container array in host*.conf folder that can add the container to the other
servers, just thinkig of future design
when a owner offboards and there is more than 1 server left, one needs to become the owner, we could do this in multiple ways or a combination, strongest server and BANDWIDTH,whos contributing more.... maybe we just promt and ask them
look into an overall, setup script, pull as many vars as possible without user having to add. like docker names, and so on.
fallback, should partnership be authoritive. like no partners fallback fails and after x amount of time dissables untill a new remote is found
i need a way to get rid of git. the apllication becomes the source for each mesh cluster... so it gets dl from github, or ca. i need the app layer to be able to open and edit settings, like open conf first. things have to filled out for it to work. , and partnership deploy to second server, third server........... and push the correct host.conf. and only update that file if new things are added or completely wiped....... cant mod a var nut can rearrang the script if structure changes again......... even if git it used as a core and is auto set from the start..... the app layer is king moving forward, and scipts are cores that just get called. i have all the flexabiliy in the world being able to change things and the app layers call and use it............
we may need a startup guide.. first launch, open master.conf. then host1.
we need to updata the folder Unraid_Scripts to Varaverk every where and update git script, and update all traces and scripts that call it and update the user script plugin......
with this app layer i think we modle the user script to our system and no longer user facing..... maybe
uide. owner sets up first.. establishes himself in master.conf as host1.... sets up the rest of master.conf.. little changes may be needed.. then fills out host1.conf.... then makes sure machines are shared in tail scale.. start onboard.......... then host2, host3, what ever is in master.conf it will reachout, uinstall correct containers,and leave the state file... install varaverk on host 2 and host 3. on install it sees the state file and defuaults to mirror. they fill out their setting and all should start working.......
working on getting this down to just web ui. on intitial no state file opens master.conf, locks you out of enabling any orchs untill set up.....
Orchestrators/transcode_management.sh
+12 -36
View File
@@ -88,8 +88,9 @@
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
source "$SCRIPT_DIR/../load_config.sh"
source "$ECOSYSTEM_ROOT/load_config.sh"
parse_args "$@"
@@ -158,17 +159,6 @@ if [[ "$SHOW_STATUS" == true ]]; then
exit 0
fi
# ==============================================================================================
# ━━━ Validate Child Scripts ━━━
# ==============================================================================================
for _entry in "${TRANSCODE_MANAGEMENT_SCRIPTS[@]}"; do
_script_path="$SCRIPT_DIR/../$_entry"
if [[ ! -f "$_script_path" ]]; then
error "$(basename "$_entry") not found: $_script_path"
exit 1
fi
done
# ==============================================================================================
# ━━━ Pre-run State Snapshot ━━━
# ==============================================================================================
@@ -188,36 +178,22 @@ fi
# ==============================================================================================
# transcode_manager.sh writes to TRANSCODE_DAILY_LOG after each run — no flag here
# suppresses that; it owns the log write for this cycle regardless of position ✅
DRY_FLAG=""
[[ "$DRY_RUN" == true ]] && DRY_FLAG="--dry-run"
WORST_EXIT=0
PASS_COUNT=0
FAIL_NAMES=()
JOB_PASS=()
JOB_FAIL=()
for _entry in "${TRANSCODE_MANAGEMENT_SCRIPTS[@]}"; do
_script_path="$SCRIPT_DIR/../$_entry"
_script_name=$(basename "$_entry" .sh)
_start=$(date +%s)
bash "$_script_path" $DRY_FLAG
_exit=$?
log "$_script_name: $(format_duration $(( $(date +%s) - _start ))) (exit $_exit)"
if [[ "$_exit" -ne 0 ]]; then
WORST_EXIT=1
FAIL_NAMES+=("$_script_name")
else
PASS_COUNT=$(( PASS_COUNT + 1 ))
fi
[[ -z "$_entry" ]] && continue
run_orch_child "$_entry"
done
# ==============================================================================================
# ━━━ Summary — minimal one-liner by default (7-min cadence — keep it quiet when healthy) ━━━
# ==============================================================================================
if [[ "$WORST_EXIT" -eq 0 ]]; then
echo "$ICON_SUCCESS Transcode cycle — $PASS_COUNT/${#TRANSCODE_MANAGEMENT_SCRIPTS[@]} passed"
if [[ "${#JOB_FAIL[@]}" -eq 0 ]]; then
echo "$ICON_SUCCESS Transcode cycle — ${#JOB_PASS[@]}/${#TRANSCODE_MANAGEMENT_SCRIPTS[@]} passed"
else
error "Transcode cycle — failed: ${FAIL_NAMES[*]}"
error "Transcode cycle — failed: ${JOB_FAIL[*]}"
if [[ "$DRY_RUN" != true ]]; then
notify "Transcode management failure on $(hostname) ($MY_ID) — ${FAIL_NAMES[*]}" \
notify "Transcode management failure on $(hostname) ($MY_ID) — ${JOB_FAIL[*]}" \
"Transcode Management" "warning"
fi
fi
@@ -225,5 +201,5 @@ fi
# ==============================================================================================
# ━━━ Exit ━━━
# ==============================================================================================
# Return worst exit code — caller knows if any script failed
exit "$WORST_EXIT"
[[ "${#JOB_FAIL[@]}" -gt 0 ]] && exit 1
exit 0
Orchestrators/watchdog_orchestrator.sh
+10 -38
View File
@@ -177,40 +177,12 @@ log "Startup grace: past — uptime $(format_duration $UPTIME_SECONDS)"
# ━━━ Run Watchdog Cycle ━━━
# ==============================================================================================
CYCLE_START=$(date +%s)
PASS=()
FAIL=()
run_watchdog() {
local name="$1" script="$2"
if [[ ! -f "$script" ]]; then
error "$name — not found: $script"
FAIL+=("$name:missing")
return 1
fi
[[ ! -x "$script" ]] && chmod +x "$script"
local extra_args=()
[[ "$DRY_RUN" == true ]] && extra_args+=("--dry-run")
[[ "$ENABLE_LOGGING" == true ]] && extra_args+=("--log")
local _ws
_ws=$(date +%s)
log "$ICON_START $name"
if bash "$script" "${extra_args[@]}"; then
log "$ICON_DONE $name — done in $(format_duration $(( $(date +%s) - _ws )))"
PASS+=("$name")
return 0
else
error "$name — non-zero exit ($(format_duration $(( $(date +%s) - _ws ))))"
FAIL+=("$name")
return 1
fi
}
JOB_PASS=()
JOB_FAIL=()
for _entry in "${WATCHDOG_ORCHESTRATOR_SCRIPTS[@]}"; do
run_watchdog "$(_watchdog_display_name "$_entry")" "$ECOSYSTEM_ROOT/$_entry"
[[ -z "$_entry" ]] && continue
run_orch_child "$_entry"
done
CYCLE_END=$(date +%s)
@@ -236,19 +208,19 @@ fi
# ==============================================================================================
# ━━━ Summary — minimal one-liner by default, full breakdown on failure or --log ━━━
# ==============================================================================================
if [[ "${#FAIL[@]}" -gt 0 || "$ENABLE_LOGGING" == true ]]; then
if [[ "${#JOB_FAIL[@]}" -gt 0 || "$ENABLE_LOGGING" == true ]]; then
echo ""
echo "━━━━━ $ICON_SUMMARY WATCHDOG CYCLE — $MY_ID — $(date '+%H:%M:%S') ━━━━━"
for p in "${PASS[@]}"; do log " $ICON_DONE $p"; done
for f in "${FAIL[@]}"; do error " $ICON_ERROR $f"; done
for p in "${JOB_PASS[@]}"; do log " $ICON_DONE $p"; done
for f in "${JOB_FAIL[@]}"; do error " $ICON_ERROR $f"; done
echo "$ICON_TIME Duration: $(format_duration $DURATION)"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
else
echo "$ICON_DONE Watchdog cycle — ${#PASS[@]}/${#WATCHDOG_ORCHESTRATOR_SCRIPTS[@]} passed ($(format_duration $DURATION))"
echo "$ICON_DONE Watchdog cycle — ${#JOB_PASS[@]}/${#WATCHDOG_ORCHESTRATOR_SCRIPTS[@]} passed ($(format_duration $DURATION))"
fi
if [[ "${#FAIL[@]}" -gt 0 ]]; then
notify "Watchdog cycle failure on $(hostname) ($MY_ID) — ${FAIL[*]}" \
if [[ "${#JOB_FAIL[@]}" -gt 0 ]]; then
notify "Watchdog cycle failure on $(hostname) ($MY_ID) — ${JOB_FAIL[*]}" \
"Watchdog Orchestrator" "warning"
exit 1
fi
Partnership/partnership_manager.sh
+31 -66
View File
@@ -249,22 +249,7 @@ fi
detect_hosts
# ── Derive owner and mirror from PARTNERSHIP_OWNER_HOST ───────────────────────────────────────
OWNER_ID="${PARTNERSHIP_OWNER_HOST:-HOST1}" # e.g. "HOST1"
MIRROR_ID=$( [[ "$OWNER_ID" == "HOST1" ]] && echo "HOST2" || echo "HOST1" )
OWNER="${!OWNER_ID}" # hostname string
MIRROR="${!MIRROR_ID}"
# SSH_KEY (set by detect_hosts) is this server's own private key.
# The remote accepts it because this server's PUBLIC key was installed there via ssh_setup.sh.
# With sparse checkout, each server only has its own host{N}.conf — the other server's
# key path is never available here. Use SSH_KEY for all outbound SSH regardless of mode.
MIRROR_SSH_KEY="$SSH_KEY"
OWNER_SSH_KEY="$SSH_KEY"
AM_OWNER=false
AM_MIRROR=false
[[ "$MY_ID" == "$OWNER_ID" ]] && AM_OWNER=true
[[ "$MY_ID" == "$MIRROR_ID" ]] && AM_MIRROR=true
partnership_resolve_roles
# State files
LOCAL_STATE_FILE="${STATE_DIR}/partnership_${LOCAL_SERVER_NAME}.db"
@@ -355,7 +340,7 @@ push_state_to_remote() {
warn "DRY RUN — would push state file to remote"
return 0
fi
timeout "$SSH_TIMEOUT" scp -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" \
timeout "$SSH_TIMEOUT" scp -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes \
"$local_file" "root@${remote_ip}:${local_file}" 2>/dev/null && \
echo "State file pushed to remote ✅" || \
warn "Could not push state file to remote — will propagate on next sync"
@@ -364,7 +349,7 @@ push_state_to_remote() {
read_remote_state() {
local remote_ip="$1" ssh_key="$2" remote_file="$3"
timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
-o ConnectTimeout="$SSH_TIMEOUT" root@"$remote_ip" \
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$remote_ip" \
"cat '$remote_file' 2>/dev/null" 2>/dev/null
}
@@ -524,34 +509,9 @@ gather_partner_fallback_containers() {
done
}
# Read a scalar var from the mirror's own config via SSH.
# Sources load_config.sh + detect_hosts() on the remote so HOST* aliasing works.
read_remote_conf_var() {
local mirror_ip="$1" var_name="$2"
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \
"source '$SCRIPT_DIR/../load_config.sh' 2>/dev/null
detect_hosts 2>/dev/null
printf '%s' \"\${${var_name}:-}\"" 2>/dev/null
}
# read_remote_conf_var() / read_remote_conf_array() — provided by common.sh
# Read an array var from the mirror's own config via SSH — one element per line.
read_remote_conf_array() {
local mirror_ip="$1" var_name="$2"
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \
"source '$SCRIPT_DIR/../load_config.sh' 2>/dev/null
detect_hosts 2>/dev/null
printf '%s\n' \"\${${var_name}[@]:-}\"" 2>/dev/null
}
# Returns just the short name portion: "unRAID-Gmer4Lfe" → "Gmer4Lfe"
derive_short_name() {
local hostname="$1"
local short="${hostname,,}"
[[ "$short" == unraid-* ]] && short="${short:7}"
echo "${short^}"
}
# derive_short_name() — provided by common.sh
# Start this server's own parked containers after partnership ends.
start_own_stack() {
@@ -644,7 +604,7 @@ cleanup_owner_containers_on_mirror() {
local container_list
container_list=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
-o ConnectTimeout="$SSH_TIMEOUT" root@"$mirror_ip" \
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \
"docker ps -a --format '{{.Names}}' 2>/dev/null | grep -i -- '-${owner_short}$'" 2>/dev/null)
if [[ -z "$container_list" ]]; then
@@ -658,12 +618,12 @@ cleanup_owner_containers_on_mirror() {
# Collect appdata paths before removal
local appdata_paths
appdata_paths=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
-o ConnectTimeout="$SSH_TIMEOUT" root@"$mirror_ip" \
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \
"docker inspect --format '{{range .HostConfig.Binds}}{{println .}}{{end}}' '$container' 2>/dev/null \
| awk -F: '{print \$1}' | grep '^/mnt/.*/appdata'" 2>/dev/null)
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
-o ConnectTimeout="$SSH_TIMEOUT" root@"$mirror_ip" \
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \
"docker stop '$container' >/dev/null 2>&1
docker rm '$container' >/dev/null 2>&1 && echo removed" 2>/dev/null | \
grep -q removed && \
@@ -674,7 +634,7 @@ cleanup_owner_containers_on_mirror() {
while IFS= read -r path; do
[[ -z "$path" ]] && continue
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
-o ConnectTimeout="$SSH_TIMEOUT" root@"$mirror_ip" \
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \
"rm -rf '$path' && echo removed" 2>/dev/null | grep -q removed && \
echo " Appdata removed on $MIRROR: $path ✅" || \
warn " Failed to remove appdata on $MIRROR: $path"
@@ -707,7 +667,7 @@ start_mirror_own_stack() {
continue
fi
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
-o ConnectTimeout="$SSH_TIMEOUT" root@"$mirror_ip" \
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \
"docker start '$container' >/dev/null 2>&1 && echo started" 2>/dev/null | \
grep -q started && \
echo "$container started on $MIRROR ✅" || \
@@ -881,7 +841,7 @@ check_both_healthy() {
[[ -z "$mirror_ip" ]] && { error "Cannot resolve $MIRROR Tailscale IP"; return 1; }
timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" \
-o ConnectTimeout="$SSH_TIMEOUT" root@"$mirror_ip" \
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \
"mountpoint -q '$REMOTE_STORAGE_PATH' && timeout 10 docker ps" >/dev/null 2>&1 || {
error "Mirror $MIRROR not healthy"
return 1
@@ -913,7 +873,8 @@ do_final_sync() {
warn "Final sync complete — mirror has current state ✅"
}
# Safe master.conf modification with error handling
# Safe master.conf modification with error handling — appends the key if not already present,
# since sed -i returns 0 whether or not it matched anything.
update_master_conf() {
local key="$1" value="$2"
local conf="$CONF_DIR/master.conf"
@@ -921,12 +882,22 @@ update_master_conf() {
error "master.conf not found at $conf"
return 1
fi
if sed -i "s|^[[:space:]]*${key}=.*| ${key}=${value}|" "$conf" 2>/dev/null; then
echo "master.conf updated: ${key}=${value}"
return 0
if grep -q "^[[:space:]]*${key}=" "$conf" 2>/dev/null; then
if sed -i "s|^[[:space:]]*${key}=.*| ${key}=${value}|" "$conf" 2>/dev/null; then
echo "master.conf updated: ${key}=${value}"
return 0
else
error "Failed to update master.conf: ${key}=${value}"
return 1
fi
else
error "Failed to update master.conf: ${key}=${value}"
return 1
if echo " ${key}=${value}" >> "$conf" 2>/dev/null; then
echo "master.conf updated: ${key}=${value} (appended)"
return 0
else
error "Failed to append to master.conf: ${key}=${value}"
return 1
fi
fi
}
@@ -1162,13 +1133,7 @@ if [[ "$MODE" == "onboard" ]]; then
echo ""
echo "Enabling partnership in master.conf..."
if [[ "$DRY_RUN" == false ]]; then
_master_conf="$CONF_DIR/master.conf"
if grep -q "^[[:space:]]*PARTNERSHIP_ENABLED=" "$_master_conf" 2>/dev/null; then
sed -i "s|^[[:space:]]*PARTNERSHIP_ENABLED=.*|PARTNERSHIP_ENABLED=true|" "$_master_conf"
else
echo "PARTNERSHIP_ENABLED=true" >> "$_master_conf"
fi
echo "PARTNERSHIP_ENABLED=true in master.conf ✅"
update_master_conf "PARTNERSHIP_ENABLED" "true"
platform_push_conf | while IFS= read -r line; do log "$line"; done
else
warn "DRY RUN — would set PARTNERSHIP_ENABLED=true in master.conf and push"
@@ -1435,9 +1400,9 @@ if false; then
# Update remote master.conf
timeout "$SSH_TIMEOUT" ssh -i "$NEW_MIRROR_SSH_KEY" \
-o ConnectTimeout="$SSH_TIMEOUT" root@"$NEW_MIRROR_IP" \
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$NEW_MIRROR_IP" \
"sed -i 's|^[[:space:]]*PARTNERSHIP_OWNER_HOST=.*| PARTNERSHIP_OWNER_HOST=\"$NEW_OWNER_ID\"|' \
'$SCRIPT_DIR/../master.conf'" 2>/dev/null && \
'$SCRIPT_DIR/../Configurations/master.conf'" 2>/dev/null && \
echo "Remote master.conf updated ✅" || \
error "Failed to update remote master.conf — update manually"
else
Partnership/partnership_offboard.sh
+1 -11
View File
@@ -108,17 +108,7 @@ fi
detect_hosts
OWNER_ID="${PARTNERSHIP_OWNER_HOST:-HOST1}"
MIRROR_ID=$( [[ "$OWNER_ID" == "HOST1" ]] && echo "HOST2" || echo "HOST1" )
OWNER="${!OWNER_ID}"
MIRROR="${!MIRROR_ID}"
MIRROR_SSH_KEY="$SSH_KEY"
OWNER_SSH_KEY="$SSH_KEY"
AM_OWNER=false
AM_MIRROR=false
[[ "$MY_ID" == "$OWNER_ID" ]] && AM_OWNER=true
[[ "$MY_ID" == "$MIRROR_ID" ]] && AM_MIRROR=true
partnership_resolve_roles
LOCAL_STATE_FILE="${STATE_DIR}/partnership_${LOCAL_SERVER_NAME}.db"
REMOTE_STATE_FILE="${STATE_DIR}/partnership_${REMOTE_SERVER_NAME}.db"
Partnership/partnership_onboard.sh
+8 -37
View File
@@ -223,20 +223,7 @@ fi
detect_hosts
OWNER_ID="${PARTNERSHIP_OWNER_HOST:-HOST1}"
MIRROR_ID=$( [[ "$OWNER_ID" == "HOST1" ]] && echo "HOST2" || echo "HOST1" )
OWNER="${!OWNER_ID}"
MIRROR="${!MIRROR_ID}"
# SSH_KEY (set by detect_hosts) is this server's own private key.
# The remote accepts it because this server's PUBLIC key was installed there via ssh_setup.sh.
# HOST{N}_SSH_KEY lives in host{N}.conf — with sparse checkout, the other server's
# conf is never present here. Always use SSH_KEY (local private key) for outbound SSH.
MIRROR_SSH_KEY="$SSH_KEY"
AM_OWNER=false
AM_MIRROR=false
[[ "$MY_ID" == "$OWNER_ID" ]] && AM_OWNER=true
[[ "$MY_ID" == "$MIRROR_ID" ]] && AM_MIRROR=true
partnership_resolve_roles
EXTRA_FLAGS=()
[[ "$DRY_RUN" == true ]] && EXTRA_FLAGS+=("--dry-run")
@@ -250,11 +237,7 @@ write_onboard_phase() {
local key="${target_id}_PHASE${phase}_DONE"
local state_file="$(platform_setup_db_path)"
[[ "$DRY_RUN" == true ]] && { warn "DRY RUN — would write ${key}=true"; return 0; }
if grep -q "^${key}=" "$state_file" 2>/dev/null; then
sed -i "s|^${key}=.*|${key}=true|" "$state_file"
else
echo "${key}=true" >> "$state_file"
fi
set_state_var "$state_file" "$key" "true"
platform_push_setup_state
}
@@ -282,13 +265,7 @@ stop_mirror_stack() {
local config_var="$1" label="$2"
local -a to_stop=()
mapfile -t to_stop < <(
timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" \
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$MIRROR_IP" \
"source '$SCRIPTS_ROOT/load_config.sh' 2>/dev/null
detect_hosts 2>/dev/null
printf '%s\n' \"\${${config_var}[@]:-}\"" 2>/dev/null | grep -v '^$'
)
mapfile -t to_stop < <(read_remote_conf_array "$MIRROR_IP" "$config_var" | grep -v '^$')
if [[ ${#to_stop[@]} -eq 0 ]]; then
log "No $label containers to stop on $MIRROR — skipping"
@@ -302,7 +279,7 @@ stop_mirror_stack() {
continue
fi
timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" \
-o ConnectTimeout="$SSH_TIMEOUT" root@"$MIRROR_IP" \
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$MIRROR_IP" \
"docker stop '$container' 2>/dev/null
docker rm '$container' 2>/dev/null && echo removed" 2>/dev/null | \
grep -q removed && \
@@ -338,11 +315,7 @@ if [[ "$AM_MIRROR" == true ]]; then
if [[ -n "$OWNER_IP" ]]; then
# Read OWNER's SCRIPTS_DIR via platform probe command — don't assume same path as mirror
_probe_cmd=$(platform_scripts_dir_probe_cmd)
OWNER_SCRIPTS_DIR=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$OWNER_IP" \
"$_probe_cmd" 2>/dev/null | tr -d '[:space:]')
OWNER_SCRIPTS_DIR="${OWNER_SCRIPTS_DIR:-$SCRIPTS_DIR}"
OWNER_SCRIPTS_DIR=$(resolve_remote_scripts_dir "$OWNER_IP")
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would SSH to $OWNER ($OWNER_IP) and trigger Phase 2"
@@ -444,11 +417,9 @@ elif [[ "$PHASE1_ONLY" == true ]]; then
echo " Then click 'Push Conf' in the Partnership tab."
# Write key-ready flag so UI can show the manual-install state
[[ "$DRY_RUN" == false ]] && {
local kflag="${MIRROR_ID}_KEY_READY"
local _setup_f="$(platform_setup_db_path)"
grep -q "^${kflag}=" "$_setup_f" 2>/dev/null \
&& sed -i "s|^${kflag}=.*|${kflag}=true|" "$_setup_f" \
|| echo "${kflag}=true" >> "$_setup_f"
kflag="${MIRROR_ID}_KEY_READY"
_setup_f="$(platform_setup_db_path)"
set_state_var "$_setup_f" "$kflag" "true"
}
fi
STEP_SSH_OK=false
Partnership/partnership_transfer.sh
+3 -16
View File
@@ -128,17 +128,7 @@ fi
detect_hosts
OWNER_ID="${PARTNERSHIP_OWNER_HOST:-HOST1}"
MIRROR_ID=$( [[ "$OWNER_ID" == "HOST1" ]] && echo "HOST2" || echo "HOST1" )
OWNER="${!OWNER_ID}"
MIRROR="${!MIRROR_ID}"
MIRROR_SSH_KEY="$SSH_KEY"
OWNER_SSH_KEY="$SSH_KEY"
AM_OWNER=false
AM_MIRROR=false
[[ "$MY_ID" == "$OWNER_ID" ]] && AM_OWNER=true
[[ "$MY_ID" == "$MIRROR_ID" ]] && AM_MIRROR=true
partnership_resolve_roles
LOCAL_STATE_FILE="${STATE_DIR}/partnership_${LOCAL_SERVER_NAME}.db"
REMOTE_STATE_FILE="${STATE_DIR}/partnership_${REMOTE_SERVER_NAME}.db"
@@ -286,14 +276,11 @@ if [[ "$DRY_RUN" == false ]]; then
# Push updated master.conf to new owner so both servers agree immediately.
# master.conf is shared — host-specific credentials live in host*.conf.
_probe_cmd=$(platform_scripts_dir_probe_cmd)
_REMOTE_SD=$(ssh -i "$SSH_KEY" -o ConnectTimeout=5 -o StrictHostKeyChecking=no \
"root@${MIRROR_IP}" "$_probe_cmd" \
2>/dev/null | tr -d '[:space:]')
_REMOTE_SD="${_REMOTE_SD:-$SCRIPTS_DIR}"
_REMOTE_SD=$(resolve_remote_scripts_dir "$MIRROR_IP" "$SSH_KEY" "no")
scp -i "$SSH_KEY" \
-o ConnectTimeout="$SSH_TIMEOUT" \
-o StrictHostKeyChecking=no \
-o BatchMode=yes \
"$SCRIPTS_ROOT/Configurations/master.conf" \
"root@${MIRROR_IP}:${_REMOTE_SD}/Configurations/master.conf" 2>/dev/null && \
echo "master.conf pushed to $NEW_OWNER ✅" || \
Partnership/share_setup.sh
+1 -1
View File
@@ -52,7 +52,7 @@ MIRROR_IP=$(resolve_tailscale_ip "$MIRROR")
[[ -z "$MIRROR_IP" ]] && { error "Cannot resolve $MIRROR Tailscale IP — is Tailscale running?"; exit 1; }
_ssh() { ssh -i "$SSH_KEY" -o ConnectTimeout=10 -o BatchMode=yes -o StrictHostKeyChecking=no root@"$MIRROR_IP" "$@"; }
_scp() { scp -i "$SSH_KEY" -o ConnectTimeout=10 -o StrictHostKeyChecking=no "$@"; }
_scp() { scp -i "$SSH_KEY" -o ConnectTimeout=10 -o BatchMode=yes -o StrictHostKeyChecking=no "$@"; }
# ── Detect remote appdata pool ────────────────────────────────────────────────────────────────
Partnership/ssh_setup.sh
+4 -3
View File
@@ -74,9 +74,10 @@ acquire_lock
detect_hosts
# ── Derive key name from hostname ─────────────────────────────────────────────────────────────
# Strip unraid- prefix (case-insensitive) if present → gmer4lfe_rsync_automation
SHORT_NAME="${LOCAL_SERVER_NAME,,}"
[[ "${SHORT_NAME}" == unraid-* ]] && SHORT_NAME="${SHORT_NAME:7}"
# derive_short_name() title-cases the result (for display elsewhere) — lowercase it here,
# same as before, since this feeds a filename → gmer4lfe_rsync_automation
SHORT_NAME="$(derive_short_name "$LOCAL_SERVER_NAME")"
SHORT_NAME="${SHORT_NAME,,}"
SSH_KEY_NAME="${SHORT_NAME}_rsync_automation"
SSH_KEY_PATH="/root/.ssh/${SSH_KEY_NAME}"
SSH_PUB_PATH="${SSH_KEY_PATH}.pub"
Plugin/unraid/Partnership/containers.sh
+7 -7
View File
@@ -220,7 +220,7 @@ deploy_container_from_xml() {
log "Deploying $name..."
if [[ "$DRY_RUN" == false ]]; then
timeout "$SSH_TIMEOUT" scp -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" \
timeout "$SSH_TIMEOUT" scp -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes \
"$xml_file" "root@${remote_ip}:${TEMPLATES_DIR}/${xml_name}" 2>/dev/null || {
rm -f "$_gpu_tmp"
warn " SCP failed for $xml_name — skipping $name"
@@ -291,9 +291,9 @@ deploy_container_from_xml() {
local remote_script="/tmp/deploy_${name//[^a-zA-Z0-9_]/_}.sh"
if timeout "$SSH_TIMEOUT" scp -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" \
if timeout "$SSH_TIMEOUT" scp -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes \
"$tmp_script" "root@${remote_ip}:${remote_script}" 2>/dev/null && \
timeout 120 ssh -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" root@"$remote_ip" \
timeout 120 ssh -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$remote_ip" \
"bash '$remote_script' 2>&1; rc=\$?; rm -f '$remote_script'; exit \$rc" 2>/dev/null | \
grep -q "deployed:${name}"; then
echo " $name deployed ✅"
@@ -386,7 +386,7 @@ cleanup_deployed_stack_on_remote() {
| awk -F: '{print \$1}' | grep '^/mnt/.*/appdata'" 2>/dev/null)
timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
-o ConnectTimeout="$SSH_TIMEOUT" root@"$remote_ip" \
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$remote_ip" \
"docker stop '$cname' >/dev/null 2>&1
docker rm '$cname' >/dev/null 2>&1 && echo removed" 2>/dev/null | \
grep -q removed && \
@@ -396,7 +396,7 @@ cleanup_deployed_stack_on_remote() {
while IFS= read -r path; do
[[ -z "$path" ]] && continue
timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
-o ConnectTimeout="$SSH_TIMEOUT" root@"$remote_ip" \
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$remote_ip" \
"rm -rf '$path' && echo removed" 2>/dev/null | grep -q removed && \
echo " Appdata removed on $MIRROR: $path ✅" || \
warn " Failed to remove appdata on $MIRROR: $path"
@@ -504,7 +504,7 @@ reconfigure_webui() {
local template
template=$(timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
-o ConnectTimeout="$SSH_TIMEOUT" root@"$remote_ip" \
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$remote_ip" \
"grep -rl '<WebUI>' '$TEMPLATES_DIR/' 2>/dev/null | \
xargs grep -l '\"$container\"' 2>/dev/null | head -1" 2>/dev/null)
@@ -514,7 +514,7 @@ reconfigure_webui() {
fi
timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
-o ConnectTimeout="$SSH_TIMEOUT" root@"$remote_ip" \
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$remote_ip" \
"sed -i 's|<WebUI>.*</WebUI>|<WebUI>http://${target_ip}:${port}/</WebUI>|g' '$template'" \
2>/dev/null && \
echo "$container → http://${target_ip}:${port}/ ✅" || {
Rsync/rsync.sh
+7 -2
View File
@@ -56,7 +56,7 @@
#
# Remote Health Pre-flights
# check_connectivity() — Tailscale IP reachable before any SSH
# check_remote_rootfs() — aborts if remote rootfs exceeds ROOTFS_WARN_PCT
# check_remote_rootfs() — aborts if remote rootfs exceeds ROOTFS_WARN
# check_remote_share() — aborts if target directory missing or empty on remote
# check_remote_disks() — verifies all backing disks online on remote
#
@@ -98,7 +98,7 @@
# SLEEP
# Default seconds between retry attempts. (default: 60)
#
# ROOTFS_WARN_PCT
# ROOTFS_WARN
# Abort threshold for remote rootfs percentage full. (default: 75)
#
# PROFILES["profile_KEY"]
@@ -223,6 +223,11 @@ VV_LAST_LOG="/tmp/unraid_locks/rsync_${PROFILE_NAME}.last.log"
exec 1> >(tee -a "$VV_LIVE_LOG") 2>&1
# ── Load profile settings ─────────────────────────────────────────────────────────────────────
# NOTE: DEFAULT_RSYNC_OPTS and PROFILE_RSYNC_OPTS are bash arrays defined in master.conf —
# any "$BW_LIMIT" inside them is expanded once, when master.conf is sourced, before this
# override runs. So this recomputed $BW_LIMIT only affects the log line below and --status;
# it does NOT change the actual rsync --bwlimit unless a profile also bakes its own
# --bwlimit="${PROFILE_BW_LIMIT[name]}" directly into that profile's PROFILE_RSYNC_OPTS entry.
BW_LIMIT=${PROFILE_BW_LIMIT[$PROFILE_NAME]:-$BW_LIMIT}
RETRY_COUNT=${PROFILE_RETRY_COUNT[$PROFILE_NAME]:-$RETRY_COUNT}
SLEEP=${PROFILE_SLEEP[$PROFILE_NAME]:-$SLEEP}
System_Essentials/conf_cache_restore.sh
+2 -2
View File
@@ -53,7 +53,7 @@ parse_args "$@"
detect_hosts
require_partnership
RAM_CACHE="/tmp/.cache/vv/d"
RAM_CACHE="$CONF_RAM_CACHE_DIR"
SAVE_DIR="$PERSISTENT_CONF_CACHE"
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
@@ -67,7 +67,7 @@ restored=0
for conf in "$SAVE_DIR"/host*.conf; do
[[ -f "$conf" ]] || continue
base="$(basename "$conf")"
[[ "${base,,}" == "${MY_ID,,}.conf" ]] && continue
is_own_conf_file "$base" && continue
if [[ -f "$RAM_CACHE/$base" ]]; then
log "$base already in RAM cache (conf_sync succeeded) — skipping"
System_Essentials/conf_cache_save.sh
+2 -2
View File
@@ -47,7 +47,7 @@ parse_args "$@"
detect_hosts
require_partnership
RAM_CACHE="/tmp/.cache/vv/d"
RAM_CACHE="$CONF_RAM_CACHE_DIR"
SAVE_DIR="$PERSISTENT_CONF_CACHE"
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be written"
@@ -61,7 +61,7 @@ saved=0
for conf in "$RAM_CACHE"/host*.conf; do
[[ -f "$conf" ]] || continue
base="$(basename "$conf")"
[[ "${base,,}" == "${MY_ID,,}.conf" ]] && continue
is_own_conf_file "$base" && continue
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would save $base → $SAVE_DIR/"
System_Essentials/conf_sync.sh
+1 -1
View File
@@ -81,7 +81,7 @@ if [[ "${CONF_SYNC_ENABLED:-true}" == false ]]; then
exit 0
fi
CACHE_DIR="/tmp/.cache/vv/d"
CACHE_DIR="$CONF_RAM_CACHE_DIR"
MY_CONF="$SCRIPTS_DIR/Configurations/${MY_ID,,}.conf"
SSH_TIMEOUT=10
Tools/container_data_export.sh
+6 -71
View File
@@ -146,17 +146,7 @@ cleanup_on_exit() {
warn "Removing partial archive: $ARCHIVE_PATH"
rm -f "$ARCHIVE_PATH" 2>/dev/null
fi
# Always restart container if it was running
if [[ "$CONTAINER_WAS_RUNNING" == true && "$DRY_RUN" == false ]]; then
local status
status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \
"$CONTAINER_NAME" 2>/dev/null)
if [[ "$status" != "true" ]]; then
warn "Restarting $CONTAINER_NAME (cleanup)..."
timeout "$DOCKER_TIMEOUT" docker start "$CONTAINER_NAME" >/dev/null 2>&1 || \
error "Failed to restart $CONTAINER_NAME — start it manually"
fi
fi
container_force_restart_if_needed "$CONTAINER_NAME" "$CONTAINER_WAS_RUNNING"
}
trap cleanup_on_exit EXIT
@@ -166,35 +156,10 @@ trap cleanup_on_exit EXIT
echo ""
echo "━━━ $ICON_STOP Stop Container ━━━"
STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \
"$CONTAINER_NAME" 2>/dev/null)
case "$STATUS" in
true)
CONTAINER_WAS_RUNNING=true
log "Stopping $CONTAINER_NAME for clean export..."
if [[ "$DRY_RUN" == false ]]; then
if timeout "$DOCKER_TIMEOUT" docker stop "$CONTAINER_NAME" >/dev/null 2>&1; then
echo "$CONTAINER_NAME stopped ✅"
else
error "Failed to stop $CONTAINER_NAME — aborting export"
exit 1
fi
else
warn "DRY RUN — would stop $CONTAINER_NAME"
fi
;;
false)
echo "$CONTAINER_NAME is not running — archiving as-is (was stopped state respected)"
;;
"")
error "$CONTAINER_NAME not found — check container name"
exit 1
;;
*)
warn "$CONTAINER_NAME status: $STATUS — proceeding with caution"
;;
esac
container_stop_for_maintenance "$CONTAINER_NAME" CONTAINER_WAS_RUNNING \
"Stopping $CONTAINER_NAME for clean export..." log || exit 1
[[ "$CONTAINER_WAS_RUNNING" == false ]] && \
echo "$CONTAINER_NAME is not running — archiving as-is (was stopped state respected)"
# ==============================================================================================
# ━━━ Archive ━━━
@@ -249,37 +214,7 @@ END=$(date +%s)
echo ""
echo "━━━ $ICON_START Restart Container ━━━"
RESTART_OK=false
if [[ "$CONTAINER_WAS_RUNNING" == true ]]; then
log "Restarting $CONTAINER_NAME..."
if [[ "$DRY_RUN" == false ]]; then
if timeout "$DOCKER_TIMEOUT" docker start "$CONTAINER_NAME" >/dev/null 2>&1; then
# Brief settle then verify
sleep 3
POST_STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
'{{.State.Running}}' "$CONTAINER_NAME" 2>/dev/null)
if [[ "$POST_STATUS" == "true" ]]; then
echo "$CONTAINER_NAME restarted and running ✅"
RESTART_OK=true
else
error "$CONTAINER_NAME started but crashed immediately — check container logs"
notify "$CONTAINER_NAME failed to stay running after export on $(hostname)" \
"Container Export" "warning"
fi
else
error "Failed to restart $CONTAINER_NAME — start it manually"
notify "$CONTAINER_NAME failed to restart after export on $(hostname)" \
"Container Export" "warning"
fi
else
warn "DRY RUN — would restart $CONTAINER_NAME"
RESTART_OK=true
fi
else
echo "$CONTAINER_NAME was not running — leaving stopped (state respected) ✅"
RESTART_OK=true
fi
container_restart_after_maintenance "$CONTAINER_NAME" "$CONTAINER_WAS_RUNNING" 3 "Container Export"
# Clear trap — clean exit, cleanup_on_exit no longer needed
trap - EXIT
Tools/emby_database_repair.sh
+7 -71
View File
@@ -171,16 +171,7 @@ EMBY_WAS_RUNNING=false
cleanup_on_exit() {
local exit_code=$?
if [[ "$EMBY_WAS_RUNNING" == true && "$DRY_RUN" == false ]]; then
local status
status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \
"$EMBY_CONTAINER" 2>/dev/null)
if [[ "$status" != "true" ]]; then
warn "Restarting $EMBY_CONTAINER (cleanup)..."
timeout "$DOCKER_TIMEOUT" docker start "$EMBY_CONTAINER" >/dev/null 2>&1 || \
error "Failed to restart $EMBY_CONTAINER — start it manually"
fi
fi
container_force_restart_if_needed "$EMBY_CONTAINER" "$EMBY_WAS_RUNNING"
}
trap cleanup_on_exit EXIT
@@ -191,36 +182,10 @@ trap cleanup_on_exit EXIT
echo ""
echo "━━━ $ICON_STOP Stop Emby ━━━"
STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \
"$EMBY_CONTAINER" 2>/dev/null)
case "$STATUS" in
true)
EMBY_WAS_RUNNING=true
warn "Stopping $EMBY_CONTAINER — active sessions will be interrupted"
if [[ "$DRY_RUN" == false ]]; then
if timeout "$DOCKER_TIMEOUT" docker stop "$EMBY_CONTAINER" >/dev/null 2>&1; then
echo "$EMBY_CONTAINER stopped ✅"
sleep 3 # let file handles release
else
error "Failed to stop $EMBY_CONTAINER — aborting"
exit 1
fi
else
warn "DRY RUN — would stop $EMBY_CONTAINER"
fi
;;
false)
log "$EMBY_CONTAINER is not running — proceeding with checks"
;;
"")
error "$EMBY_CONTAINER not found — check container name"
exit 1
;;
*)
warn "$EMBY_CONTAINER status: $STATUS — proceeding with caution"
;;
esac
container_stop_for_maintenance "$EMBY_CONTAINER" EMBY_WAS_RUNNING \
"Stopping $EMBY_CONTAINER — active sessions will be interrupted" warn 3 || exit 1
[[ "$EMBY_WAS_RUNNING" == false ]] && \
log "$EMBY_CONTAINER is not running — proceeding with checks"
# ==============================================================================================
# ━━━ Database Integrity Check ━━━
@@ -301,37 +266,8 @@ END=$(date +%s)
echo ""
echo "━━━ $ICON_START Restart Emby ━━━"
RESTART_OK=false
if [[ "$EMBY_WAS_RUNNING" == true ]]; then
if [[ "$DRY_RUN" == false ]]; then
log "Restarting $EMBY_CONTAINER..."
if timeout "$DOCKER_TIMEOUT" docker start "$EMBY_CONTAINER" >/dev/null 2>&1; then
sleep 5 # Emby takes longer to initialise than most containers
POST_STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
'{{.State.Running}}' "$EMBY_CONTAINER" 2>/dev/null)
if [[ "$POST_STATUS" == "true" ]]; then
echo "$EMBY_CONTAINER restarted and running ✅"
RESTART_OK=true
else
error "$EMBY_CONTAINER started but crashed — database may be corrupt"
error "Check Docker logs: docker logs $EMBY_CONTAINER"
notify "$EMBY_CONTAINER crashed on restart — possible database corruption on $(hostname)" \
"Emby DB Repair" "warning"
fi
else
error "Failed to restart $EMBY_CONTAINER — start it manually"
notify "$EMBY_CONTAINER failed to restart after integrity check on $(hostname)" \
"Emby DB Repair" "warning"
fi
else
warn "DRY RUN — would restart $EMBY_CONTAINER"
RESTART_OK=true
fi
else
echo "$EMBY_CONTAINER was not running — leaving stopped (state respected) ✅"
RESTART_OK=true
fi
container_restart_after_maintenance "$EMBY_CONTAINER" "$EMBY_WAS_RUNNING" 5 "Emby DB Repair"
[[ "$RESTART_OK" == false ]] && error "Check Docker logs: docker logs $EMBY_CONTAINER"
# Clear EXIT trap — clean exit
trap - EXIT
Tools/smart_long_test.sh
+3 -10
View File
@@ -138,11 +138,8 @@ _build_drive_list() {
[[ ! -e "$drive" ]] && continue
local drive_name
drive_name=$(basename "$drive")
local ignored=false
for ignore in "${SMART_IGNORE_DRIVES[@]:-}"; do
[[ "$drive_name" == "$ignore" ]] && ignored=true && break
done
[[ "$ignored" == true ]] && { log "Skipping $drive_name (SMART_IGNORE_DRIVES)"; continue; }
is_in_list "$drive_name" "${SMART_IGNORE_DRIVES[@]:-}" && \
{ log "Skipping $drive_name (SMART_IGNORE_DRIVES)"; continue; }
if ! smartctl -i "$drive" 2>/dev/null | grep -q "SMART support is: Enabled"; then
log "Skipping $drive_name — SMART not enabled"
continue
@@ -166,11 +163,7 @@ if [[ "$SHOW_STATUS" == true ]]; then
drive_name=$(basename "$drive")
local_result=$(smartctl -l selftest "$drive" 2>/dev/null | \
grep -m1 "Extended" | awk '{print $NF}')
ignored=false
for ignore in "${SMART_IGNORE_DRIVES[@]:-}"; do
[[ "$drive_name" == "$ignore" ]] && ignored=true && break
done
if [[ "$ignored" == true ]]; then
if is_in_list "$drive_name" "${SMART_IGNORE_DRIVES[@]:-}"; then
echo " $ICON_WARN $drive_name — ignored"
else
echo " $ICON_SMART $drive_name — last extended: ${local_result:-no result}"
Transcodes/transcode_cleanup.sh
+2 -10
View File
@@ -240,15 +240,7 @@ cleanup_location() {
# Format bytes freed
local freed_human
if (( bytes_freed > 1073741824 )); then
freed_human=$(awk "BEGIN {printf \"%.1fGB\", $bytes_freed / 1073741824}")
elif (( bytes_freed > 1048576 )); then
freed_human=$(awk "BEGIN {printf \"%.1fMB\", $bytes_freed / 1048576}")
elif (( bytes_freed > 0 )); then
freed_human="${bytes_freed}B"
else
freed_human="0B"
fi
freed_human=$(format_bytes "$bytes_freed")
log "$label — removed $files_removed ($freed_human) | active(fresh): $files_too_young | streaming(lsof): $files_streaming | protected(old+open): $files_active | skipped: $files_skipped | failed: $files_failed"
@@ -299,7 +291,7 @@ fi
# Post-cleanup — check if ramdisk recovered enough to flip symlink back to ramdisk
if [[ "$DRY_RUN" == false ]] && mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
RAMDISK_USED_KB=$(df "$RAMDISK_PATH" --output=used 2>/dev/null | tail -1 | tr -d ' ')
RAMDISK_USED_GB=$(awk "BEGIN {printf \"%.2f\", ${RAMDISK_USED_KB:-0} / 1048576}")
RAMDISK_USED_GB=$(kb_to_gb "$RAMDISK_USED_KB")
LOW_RECOVERED=$(awk "BEGIN {print ($RAMDISK_USED_GB < $RAMDISK_LOW_GB) ? 1 : 0}")
CURRENT_TARGET=$(grep "^current_target=" "$STATE_FILE" 2>/dev/null | cut -d= -f2)
Transcodes/transcode_manager.sh
+3 -6
View File
@@ -234,18 +234,15 @@ fix_permissions() {
}
get_ramdisk_used_gb() {
df "$RAMDISK_PATH" --output=used 2>/dev/null | \
tail -1 | tr -d ' ' | awk '{printf "%.2f", $1/1048576}'
kb_to_gb "$(df "$RAMDISK_PATH" --output=used 2>/dev/null | tail -1 | tr -d ' ')"
}
get_ramdisk_avail_gb() {
df "$RAMDISK_PATH" --output=avail 2>/dev/null | \
tail -1 | tr -d ' ' | awk '{printf "%.2f", $1/1048576}'
kb_to_gb "$(df "$RAMDISK_PATH" --output=avail 2>/dev/null | tail -1 | tr -d ' ')"
}
get_ssd_free_gb() {
df "$TRANSCODE_SSD" --output=avail 2>/dev/null | \
tail -1 | tr -d ' ' | awk '{printf "%.2f", $1/1048576}'
kb_to_gb "$(df "$TRANSCODE_SSD" --output=avail 2>/dev/null | tail -1 | tr -d ' ')"
}
get_flip_count() {
Watchdogs/System/network_watchdog.sh
+3 -10
View File
@@ -125,16 +125,9 @@ log "$ICON_NET DDNS: ${DDNS_DOMAIN:-not configured} → ${DDNS_CONTAINER:-no c
touch "${NETWORK_WATCHDOG_NPM_STATE_FILE}" 2>/dev/null
# ━━━ Strike helpers ━━━
get_strikes() { grep "^${1}:" "${2}" 2>/dev/null | cut -d: -f2 || echo "0"; }
set_strikes() {
local key="$1" count="$2" file="$3"
if grep -q "^${key}:" "$file" 2>/dev/null; then
sed -i "s|^${key}:.*|${key}:${count}|" "$file"
else
echo "${key}:${count}" >> "$file"
fi
}
# ━━━ Strike helpers — wrap common.sh's wd_state_get()/wd_state_set() ━━━
get_strikes() { wd_state_get "$1" "$2"; }
set_strikes() { wd_state_set "$1" "$2" "$3"; }
# ==============================================================================================
# ━━━ Status ━━━
Watchdogs/System/storage_watchdog.sh
+3 -10
View File
@@ -164,16 +164,9 @@ log "$ICON_DISK Paths: ${WATCHDOG_APPDATA_PATHS[*]:-none configured}"
touch "$STORAGE_WATCHDOG_STATE_FILE" 2>/dev/null
touch "$WATCHDOG_APPDATA_GROWTH_FILE" 2>/dev/null
# ━━━ Strike helpers ━━━
get_strikes() { grep "^${1}:" "${2}" 2>/dev/null | cut -d: -f2 || echo "0"; }
set_strikes() {
local key="$1" count="$2" file="$3"
if grep -q "^${key}:" "$file" 2>/dev/null; then
sed -i "s|^${key}:.*|${key}:${count}|" "$file"
else
echo "${key}:${count}" >> "$file"
fi
}
# ━━━ Strike helpers — wrap common.sh's wd_state_get()/wd_state_set() ━━━
get_strikes() { wd_state_get "$1" "$2"; }
set_strikes() { wd_state_set "$1" "$2" "$3"; }
# ==============================================================================================
# ━━━ Status ━━━
Watchdogs/docker_watchdog.sh
+9 -10
View File
@@ -194,6 +194,9 @@
# WATCHDOG_CONTAINER_RESTART_LIMIT / WATCHDOG_CONTAINER_RESTART_WINDOW
# Restart loop protection: attempt limit and rolling window in hours
#
# WATCHDOG_REQUIRED_STRIKE_LIMIT
# Consecutive down-checks on a required container before a restart is attempted (default: 2)
#
# WATCHDOG_BATCH_NOTIFY
# Collect cycle events and send as one notification (default: true)
#
@@ -337,19 +340,15 @@ fi
# ── HELPER FUNCTIONS ──────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Get strike count for a container from state file
# Get strike count for a container from state file — wraps common.sh's wd_state_get()
get_strikes() {
grep "^${1}:" "${2}" 2>/dev/null | cut -d: -f2 || echo "0"
wd_state_get "$1" "$2"
}
# Set strike count for a container in state file
# Set strike count for a container in state file — wraps common.sh's wd_state_set()
set_strikes() {
local container="$1" count="$2" file="$3"
if grep -q "^${container}:" "$file" 2>/dev/null; then
sed -i "s/^${container}:.*/${container}:${count}/" "$file"
else
echo "${container}:${count}" >> "$file"
fi
wd_state_set "$container" "$count" "$file"
}
# Check if container is on the persistent skip list
@@ -763,10 +762,10 @@ CYCLE_START=$(date +%s)
STRIKES=$(get_strikes "$container" "$WATCHDOG_STATE_FILE")
STRIKES=$(( STRIKES + 1 ))
set_strikes "$container" "$STRIKES" "$WATCHDOG_STATE_FILE"
warn "$container — not running, exit ${LAST_EXIT} (strike $STRIKES/$SYS_WATCHDOG_STRIKE_LIMIT)"
warn "$container — not running, exit ${LAST_EXIT} (strike $STRIKES/${WATCHDOG_REQUIRED_STRIKE_LIMIT:-2})"
((T1_WARNINGS++))
if [[ "$STRIKES" -ge "$SYS_WATCHDOG_STRIKE_LIMIT" ]]; then
if [[ "$STRIKES" -ge "${WATCHDOG_REQUIRED_STRIKE_LIMIT:-2}" ]]; then
result=0
safe_restart "$container" "required container down (exit ${LAST_EXIT})" || result=$?
case $result in
Watchdogs/resource_watchdog.sh
+12 -12
View File
@@ -165,27 +165,25 @@ trap "_release_all_locks; _rw_trap_restart_stopped" EXIT
# ==============================================================================================
# rm_state_get/set use : separator for RM-internal state
# rm_state_get_eq/set_eq use = separator for docker_watchdog coordination flags
# Both wrap common.sh's wd_state_get()/wd_state_set() — file is implicit here (this
# script's own state file), unlike the generic helper which always takes it as an argument.
rm_state_get() {
grep -E "^${1}:" "$RW_STATE_FILE" 2>/dev/null | cut -d: -f2-
wd_state_get "$1" "$RW_STATE_FILE"
}
rm_state_set() {
local key="$1" val="$2"
grep -vE "^${key}:" "$RW_STATE_FILE" 2>/dev/null > "${RW_STATE_FILE}.tmp"
echo "${key}:${val}" >> "${RW_STATE_FILE}.tmp"
mv "${RW_STATE_FILE}.tmp" "$RW_STATE_FILE"
wd_state_set "$key" "$val" "$RW_STATE_FILE"
}
rm_state_get_eq() {
grep -E "^${1}=" "$RW_STATE_FILE" 2>/dev/null | cut -d= -f2-
wd_state_get "$1" "$RW_STATE_FILE" "="
}
rm_state_set_eq() {
local key="$1" val="$2"
grep -vE "^${key}=" "$RW_STATE_FILE" 2>/dev/null > "${RW_STATE_FILE}.tmp"
echo "${key}=${val}" >> "${RW_STATE_FILE}.tmp"
mv "${RW_STATE_FILE}.tmp" "$RW_STATE_FILE"
wd_state_set "$key" "$val" "$RW_STATE_FILE" "="
}
# ==============================================================================================
@@ -376,7 +374,8 @@ unpause_containers() {
}
# Stop a list of containers — returns comma-separated list of actually-stopped containers
stop_containers() {
# Named rw_ to avoid colliding with common.sh's stop_containers() (different signature/semantics)
rw_stop_containers() {
local actually_stopped=()
for container in "$@"; do
[[ -z "$container" ]] && continue
@@ -407,7 +406,8 @@ stop_containers() {
}
# Start a comma-separated list of containers (only those RM stopped)
start_containers() {
# Named rw_ to avoid colliding with common.sh's start_containers() (different signature/semantics)
rw_start_containers() {
local IFS=','
for container in $1; do
[[ -z "$container" ]] && continue
@@ -468,7 +468,7 @@ apply_level_3() {
if [[ ${#RW_STOP_CONTAINERS[@]} -gt 0 ]]; then
local newly_stopped
newly_stopped=$(stop_containers "${RW_STOP_CONTAINERS[@]}")
newly_stopped=$(rw_stop_containers "${RW_STOP_CONTAINERS[@]}")
if [[ -n "$newly_stopped" ]]; then
if [[ -n "$STOPPED_LIST" ]]; then
STOPPED_LIST="${STOPPED_LIST},${newly_stopped}"
@@ -490,7 +490,7 @@ apply_level_3() {
restore_level_3() {
echo "Restoring from level 3 — starting stopped containers"
if [[ -n "$STOPPED_LIST" ]]; then
start_containers "$STOPPED_LIST"
rw_start_containers "$STOPPED_LIST"
STOPPED_LIST=""
fi
rm_state_set_eq "mem_shutdown_active" "false"
Watchdogs/stability_watchdog.sh
+6 -8
View File
@@ -192,15 +192,15 @@ fi
# ── STATE HELPERS ─────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Wrap common.sh's wd_state_get()/wd_state_set() — file is implicit here (this script's
# own state file), unlike the generic helper which always takes it as an argument.
get_strikes() {
grep -E "^${1}:" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | cut -d':' -f2
wd_state_get "$1" "$SYS_WATCHDOG_STATE_FILE"
}
set_strikes() {
local key="$1" count="$2"
grep -vE "^${key}:" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null > "${SYS_WATCHDOG_STATE_FILE}.tmp"
echo "${key}:${count}" >> "${SYS_WATCHDOG_STATE_FILE}.tmp"
mv "${SYS_WATCHDOG_STATE_FILE}.tmp" "$SYS_WATCHDOG_STATE_FILE"
wd_state_set "$key" "$count" "$SYS_WATCHDOG_STATE_FILE"
}
increment_strikes() {
@@ -218,14 +218,12 @@ reset_strikes() {
}
get_state_val() {
grep -E "^${1}=" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | cut -d'=' -f2
wd_state_get "$1" "$SYS_WATCHDOG_STATE_FILE" "="
}
set_state_val() {
local key="$1" val="$2"
grep -vE "^${key}=" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null > "${SYS_WATCHDOG_STATE_FILE}.tmp"
echo "${key}=${val}" >> "${SYS_WATCHDOG_STATE_FILE}.tmp"
mv "${SYS_WATCHDOG_STATE_FILE}.tmp" "$SYS_WATCHDOG_STATE_FILE"
wd_state_set "$key" "$val" "$SYS_WATCHDOG_STATE_FILE" "="
}
purge_old_reboots() {
common.sh
+653 -107
View File
@@ -49,8 +49,7 @@
# check_arr_version() added — API version safety gate before arr operations
# check_api() added — pre-flight API reachability check
# ping_remote(), ping_internet() — non-fatal ping for fallback use
# check_local_array(), check_remote_array() — array health checks
# check_remote_docker() — Docker daemon health check
# check_remote_array() — array health check
# get_unraid_temp_thresholds() — reads thresholds from dynamix.cfg
# check_local_disk_temps() — pre-rsync temp check with exit codes 0/1/2
# stop/start local containers added alongside existing remote variants
@@ -79,15 +78,12 @@
# Called by lidarr/sonarr/radarr_cleanup.sh when files are deleted
#
#
# Three new safety functions added:
# Two new safety functions added:
# check_os_version_parity() — refuses remote ops on version mismatch
# reads OS version via platform_get_os_version() / platform_os_version_probe_cmd()
# major mismatch → abort | minor mismatch → configurable warn/abort
# check_remote_docker_daemon() — verifies remote Docker daemon before
# issuing any remote container commands — strike system → skip/retry/exit
# validate_unraid_cmd() — verifies unRAID-specific commands exist and
# produce expected output before use — notifies and exits calling script
# if command changed or disappeared after upgrade — other scripts unaffected
#
# ── VERSION ───────────────────────────────────────────────────────────────────────────────────
# Current: v3.5
@@ -130,6 +126,7 @@ ICON_SYNC="🔄" # transfer section header
ICON_RUN="🚀" # sync starting / rsync attempt
ICON_RETRY="🔁" # retry attempt
ICON_DONE="🏁" # transfer complete
ICON_GIT="🌱" # git pull/push section header
# Summary
ICON_SUMMARY="📋" # summary section header
@@ -269,6 +266,33 @@ format_duration() {
else echo "${rem}s"; fi
}
# Tiered human-readable size — picks B/MB/GB based on magnitude.
# Usage: format_bytes 5368709120 → 5.0GB
format_bytes() {
local bytes=${1:-0}
if (( bytes > 1073741824 )); then
awk "BEGIN {printf \"%.1fGB\", $bytes / 1073741824}"
elif (( bytes > 1048576 )); then
awk "BEGIN {printf \"%.1fMB\", $bytes / 1048576}"
else
echo "${bytes}B"
fi
}
# Fixed-precision bytes → GB (no tiering) — for values always compared against a GB threshold.
# Usage: bytes_to_gb 5368709120 [decimals=2] → 5.00
bytes_to_gb() {
local bytes=${1:-0} decimals=${2:-2}
awk "BEGIN {printf \"%.${decimals}f\", $bytes / 1073741824}"
}
# Fixed-precision KB → GB (no tiering) — df --output=used/avail reports in KB.
# Usage: kb_to_gb 5242880 [decimals=2] → 5.00
kb_to_gb() {
local kb=${1:-0} decimals=${2:-2}
awk "BEGIN {printf \"%.${decimals}f\", $kb / 1048576}"
}
# ==============================================================================================
# ── ARG PARSER ────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
@@ -355,6 +379,17 @@ validate_int() {
log "$name validated: $value"
}
# Membership check against a list of values (e.g. an ignore list).
# Usage: is_in_list "$needle" "${HAYSTACK_ARRAY[@]}"
is_in_list() {
local needle="$1"; shift
local item
for item in "$@"; do
[[ "$needle" == "$item" ]] && return 0
done
return 1
}
# ==============================================================================================
# ── HOST DETECTION ────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
@@ -506,6 +541,7 @@ detect_hosts() {
SYS_WATCHDOG_CHECK_MDSTAT SYS_WATCHDOG_CHECK_NETWORK
SYS_WATCHDOG_CHECK_SSHD SYS_WATCHDOG_CHECK_RUNAWAY
)
local _wd_var
for _wd_var in "${_wd_checks[@]}"; do
local _wd_src="${MY_ID}_${_wd_var}"
# Only alias if the host-specific var is set — preserves master.conf defaults
@@ -625,6 +661,86 @@ detect_hosts() {
log "$ICON_HOST Host: $LOCAL_SERVER_NAME → $REMOTE_SERVER_NAME"
}
# Derives owner/mirror roles from PARTNERSHIP_OWNER_HOST — used by every Partnership/*.sh
# script. Requires detect_hosts() to have already run (needs MY_ID, SSH_KEY).
# Sets: OWNER_ID MIRROR_ID OWNER MIRROR MIRROR_SSH_KEY OWNER_SSH_KEY AM_OWNER AM_MIRROR
# Does NOT set any *_STATE_FILE vars — callers that need those set them after calling this,
# since not every Partnership script needs the same set (transfer/onboard need fewer than
# manager/offboard).
partnership_resolve_roles() {
OWNER_ID="${PARTNERSHIP_OWNER_HOST:-HOST1}"
MIRROR_ID=$( [[ "$OWNER_ID" == "HOST1" ]] && echo "HOST2" || echo "HOST1" )
OWNER="${!OWNER_ID}"
MIRROR="${!MIRROR_ID}"
# SSH_KEY (set by detect_hosts) is this server's own private key. The remote accepts it
# because this server's PUBLIC key was installed there via ssh_setup.sh. With sparse
# checkout, each server only has its own host{N}.conf — the other server's key path is
# never available here. Use SSH_KEY for all outbound SSH regardless of mode.
MIRROR_SSH_KEY="$SSH_KEY"
OWNER_SSH_KEY="$SSH_KEY"
AM_OWNER=false
AM_MIRROR=false
[[ "$MY_ID" == "$OWNER_ID" ]] && AM_OWNER=true
[[ "$MY_ID" == "$MIRROR_ID" ]] && AM_MIRROR=true
}
# Read a scalar/array var from a partner's own config via SSH — the partner sources its
# own load_config.sh + detect_hosts() remotely so the value reflects ITS host*.conf, not
# ours. Requires $SCRIPT_DIR to be set by the caller (one level above the repo root, same
# convention every Partnership/*.sh script already uses for its own SCRIPT_DIR).
# Usage: read_remote_conf_var "$mirror_ip" "VAR_NAME"
# read_remote_conf_array "$mirror_ip" "ARRAY_NAME" # one element per line
read_remote_conf_var() {
local mirror_ip="$1" var_name="$2"
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \
"source '$SCRIPT_DIR/../load_config.sh' 2>/dev/null
detect_hosts 2>/dev/null
printf '%s' \"\${${var_name}:-}\"" 2>/dev/null
}
read_remote_conf_array() {
local mirror_ip="$1" var_name="$2"
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \
"source '$SCRIPT_DIR/../load_config.sh' 2>/dev/null
detect_hosts 2>/dev/null
printf '%s\n' \"\${${var_name}[@]:-}\"" 2>/dev/null
}
# Probes a remote host's SCRIPTS_DIR (in case it differs from ours — e.g. one host runs
# internal storage mode, the other flash), falling back to our own $SCRIPTS_DIR if the
# probe fails or the remote isn't reachable. strict_host_key defaults to "yes" (matches
# existing behavior everywhere except partnership_transfer.sh, which passes "no" since
# it may be contacting a mirror for the first time during an ownership transfer).
# Usage: resolve_remote_scripts_dir "$ip" ["$ssh_key"] ["no"|"yes"]
resolve_remote_scripts_dir() {
local ip="$1" ssh_key="${2:-$SSH_KEY}" strict_host_key="${3:-yes}"
local probe_cmd result
local -a opts=(-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes)
[[ "$strict_host_key" == "no" ]] && opts+=(-o StrictHostKeyChecking=no)
probe_cmd=$(platform_scripts_dir_probe_cmd)
result=$(timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" "${opts[@]}" root@"$ip" "$probe_cmd" \
2>/dev/null | tr -d '[:space:]')
echo "${result:-$SCRIPTS_DIR}"
}
# ==============================================================================================
# ── MULTI-NODE DISCOVERY ──────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Scans HOST1..HOST8 for every configured node other than this one. Requires detect_hosts()
# to have already run (needs MY_ID). Sets the global REMOTE_NODES array — empty if none found.
discover_remote_nodes() {
REMOTE_NODES=()
local _hv
for _hv in HOST1 HOST2 HOST3 HOST4 HOST5 HOST6 HOST7 HOST8; do
[[ "$_hv" == "$MY_ID" ]] && continue
[[ -z "${!_hv:-}" ]] && continue
REMOTE_NODES+=("$_hv")
done
}
# ==============================================================================================
# ── REMOTE IP RESOLUTION ──────────────────────────────────────────────────────────────────────
# ==============================================================================================
@@ -669,6 +785,34 @@ resolve_tailscale_ip() {
echo "$ip"
}
# Strips the "unraid-" prefix (case-insensitive) and title-cases what remains.
# Usage: derive_short_name "unRAID-Gmer4Lfe" → "Gmer4lfe"
derive_short_name() {
local hostname="$1"
local short="${hostname,,}"
[[ "$short" == unraid-* ]] && short="${short:7}"
echo "${short^}"
}
# True if the given host*.conf basename belongs to this host (case-insensitive).
# Usage: is_own_conf_file "$(basename "$conf")"
is_own_conf_file() {
[[ "${1,,}" == "${MY_ID,,}.conf" ]]
}
# Sets key=value in a flat state file (setup.db style) — updates in place if the
# key exists, appends if not. No indentation handling — for master.conf use
# partnership_manager.sh's update_master_conf() instead.
# Usage: set_state_var "$state_file" "$key" "$value"
set_state_var() {
local file="$1" key="$2" value="$3"
if grep -q "^${key}=" "$file" 2>/dev/null; then
sed -i "s|^${key}=.*|${key}=${value}|" "$file"
else
echo "${key}=${value}" >> "$file"
fi
}
# ==============================================================================================
# ── CONNECTIVITY CHECKS ───────────────────────────────────────────────────────────────────────
# ==============================================================================================
@@ -755,26 +899,6 @@ is_vm_manager_enabled() {
# ── LOCAL HEALTH CHECKS ───────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Verifies local storage is mounted and has shares.
# Non-fatal — returns status for caller to decide.
# Used by fallback before starting remote containers locally.
check_local_array() {
log "Checking local array..."
if ! platform_storage_healthy; then
error "$ICON_DISK Local array is not started — storage not mounted"
return 1
fi
local _storage_path file_count
_storage_path=$(platform_storage_path)
file_count=$(ls "$_storage_path" 2>/dev/null | wc -l)
if [[ "$file_count" -eq 0 ]]; then
error "$ICON_DISK Local array appears empty — shares may not be available"
return 1
fi
log "Local array is healthy"
return 0
}
# ==============================================================================================
# ── REMOTE HEALTH CHECKS ─────────────────────────────────────────────────────────────────────
# ==============================================================================================
@@ -796,19 +920,6 @@ check_remote_array() {
return 0
}
# Verifies remote Docker daemon is responding.
# Non-fatal — returns status. A hung daemon means container commands silently fail.
check_remote_docker() {
log "Checking remote Docker daemon on $REMOTE_SERVER_NAME..."
if ! ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \
"timeout 10 docker ps" >/dev/null 2>&1; then
error "$ICON_CONTAINERS Remote Docker daemon not responding on $REMOTE_SERVER_NAME"
return 1
fi
log "Remote Docker daemon is healthy"
return 0
}
# Aborts if remote rootfs (/) usage is at or above ROOTFS_WARN threshold.
# Fatal — exits the calling script.
# When remote array is down rsync writes land on rootfs and fill it rapidly.
@@ -861,9 +972,14 @@ check_remote_share() {
# Returns 0 (true) if device is non-rotational (SSD/NVMe), 1 (false) if HDD.
# Checks /sys/block/<device>/queue/rotational — 0=SSD, 1=HDD.
is_ssd() {
local base
base=$(basename "$1")
[[ "$(cat "/sys/block/$base/queue/rotational" 2>/dev/null)" == "0" ]]
local drive="$1"
local base rotational
base=$(basename "$drive")
rotational="/sys/block/$base/queue/rotational"
[[ -f "$rotational" ]] && [[ "$(cat "$rotational" 2>/dev/null)" == "0" ]] && return 0
# NVMe is always SSD — some NVMe controllers don't expose rotational correctly
[[ "$drive" == *nvme* ]] && return 0
return 1
}
# Sets globals: UNRAID_DISK_HOT UNRAID_DISK_MAX UNRAID_SSD_HOT UNRAID_SSD_MAX
@@ -878,6 +994,37 @@ get_unraid_temp_thresholds() {
}
# Checks local disk temperatures before rsync.
# Fetches a domain's TLS cert expiry via openssl s_client + x509 and computes days
# remaining. Pure data — no logging, no thresholds — callers classify warn/crit
# themselves against their own CERT_WARN_DAYS/CERT_CRIT_DAYS and log however they like.
#
# Sets _CERT_DAYS, _CERT_EXPIRY (YYYY-MM-DD) on success. Sets _CERT_EXPIRY_RAW to the
# unparsed openssl date string only when parsing failed (empty when unreachable) so a
# caller that wants a more specific error message can distinguish the two failure modes.
#
# Usage: check_cert_expiry "$domain" [port=443] [timeout=$CERT_TIMEOUT or 10]
# Returns: 0 = fetched and parsed | 1 = unreachable or unparseable
check_cert_expiry() {
local domain="$1" port="${2:-443}" cert_timeout="${3:-${CERT_TIMEOUT:-10}}"
_CERT_DAYS=""
_CERT_EXPIRY=""
_CERT_EXPIRY_RAW=""
local expiry_str
expiry_str=$(echo | timeout "$cert_timeout" openssl s_client \
-connect "${domain}:${port}" -servername "$domain" \
2>/dev/null | openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2)
[[ -z "$expiry_str" ]] && return 1
local expiry_epoch
expiry_epoch=$(date -d "$expiry_str" +%s 2>/dev/null)
if [[ -z "$expiry_epoch" ]]; then
_CERT_EXPIRY_RAW="$expiry_str"
return 1
fi
_CERT_DAYS=$(( (expiry_epoch - $(date +%s)) / 86400 ))
_CERT_EXPIRY=$(date -d "$expiry_str" '+%Y-%m-%d' 2>/dev/null)
return 0
}
# Reads temps and rotational flag from /var/local/emhttp/disks.ini.
# Uses unRAID's own thresholds from dynamix.cfg via get_unraid_temp_thresholds().
#
@@ -1305,6 +1452,62 @@ start_local_containers() {
done
}
# Builds a dependency-safe restart order — dependency containers (anything named as a
# value in WATCHDOG_DEPENDENCIES) restart first, then everything else in original order.
# Sets ORDERED_RESTART. Usage: build_restart_order CONTAINERS_ARRAY_NAME (bare name, e.g.
# build_restart_order DAILY_RESTART_CONTAINERS — not "${DAILY_RESTART_CONTAINERS[@]}").
build_restart_order() {
local -n _bro_containers="$1"
ORDERED_RESTART=()
local remaining=("${_bro_containers[@]}")
local placed=()
# First pass — add dependency containers that appear in our list
for container in "${remaining[@]}"; do
[[ -z "$container" ]] && continue
local is_dependency=false
for dependent in "${!WATCHDOG_DEPENDENCIES[@]}"; do
if [[ "${WATCHDOG_DEPENDENCIES[$dependent]}" == *"$container"* ]]; then
is_dependency=true
break
fi
done
if [[ "$is_dependency" == true ]]; then
local already=false
for p in "${placed[@]}"; do [[ "$p" == "$container" ]] && already=true && break; done
if [[ "$already" == false ]]; then
ORDERED_RESTART+=("$container")
placed+=("$container")
fi
fi
done
# Second pass — add remaining containers (dependents and independents)
for container in "${remaining[@]}"; do
[[ -z "$container" ]] && continue
local already=false
for p in "${placed[@]}"; do [[ "$p" == "$container" ]] && already=true && break; done
if [[ "$already" == false ]]; then
ORDERED_RESTART+=("$container")
placed+=("$container")
fi
done
log "Restart order: ${ORDERED_RESTART[*]}"
}
# Waits CONTAINER_DELAY if this container depends on the last restarted one.
# Usage: check_dependency_delay "$container" "$last_restarted"
check_dependency_delay() {
local container="$1" last="$2"
[[ -z "$last" ]] && return
local deps="${WATCHDOG_DEPENDENCIES[$container]:-}"
if [[ -n "$deps" ]] && [[ "$deps" == *"$last"* ]]; then
log "Waiting ${CONTAINER_DELAY}s — $container depends on $last..."
sleep "$CONTAINER_DELAY"
fi
}
# ==============================================================================================
# ── RSYNC OPTIONS ─────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
@@ -1380,12 +1583,6 @@ _lock_file() {
echo "$LOCK_DIR/${1:-$(_lock_name)}.lock"
}
# Internal — release lock on exit (registered via EXIT trap)
_release_on_exit() {
local lockfile="$1"
[[ -f "$lockfile" ]] && rm -f "$lockfile"
}
# Acquire exclusive lock for this script.
# Usage: acquire_lock [strict|wait|continuous]
acquire_lock() {
@@ -1524,17 +1721,30 @@ acquire_rsync_lock() {
log "$ICON_LOCK rsync lock acquired: profile '$profile' (PID $$, active: $(( current_count + 1 ))/$RSYNC_MAX_CONCURRENT)"
}
# Internal — release rsync lock and decrement global counter on exit
_release_rsync_on_exit() {
local profile_lock="$1"
[[ -f "$profile_lock" ]] && rm -f "$profile_lock"
if [[ -f "$RSYNC_COUNT_FILE" ]]; then
local count
count=$(cat "$RSYNC_COUNT_FILE" 2>/dev/null || echo 1)
count=$(( count - 1 ))
[[ "$count" -lt 0 ]] && count=0
echo "$count" > "$RSYNC_COUNT_FILE"
fi
# ==============================================================================================
# ── FLAT STATE-FILE KEY/VALUE HELPERS ─────────────────────────────────────────────────────────
# ==============================================================================================
# Generic get/set for flat "key<sep>value" state files (one entry per line) — the pattern
# every watchdog's strike-tracking and state-file logic was independently reimplementing.
# grep -v + tempfile-swap on write, not sed -i in place — avoids sed treating a key containing
# regex metacharacters (container names, etc.) as part of the substitution pattern.
#
# Usage: wd_state_get "$key" "$file" [sep=:]
# wd_state_set "$key" "$value" "$file" [sep=:]
#
# Callers with a fixed state file and/or separator (e.g. stability_watchdog.sh's
# get_strikes/get_state_val, resource_watchdog.sh's rm_state_get/rm_state_get_eq) should
# keep their own thin same-named wrapper around these rather than changing call sites.
wd_state_get() {
local key="$1" file="$2" sep="${3:-:}"
grep -E "^${key}${sep}" "$file" 2>/dev/null | cut -d"$sep" -f2-
}
wd_state_set() {
local key="$1" value="$2" file="$3" sep="${4:-:}"
grep -vE "^${key}${sep}" "$file" 2>/dev/null > "${file}.tmp"
echo "${key}${sep}${value}" >> "${file}.tmp"
mv "${file}.tmp" "$file"
}
# ==============================================================================================
@@ -1632,6 +1842,348 @@ translate_path() {
fi
}
# ==============================================================================================
# ── CONTAINER STOP/RESTART FOR MAINTENANCE ────────────────────────────────────────────────────
# ==============================================================================================
# The "stop a container for a maintenance window, guarantee it comes back" shape shared by
# container_data_export.sh and emby_database_repair.sh. Callers still register their own EXIT
# trap (they may need extra cleanup, e.g. removing a partial archive) — the trap should call
# container_force_restart_if_needed() for the actual "still down? bring it back" part.
# Usage: container_stop_for_maintenance "$container" was_running_var_name \
# ["pre-stop reason message"] [log|warn] [post_stop_sleep=0]
# The reason message (if given) prints via the chosen level (default: log) right before the
# stop attempt, only when the container was actually running — callers use this for their
# own context ("stopping for clean export" vs "active sessions will be interrupted").
# Sets the named var true/false. Returns 1 (caller should exit) if the container doesn't
# exist or fails to stop; 0 otherwise (including the "wasn't running" case — caller logs
# its own context-specific message for that, since wording/visibility differs per caller).
container_stop_for_maintenance() {
local container="$1"
local -n _csm_was_running="$2"
local pre_stop_msg="${3:-}" pre_stop_level="${4:-log}" post_stop_sleep="${5:-0}"
_csm_was_running=false
local status
status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
case "$status" in
true)
_csm_was_running=true
if [[ -n "$pre_stop_msg" ]]; then
if [[ "$pre_stop_level" == "warn" ]]; then warn "$pre_stop_msg"; else log "$pre_stop_msg"; fi
fi
if [[ "$DRY_RUN" == false ]]; then
if timeout "$DOCKER_TIMEOUT" docker stop "$container" >/dev/null 2>&1; then
echo "$container stopped ✅"
[[ "$post_stop_sleep" -gt 0 ]] && sleep "$post_stop_sleep"
else
error "Failed to stop $container — aborting"
return 1
fi
else
warn "DRY RUN — would stop $container"
fi
;;
false) : ;; # not running — caller logs its own context-specific message
"")
error "$container not found — check container name"
return 1
;;
*)
warn "$container status: $status — proceeding with caution"
;;
esac
return 0
}
# Force-restart from inside a script's own EXIT trap if the container is still down after
# a crash mid-maintenance. No-op if it wasn't running before, in dry-run, or already up.
# Usage: container_force_restart_if_needed "$container" "$was_running"
container_force_restart_if_needed() {
local container="$1" was_running="$2"
[[ "$was_running" == true && "$DRY_RUN" == false ]] || return 0
local status
status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
if [[ "$status" != "true" ]]; then
warn "Restarting $container (cleanup)..."
timeout "$DOCKER_TIMEOUT" docker start "$container" >/dev/null 2>&1 || \
error "Failed to restart $container — start it manually"
fi
}
# Normal-path restart after maintenance completes — verifies it stayed up, notifies on failure.
# Sets RESTART_OK true/false.
# Usage: container_restart_after_maintenance "$container" "$was_running" [settle_sleep=3] [notify_label]
container_restart_after_maintenance() {
local container="$1" was_running="$2" settle_sleep="${3:-3}" notify_label="${4:-Container Maintenance}"
RESTART_OK=false
if [[ "$was_running" != true ]]; then
echo "$container was not running — leaving stopped (state respected) ✅"
RESTART_OK=true
return 0
fi
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would restart $container"
RESTART_OK=true
return 0
fi
log "Restarting $container..."
if timeout "$DOCKER_TIMEOUT" docker start "$container" >/dev/null 2>&1; then
sleep "$settle_sleep"
local post_status
post_status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
if [[ "$post_status" == "true" ]]; then
echo "$container restarted and running ✅"
RESTART_OK=true
else
error "$container started but crashed — check container logs"
notify "$container failed to stay running after maintenance on $(hostname)" \
"$notify_label" "warning"
fi
else
error "Failed to restart $container — start it manually"
notify "$container failed to restart after maintenance on $(hostname)" \
"$notify_label" "warning"
fi
}
# ==============================================================================================
# ── ARR CLEANUP SAFETY GATES ──────────────────────────────────────────────────────────────────
# ==============================================================================================
# Shared by lidarr/radarr/sonarr_cleanup.sh — these scripts DELETE files, so every function
# here is a direct byte-for-byte port of what was independently duplicated three times, not
# a rewrite. Callers should behave identically to before this consolidation.
# Safety Layer 1 — container running, not starting/unhealthy. Exits 1 (with notify) on failure.
# Usage: check_container_health "$LIDARR_CONTAINER" "$ARR_DOCKER_TIMEOUT" "Lidarr Cleanup"
check_container_health() {
local container="$1" docker_timeout="$2" notify_label="$3"
local running
running=$(timeout "$docker_timeout" docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
if [[ "$running" != "true" ]]; then
error "$container is not running — aborting"
notify "$notify_label aborted on $(hostname) — container not running" "$notify_label" "warning"
exit 1
fi
local health
health=$(timeout "$docker_timeout" docker inspect -f '{{.State.Health.Status}}' "$container" 2>/dev/null)
case "$health" in
healthy) info "$container is healthy" ;;
"") info "$container has no health check — proceeding" ;;
starting)
error "$container is still starting — aborting"
notify "$notify_label aborted on $(hostname) — container still starting" "$notify_label" "warning"
exit 1 ;;
unhealthy)
error "$container is unhealthy — aborting"
notify "$notify_label aborted on $(hostname) — container unhealthy" "$notify_label" "warning"
exit 1 ;;
*) warn "$container health: $health — proceeding with caution" ;;
esac
info "Safety layer 1 passed — container healthy"
}
# Safety Layer 6 — abort if tracked count dropped below min_pct of the last known count
# (protects against a misconfigured path / partial API response silently wiping the library).
# Writes the new baseline only if the check passes. Exits 1 (with notify) on failure.
# Usage: check_tracked_count_floor "$TRACKED_COUNT" "$LIDARR_TRACKED_COUNT_FILE" "$LIDARR_MIN_TRACKED_PCT" "Lidarr Cleanup"
check_tracked_count_floor() {
local tracked_count="$1" baseline_file="$2" min_pct="$3" notify_label="$4"
if [[ -f "$baseline_file" ]]; then
local last_count pct
last_count=$(cat "$baseline_file" 2>/dev/null || echo 0)
if [[ "$last_count" -gt 0 ]]; then
pct=$(awk "BEGIN {printf \"%d\", ($tracked_count / $last_count) * 100}")
if [[ "$pct" -lt "$min_pct" ]]; then
error "Tracked count dropped to ${pct}% of last run ($tracked_count vs $last_count)"
error "Suggests API issue — aborting to prevent mass deletion"
error "If expected (large library removal) delete: $baseline_file"
notify "$notify_label aborted on $(hostname) — tracked count dropped to ${pct}%" \
"$notify_label" "warning"
exit 1
fi
info "Tracked count: ${pct}% of last run ($tracked_count vs $last_count) ✅"
fi
else
info "No previous count on record — first run, saving baseline"
fi
echo "$tracked_count" > "$baseline_file"
}
# Filters --i-know-what-im-doing / --skip-age-check out of "$@" before parse_args sees them
# (both are cleanup-script-specific, not part of the shared arg parser).
# Sets I_KNOW, SKIP_AGE_CHECK, FILTERED_ARGS — call parse_args "${FILTERED_ARGS[@]}" after.
parse_destructive_flags() {
I_KNOW=false
SKIP_AGE_CHECK=false
FILTERED_ARGS=()
local arg
for arg in "$@"; do
case "$arg" in
--i-know-what-im-doing) I_KNOW=true ;;
--skip-age-check) SKIP_AGE_CHECK=true ;;
*) FILTERED_ARGS+=("$arg") ;;
esac
done
}
# Prints the nuclear-mode banner and sleeps 10s (Ctrl+C window) when both destructive flags
# are set and this isn't a dry run. Call after parse_args. No-op otherwise.
nuclear_mode_warning() {
if [[ "$I_KNOW" == true ]] && [[ "$SKIP_AGE_CHECK" == true ]] && [[ "$DRY_RUN" != true ]]; then
echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "⚠️ WARNING — NUCLEAR MODE ACTIVE"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo " Flags: --i-know-what-im-doing --skip-age-check"
echo " Age check: BYPASSED — deletes on first pass"
echo " Size threshold: BYPASSED — no GB limit"
echo " Data recovery: NOT POSSIBLE after deletion"
echo ""
echo " Review --dry-run output before proceeding."
echo " You have 10 seconds to cancel (Ctrl+C)..."
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
sleep 10
echo " Proceeding..."
echo ""
fi
}
# Safety Layer 7 — abort if total deletion size exceeds max_gb and --i-know-what-im-doing
# wasn't passed. Sets TOTAL_HUMAN. Exits 1 (with notify) on failure; warns and continues
# if I_KNOW is true (set by parse_destructive_flags).
# Usage: check_delete_size_threshold "$TOTAL_DELETE_BYTES" "$LIDARR_MAX_DELETE_GB" "Lidarr Cleanup"
check_delete_size_threshold() {
local total_bytes="$1" max_gb="$2" notify_label="$3"
local max_bytes
max_bytes=$(awk "BEGIN {printf \"%d\", $max_gb * 1073741824}")
if [[ "$total_bytes" -gt "$max_bytes" ]]; then
TOTAL_HUMAN=$(awk "BEGIN {printf \"%.1fGB\", $total_bytes / 1073741824}")
if [[ "$I_KNOW" != true ]]; then
echo ""
error "Deletion would exceed ${max_gb}GB — $TOTAL_HUMAN would be deleted"
error "Review ORPHAN lines above carefully before proceeding"
error "Rerun with: --i-know-what-im-doing"
error "To also bypass age check: add --skip-age-check"
notify "$notify_label halted on $(hostname) — ${TOTAL_HUMAN} requires --i-know-what-im-doing" \
"$notify_label" "warning"
exit 1
else
warn "OVERRIDE — deletion is $TOTAL_HUMAN — proceeding with --i-know-what-im-doing"
fi
fi
}
# True if filepath's extension (case-insensitive) matches one of the given extensions.
# Usage: has_extension "$filepath" "${LIDARR_EXTENSIONS[@]}"
has_extension() {
local filepath="$1"; shift
local ext="${filepath##*.}"
ext="${ext,,}"
local valid_ext
for valid_ext in "$@"; do
[[ "$ext" == "$valid_ext" ]] && return 0
done
return 1
}
# True if filepath's basename matches one of the given glob patterns.
# Usage: matches_pattern_list "$filepath" "${LIDARR_PROTECTED_PATTERNS[@]}"
matches_pattern_list() {
local filepath="$1"; shift
local filename
filename=$(basename "$filepath")
local pattern
for pattern in "$@"; do
# shellcheck disable=SC2254
case "$filename" in
$pattern) return 0 ;;
esac
done
return 1
}
# Builds the global ARR_PATH_MAP associative array (container path → host path) from
# ${MY_ID}_<ARR_TYPE>_PATH_MAP, for translate_path() to use. Usage: build_arr_path_map "LIDARR"
build_arr_path_map() {
local arr_type="$1"
declare -gA ARR_PATH_MAP=()
local _bapm_var="${MY_ID}_${arr_type}_PATH_MAP"
eval "for key in \"\${!${_bapm_var}[@]}\"; do
ARR_PATH_MAP[\"\$key\"]=\"\${${_bapm_var}[\$key]}\"
done"
}
# Generic arr API GET wrapper with HTTP status check.
# Usage: arr_api "$RADARR_URL" "$RADARR_API_KEY" "v3" "movie" "Radarr"
arr_api() {
local base_url="$1" api_key="$2" api_version="$3" endpoint="$4" label="${5:-Arr}"
local response http_code body
response=$(curl -sf \
--max-time 30 \
-H "X-Api-Key: $api_key" \
-w "\n%{http_code}" \
"${base_url}/api/${api_version}/${endpoint}" 2>/dev/null)
http_code=$(echo "$response" | tail -1)
body=$(echo "$response" | head -n -1)
if [[ "$http_code" != "200" ]]; then
error "$label API HTTP $http_code for: $endpoint"
return 1
fi
echo "$body"
}
# Triggers an arr "command" endpoint with the given JSON payload, then polls every 10s until
# completed/failed/timeout, logging progress every 60s. Best-effort pre-flight — never treats
# an unreachable/timed-out scan as fatal, matching original per-script behavior.
# Usage: trigger_and_await_command "$LIDARR_URL" "$LIDARR_API_KEY" "v1" \
# '{"name": "DownloadedAlbumsScan"}' "${LIDARR_IMPORT_SCAN_TIMEOUT:-600}"
trigger_and_await_command() {
local base_url="$1" api_key="$2" api_version="$3" payload="$4" poll_timeout="${5:-600}"
local scan_response scan_cmd_id
scan_response=$(curl -sf --max-time 30 -X POST \
-H "X-Api-Key: $api_key" \
-H "Content-Type: application/json" \
-d "$payload" \
"${base_url}/api/${api_version}/command" 2>/dev/null)
scan_cmd_id=$(echo "$scan_response" | jq -r '.id // empty' 2>/dev/null)
if [[ -z "$scan_cmd_id" ]]; then
warn "Could not trigger import scan — proceeding without pre-flight"
return 0
fi
info "Import scan queued (command ID: $scan_cmd_id) — waiting for completion..."
local polled=0 scan_status
while [[ "$polled" -lt "$poll_timeout" ]]; do
scan_status=$(curl -sf --max-time 10 \
-H "X-Api-Key: $api_key" \
"${base_url}/api/${api_version}/command/${scan_cmd_id}" 2>/dev/null | \
jq -r '.status // empty' 2>/dev/null)
case "$scan_status" in
completed) info "Import scan complete ✅"; return 0 ;;
failed) warn "Import scan reported failed — proceeding anyway"; return 0 ;;
esac
sleep 10
(( polled += 10 ))
[[ $(( polled % 60 )) -eq 0 ]] && log " Still scanning... (${polled}s elapsed)"
done
warn "Import scan timed out after ${poll_timeout}s — proceeding anyway"
return 0
}
# ==============================================================================================
# ── ARR VERSION CHECK ─────────────────────────────────────────────────────────────────────────
# ==============================================================================================
@@ -1692,6 +2244,48 @@ check_arr_version() {
fi
}
# ==============================================================================================
# ── TMDB DISCOVERY SCORING ────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Shared by playback_aware_radarr_discovery.sh and playback_aware_sonarr_discovery.sh — stage 2
# candidate scoring against TMDB rating/vote/seed-breadth data. Byte-for-byte identical between
# the two before this consolidation.
# Formats a TMDB vote_average×10 integer back to one decimal place (e.g. 78 → 7.8).
_fmt_rating() { local v="${1:-0}"; echo "${v::-1}.${v: -1}" 2>/dev/null || echo "$v"; }
# Stage 2: TMDB vote_average × 10 as integer (0-40)
_rating_score_s2() {
local v="$1"
if (( v >= 80 )); then echo 40
elif (( v >= 75 )); then echo 32
elif (( v >= 70 )); then echo 25
elif (( v >= 65 )); then echo 18
elif (( v >= 60 )); then echo 12
else echo 5
fi
}
# Stage 2: vote count (0-20)
_votes_score() {
local c="$1"
if (( c >= 10000 )); then echo 20
elif (( c >= 5000 )); then echo 15
elif (( c >= 1000 )); then echo 10
elif (( c >= 200 )); then echo 5
else echo 2
fi
}
# Stage 2: seed breadth (0-40)
_breadth_score() {
local seeds="$1"
if (( seeds >= 3 )); then echo 40
elif (( seeds == 2 )); then echo 25
else echo 10
fi
}
# ==============================================================================================
# ── EMBY LIBRARY SCAN ─────────────────────────────────────────────────────────────────────────
# ==============================================================================================
@@ -1911,54 +2505,6 @@ check_remote_docker_daemon() {
return 1
}
# ==============================================================================================
# ── UNRAID COMMAND VALIDATION ─────────────────────────────────────────────────────────────────
# ==============================================================================================
# Verifies a unRAID-specific command exists and produces expected output before use.
# Protects against commands moving, changing format, or disappearing after upgrades.
#
# If validation fails — notifies and returns 1. Caller exits its own section only.
# Other scripts that pass validation are unaffected.
#
# Usage:
# validate_unraid_cmd # "/usr/local/emhttp/plugins/dynamix/scripts/notify" # "--help" # "Usage" # "unRAID notify script"
#
# Arguments:
# $1 = full path to command
# $2 = test argument to pass (use "" for no argument)
# $3 = expected string in output (use "" to skip output check)
# $4 = human readable name for error messages
validate_unraid_cmd() {
local cmd_path="$1"
local test_arg="$2"
local expected_output="$3"
local cmd_name="${4:-$1}"
# Check command exists and is executable
if [[ ! -x "$cmd_path" ]]; then
error "unRAID command not found or not executable: $cmd_path"
error "$cmd_name may have moved or been removed — check after recent unRAID upgrade"
notify "$cmd_name not found on $(hostname) at $cmd_path — check after unRAID upgrade" "Command Validation" "warning"
return 1
fi
# Check output matches expected pattern if provided
if [[ -n "$expected_output" ]]; then
local actual_output
actual_output=$(timeout 10 "$cmd_path" $test_arg 2>&1 || true)
if ! echo "$actual_output" | grep -q "$expected_output"; then
error "$cmd_name output format changed — expected '$expected_output' not found"
error "Command may have changed after unRAID upgrade — review $cmd_path"
notify "$cmd_name output format changed on $(hostname) — may need script update after unRAID upgrade" "Command Validation" "warning"
return 1
fi
fi
log "$cmd_name validated: $cmd_path ✅"
return 0
}
# ==============================================================================================
# ── STATUS DISPLAY ────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
load_config.sh
+2 -1
View File
@@ -135,7 +135,8 @@
CONF_DIR="${SCRIPTS_DIR}/Configurations"
LOG_DIR="/var/log/varaverk"
VV_CACHE_DIR="/tmp/vv_cache"
export CONF_DIR LOG_DIR VV_CACHE_DIR
CONF_RAM_CACHE_DIR="/tmp/.cache/vv/d" # tmpfs — cleared every reboot, repopulated by conf_sync.sh
export CONF_DIR LOG_DIR VV_CACHE_DIR CONF_RAM_CACHE_DIR
# ━━━ Cleanup ━━━
unset _conf _host_confs_loaded _adapter LOAD_CONFIG_DIR