FailedProxy/Varaverk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix dead/incorrect vars and consolidate duplicated logic into common.sh

Browse Source 
Codebase-wide audit pass: fixed real bugs (SSH hangs missing BatchMode,
local-outside-function no-ops, variable name collisions, a truncated
ratio calc, wrong state-dir path, DARK vs NO_INTERNET drift, and more),
then pulled logic that was duplicated across multiple scripts — arr
cleanup safety gates, docker restart ordering, container maintenance
stop/restart, watchdog state-file helpers, partnership role resolution,
cert expiry checks, remote node discovery, and TMDB discovery scoring —
into common.sh so each now has a single implementation.
This commit is contained in:
Gmer4Lfe
2026-07-03 23:52:33 -04:00
parent ef3980cf07
commit 6623d1e776
46 changed files with 921 additions and 1398 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Arrs_Stack/arr_sync.sh
+4 -10
View File
@@ -206,13 +206,7 @@ declare -A _ID_TYPE=([lidarr]="string" [sonarr]="int" [radarr]="int")
ARR_TYPES=(lidarr sonarr radarr)
# ── Remote node discovery ──────────────────────────────────────────────────────────────────────
REMOTE_NODES=()
for _hv in HOST1 HOST2 HOST3 HOST4 HOST5 HOST6 HOST7 HOST8; do
[[ "$_hv" == "$MY_ID" ]] && continue
[[ -z "${!_hv:-}" ]] && continue
REMOTE_NODES+=("$_hv")
done
unset _hv
discover_remote_nodes
if [[ "${#REMOTE_NODES[@]}" -eq 0 ]]; then
warn "No remote nodes defined in master.conf — nothing to sync"
@@ -478,7 +472,7 @@ if [[ "$SHOW_STATUS" == true ]]; then
echo ""
echo " Arr Port URL"
for _arr in "${ARR_TYPES[@]}"; do
local _url="" _key=""
_url=""
case "$_arr" in
lidarr) _url="${LIDARR_URL:-not configured}" ;;
sonarr) _url="${SONARR_URL:-not configured}" ;;
@@ -992,8 +986,8 @@ _sync_arr() {
fi
fi
local _n_local=${#to_add_local[@]:-}; _n_local=${_n_local:-0}
local _n_remote=${#to_add_remote[@]:-}; _n_remote=${_n_remote:-0}
local _n_local=${#to_add_local[@]}
local _n_remote=${#to_add_remote[@]}
log " $node_name: +${_n_local} local | +${_n_remote} remote | $total_skipped blocklisted"
unset remote_ids to_add_local to_add_remote _n_local _n_remote
Arrs_Stack/lidarr_cleanup.sh
+17 -187
View File
@@ -60,7 +60,7 @@
#
# acquire_lock "wait" — large scans take time, wait for previous run to finish
# jq + curl validation — exits if either tool missing
# DOCKER_TIMEOUT — container checks protected against daemon hangs
# ARR_DOCKER_TIMEOUT — container checks protected against daemon hangs (script-local, not common.sh's DOCKER_TIMEOUT)
# Duplicate detection — temp file of tracked paths, grep before delete
# platform_require_cmd — notify script validated before use
# Silent by default — orphans/junk warn(), clean library logs silently
@@ -120,37 +120,12 @@ source "$SCRIPT_DIR/../load_config.sh"
# ── Special flag pre-processing ───────────────────────────────────────────────────────────────
# Filter --i-know-what-im-doing and --skip-age-check before parse_args
# to avoid unknown flag errors — both are handled separately below.
I_KNOW=false
SKIP_AGE_CHECK=false
FILTERED_ARGS=()
for arg in "$@"; do
case "$arg" in
--i-know-what-im-doing) I_KNOW=true ;;
--skip-age-check) SKIP_AGE_CHECK=true ;;
*) FILTERED_ARGS+=("$arg") ;;
esac
done
parse_destructive_flags "$@"
parse_args "${FILTERED_ARGS[@]}"
# ── Nuclear mode warning ──────────────────────────────────────────────────────────────────────
if [[ "$I_KNOW" == true ]] && [[ "$SKIP_AGE_CHECK" == true ]] && [[ "$DRY_RUN" != true ]]; then
echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "⚠️ WARNING — NUCLEAR MODE ACTIVE"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo " Flags: --i-know-what-im-doing --skip-age-check"
echo " Age check: BYPASSED — deletes on first pass"
echo " Size threshold: BYPASSED — no GB limit"
echo " Data recovery: NOT POSSIBLE after deletion"
echo ""
echo " Review --dry-run output before proceeding."
echo " You have 10 seconds to cancel (Ctrl+C)..."
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
sleep 10
echo " Proceeding..."
echo ""
fi
nuclear_mode_warning
# ==============================================================================================
# ━━━ Setup ━━━
@@ -194,15 +169,11 @@ if [[ -z "${LIDARR_URL:-}" ]] || [[ -z "${LIDARR_API_KEY:-}" ]]; then
exit 0
fi
DOCKER_TIMEOUT=15
ARR_DOCKER_TIMEOUT=15
LIDARR_CONTAINER="Lidarr" # container name on HOST1
# Build path map from MY_ID's Lidarr path map
declare -A ARR_PATH_MAP
local_path_map_var="${MY_ID}_LIDARR_PATH_MAP"
eval "for key in \"\${!${local_path_map_var}[@]}\"; do
ARR_PATH_MAP[\"\$key\"]=\"\${${local_path_map_var}[\$key]}\"
done"
build_arr_path_map "LIDARR"
# Validate required vars — detect_hosts() should have set these
require_var LIDARR_URL
@@ -251,83 +222,13 @@ fi
echo ""
echo "━━━ $ICON_SHIELD Safety Checks ━━━"
CONTAINER_RUNNING=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
'{{.State.Running}}' "$LIDARR_CONTAINER" 2>/dev/null)
if [[ "$CONTAINER_RUNNING" != "true" ]]; then
error "$LIDARR_CONTAINER is not running — aborting"
notify "Lidarr cleanup aborted on $(hostname) — container not running" \
"Lidarr Cleanup" "warning"
exit 1
fi
CONTAINER_HEALTH=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
'{{.State.Health.Status}}' "$LIDARR_CONTAINER" 2>/dev/null)
case "$CONTAINER_HEALTH" in
healthy) info "$LIDARR_CONTAINER is healthy" ;;
"") info "$LIDARR_CONTAINER has no health check — proceeding" ;;
starting)
error "$LIDARR_CONTAINER is still starting — aborting"
notify "Lidarr cleanup aborted on $(hostname) — container still starting" \
"Lidarr Cleanup" "warning"
exit 1 ;;
unhealthy)
error "$LIDARR_CONTAINER is unhealthy — aborting"
notify "Lidarr cleanup aborted on $(hostname) — container unhealthy" \
"Lidarr Cleanup" "warning"
exit 1 ;;
*) warn "$LIDARR_CONTAINER health: $CONTAINER_HEALTH — proceeding with caution" ;;
esac
info "Safety layer 1 passed — container healthy"
check_container_health "$LIDARR_CONTAINER" "$ARR_DOCKER_TIMEOUT" "Lidarr Cleanup"
# ==============================================================================================
# ── HELPER FUNCTIONS ──────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Lidarr API call with HTTP status check
# Usage: lidarr_api "artist" | lidarr_api "trackFile?artistId=123"
lidarr_api() {
local endpoint="$1"
local response http_code body
response=$(curl -sf \
--max-time 30 \
-H "X-Api-Key: $LIDARR_API_KEY" \
-w "\n%{http_code}" \
"${LIDARR_URL}/api/v1/${endpoint}" 2>/dev/null)
http_code=$(echo "$response" | tail -1)
body=$(echo "$response" | head -n -1)
if [[ "$http_code" != "200" ]]; then
error "Lidarr API HTTP $http_code for: $endpoint"
return 1
fi
echo "$body"
}
# Check if a file extension is a tracked music format
is_music_file() {
local ext="${1##*.}"
ext="${ext,,}"
for valid_ext in "${LIDARR_EXTENSIONS[@]}"; do
[[ "$ext" == "$valid_ext" ]] && return 0
done
return 1
}
# Check if a file matches any protected pattern
is_protected_file() {
local filename
filename=$(basename "$1")
for pattern in "${LIDARR_PROTECTED_PATTERNS[@]}"; do
# shellcheck disable=SC2254
case "$filename" in
$pattern) return 0 ;;
esac
done
return 1
}
# check_container_health(), arr_api(), has_extension(), matches_pattern_list() — common.sh
# ==============================================================================================
# ━━━ Pre-flight: Lidarr Import Scan ━━━
@@ -353,36 +254,7 @@ else
SCAN_PAYLOAD='{"name": "DownloadedAlbumsScan"}'
fi
SCAN_RESPONSE=$(curl -sf --max-time 30 -X POST \
-H "X-Api-Key: $LIDARR_API_KEY" \
-H "Content-Type: application/json" \
-d "$SCAN_PAYLOAD" \
"${LIDARR_URL}/api/v1/command" 2>/dev/null)
SCAN_CMD_ID=$(echo "$SCAN_RESPONSE" | jq -r '.id // empty' 2>/dev/null)
if [[ -z "$SCAN_CMD_ID" ]]; then
warn "Could not trigger import scan — proceeding without pre-flight"
else
info "Import scan queued (command ID: $SCAN_CMD_ID) — waiting for completion..."
POLL_TIMEOUT=${LIDARR_IMPORT_SCAN_TIMEOUT:-600}
POLLED=0
while [[ "$POLLED" -lt "$POLL_TIMEOUT" ]]; do
SCAN_STATUS=$(curl -sf --max-time 10 \
-H "X-Api-Key: $LIDARR_API_KEY" \
"${LIDARR_URL}/api/v1/command/${SCAN_CMD_ID}" 2>/dev/null | \
jq -r '.status // empty' 2>/dev/null)
case "$SCAN_STATUS" in
completed) info "Import scan complete ✅"; break ;;
failed) warn "Import scan reported failed — proceeding anyway"; break ;;
esac
sleep 10
(( POLLED += 10 ))
[[ $(( POLLED % 60 )) -eq 0 ]] && log " Still scanning... (${POLLED}s elapsed)"
done
[[ "$POLLED" -ge "$POLL_TIMEOUT" ]] && \
warn "Import scan timed out after ${POLL_TIMEOUT}s — proceeding anyway"
fi
trigger_and_await_command "$LIDARR_URL" "$LIDARR_API_KEY" "v1" "$SCAN_PAYLOAD" "${LIDARR_IMPORT_SCAN_TIMEOUT:-600}"
# ==============================================================================================
# ━━━ Fetch Lidarr Tracked Files ━━━
@@ -402,7 +274,7 @@ check_arr_version "$LIDARR_URL" "$LIDARR_API_KEY" "v1" "$LIDARR_VERSION_MAJOR" "
info "Querying Lidarr API..."
# Fetch all artists
ARTIST_RESPONSE=$(lidarr_api "artist") || {
ARTIST_RESPONSE=$(arr_api "$LIDARR_URL" "$LIDARR_API_KEY" "v1" "artist" "Lidarr") || {
error "Failed to fetch artists from Lidarr"
notify "Lidarr cleanup failed on $(hostname) — could not fetch artists" \
"Lidarr Cleanup" "warning"
@@ -427,7 +299,7 @@ TRACKED_FILE="$TMP_DIR/tracked_paths.txt"
while IFS= read -r artist_id; do
[[ -z "$artist_id" ]] && continue
ARTIST_TRACKS=$(lidarr_api "trackFile?artistId=${artist_id}" 2>/dev/null)
ARTIST_TRACKS=$(arr_api "$LIDARR_URL" "$LIDARR_API_KEY" "v1" "trackFile?artistId=${artist_id}" "Lidarr" 2>/dev/null)
if [[ -n "$ARTIST_TRACKS" ]]; then
while IFS= read -r api_path; do
[[ -z "$api_path" ]] && continue
@@ -459,25 +331,7 @@ fi
info "$ARTIST_COUNT artists | $TRACKED_COUNT tracked files"
# Safety Layer 6 — percentage drop vs last known count
if [[ -f "$LIDARR_TRACKED_COUNT_FILE" ]]; then
LAST_COUNT=$(cat "$LIDARR_TRACKED_COUNT_FILE" 2>/dev/null || echo 0)
if [[ "$LAST_COUNT" -gt 0 ]]; then
PCT=$(awk "BEGIN {printf \"%d\", ($TRACKED_COUNT / $LAST_COUNT) * 100}")
if [[ "$PCT" -lt "$LIDARR_MIN_TRACKED_PCT" ]]; then
error "Tracked count dropped to ${PCT}% of last run ($TRACKED_COUNT vs $LAST_COUNT)"
error "Suggests API issue — aborting to prevent mass deletion"
error "If expected (large library removal) delete: $LIDARR_TRACKED_COUNT_FILE"
notify "Lidarr cleanup aborted on $(hostname) — tracked count dropped to ${PCT}%" \
"Lidarr Cleanup" "warning"
exit 1
fi
info "Tracked count: ${PCT}% of last run ($TRACKED_COUNT vs $LAST_COUNT) ✅"
fi
else
info "No previous count on record — first run, saving baseline"
fi
echo "$TRACKED_COUNT" > "$LIDARR_TRACKED_COUNT_FILE"
check_tracked_count_floor "$TRACKED_COUNT" "$LIDARR_TRACKED_COUNT_FILE" "$LIDARR_MIN_TRACKED_PCT" "Lidarr Cleanup"
# ==============================================================================================
# ━━━ Scan Music Root ━━━
@@ -496,7 +350,6 @@ JUNK_BYTES=0
AGE_SECONDS=$(( LIDARR_ORPHAN_AGE * 86400 ))
NOW=$(date +%s)
MAX_DELETE_BYTES=$(awk "BEGIN {printf \"%d\", $LIDARR_MAX_DELETE_GB * 1073741824}")
while IFS= read -r filepath; do
[[ -z "$filepath" ]] && continue
@@ -508,7 +361,7 @@ while IFS= read -r filepath; do
fi
# Protected — never delete
if is_protected_file "$filepath"; then
if matches_pattern_list "$filepath" "${LIDARR_PROTECTED_PATTERNS[@]}"; then
log "$ICON_PROTECTED PROTECTED: $filepath"
(( PROTECTED_COUNT++ ))
continue
@@ -516,7 +369,7 @@ while IFS= read -r filepath; do
FILE_SIZE=$(stat -c%s "$filepath" 2>/dev/null || echo 0)
if is_music_file "$filepath"; then
if has_extension "$filepath" "${LIDARR_EXTENSIONS[@]}"; then
FILE_MTIME=$(stat -c %Y "$filepath" 2>/dev/null || echo 0)
FILE_AGE=$(( NOW - FILE_MTIME ))
@@ -543,21 +396,7 @@ TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT ))
# ==============================================================================================
# ━━━ Safety Layer 7 — Deletion Size Threshold ━━━
# ==============================================================================================
if [[ "$TOTAL_DELETE_BYTES" -gt "$MAX_DELETE_BYTES" ]]; then
TOTAL_HUMAN=$(awk "BEGIN {printf \"%.1fGB\", $TOTAL_DELETE_BYTES / 1073741824}")
if [[ "$I_KNOW" != true ]]; then
echo ""
error "Deletion would exceed ${LIDARR_MAX_DELETE_GB}GB — $TOTAL_HUMAN would be deleted"
error "Review ORPHAN lines above carefully before proceeding"
error "Rerun with: --i-know-what-im-doing"
error "To also bypass age check: add --skip-age-check"
notify "Lidarr cleanup halted on $(hostname) — ${TOTAL_HUMAN} requires --i-know-what-im-doing" \
"Lidarr Cleanup" "warning"
exit 1
else
warn "OVERRIDE — deletion is $TOTAL_HUMAN — proceeding with --i-know-what-im-doing"
fi
fi
check_delete_size_threshold "$TOTAL_DELETE_BYTES" "$LIDARR_MAX_DELETE_GB" "Lidarr Cleanup"
# ── Execute Deletions ─────────────────────────────────────────────────────────────────────────
# All safety layers passed — delete orphans and junk
@@ -565,12 +404,12 @@ if [[ "$DRY_RUN" == false ]]; then
while IFS= read -r filepath; do
[[ -z "$filepath" ]] && continue
[[ -n "${TRACKED_MAP[$filepath]:-}" ]] && continue
is_protected_file "$filepath" && continue
matches_pattern_list "$filepath" "${LIDARR_PROTECTED_PATTERNS[@]}" && continue
FILE_MTIME=$(stat -c %Y "$filepath" 2>/dev/null || echo 0)
FILE_AGE=$(( NOW - FILE_MTIME ))
if is_music_file "$filepath"; then
if has_extension "$filepath" "${LIDARR_EXTENSIONS[@]}"; then
[[ "$FILE_AGE" -lt "$AGE_SECONDS" ]] && \
[[ "$SKIP_AGE_CHECK" != true ]] && continue
fi
@@ -586,16 +425,7 @@ fi
END=$(date +%s)
format_bytes() {
local bytes=$1
if (( bytes > 1073741824 )); then
awk "BEGIN {printf \"%.1fGB\", $bytes / 1073741824}"
elif (( bytes > 1048576 )); then
awk "BEGIN {printf \"%.1fMB\", $bytes / 1048576}"
else
echo "${bytes}B"
fi
}
# format_bytes() — provided by common.sh
ORPHAN_HUMAN=$(format_bytes "$ORPHAN_BYTES")
JUNK_HUMAN=$(format_bytes "$JUNK_BYTES")
Arrs_Stack/lidarr_release_fixer.sh
+1 -12
View File
@@ -283,17 +283,6 @@ read_file_mbid() {
esac
}
translate_container_path() {
local cpath="$1"
for cp in "${!ARR_PATH_MAP[@]}"; do
if [[ "$cpath" == "${cp}"* ]]; then
echo "${ARR_PATH_MAP[$cp]}${cpath#$cp}"
return
fi
done
echo "$cpath"
}
# ── Safety checks ─────────────────────────────────────────────────────────────────────────────
echo ""
echo "━━━ $ICON_SHIELD Safety Checks ━━━"
@@ -322,7 +311,7 @@ while IFS= read -r artist; do
aid=$(echo "$artist" | jq -r '.id')
apath=$(echo "$artist" | jq -r '.path // empty')
aname=$(echo "$artist" | jq -r '.artistName // empty')
[[ -n "$apath" ]] && ARTIST_PATH_CACHE[$aid]=$(translate_container_path "$apath")
[[ -n "$apath" ]] && ARTIST_PATH_CACHE[$aid]=$(translate_path "$apath")
[[ -n "$aname" ]] && ARTIST_NAME_CACHE[$aid]="$aname"
done < <(echo "$ALL_ARTISTS" | jq -c '.[]' 2>/dev/null)
unset ALL_ARTISTS
Arrs_Stack/playback_aware_radarr_discovery.sh
+2 -31
View File
@@ -134,7 +134,6 @@
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../load_config.sh"
source "$SCRIPT_DIR/../Kernel/decision_engine.sh"
parse_args "$@"
@@ -185,7 +184,7 @@ log "$ICON_GEAR Config: threshold=${THRESHOLD} lookback=${LOOKBACK_DAYS}d max-se
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no movies will be added to Radarr"
_fmt_rating() { local v="${1:-0}"; echo "${v::-1}.${v: -1}" 2>/dev/null || echo "$v"; }
# _fmt_rating() — provided by common.sh
# ==============================================================================================
# ━━━ Status ━━━
@@ -275,35 +274,7 @@ _freq_score() {
fi
}
# vote_avg_int = vote_average × 10 as integer (e.g. 7.8 → 78)
_rating_score_s2() {
local v="$1"
if (( v >= 80 )); then echo 40
elif (( v >= 75 )); then echo 32
elif (( v >= 70 )); then echo 25
elif (( v >= 65 )); then echo 18
elif (( v >= 60 )); then echo 12
else echo 5
fi
}
_votes_score() {
local c="$1"
if (( c >= 10000 )); then echo 20
elif (( c >= 5000 )); then echo 15
elif (( c >= 1000 )); then echo 10
elif (( c >= 200 )); then echo 5
else echo 2
fi
}
_breadth_score() {
local seeds="$1"
if (( seeds >= 3 )); then echo 40
elif (( seeds == 2 )); then echo 25
else echo 10
fi
}
# _rating_score_s2(), _votes_score(), _breadth_score() — provided by common.sh
# ==============================================================================================
# ━━━ Fetch Emby Libraries + Recently Watched Movies ━━━
Arrs_Stack/playback_aware_sonarr_discovery.sh
+2 -36
View File
@@ -148,7 +148,6 @@
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../load_config.sh"
source "$SCRIPT_DIR/../Kernel/decision_engine.sh"
parse_args "$@"
@@ -205,7 +204,7 @@ log "$ICON_GEAR Config: threshold=${THRESHOLD} lookback=${LOOKBACK_DAYS}d max-se
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no shows will be added to Sonarr"
_fmt_rating() { local v="${1:-0}"; echo "${v::-1}.${v: -1}" 2>/dev/null || echo "$v"; }
# _fmt_rating() — provided by common.sh
# ==============================================================================================
# ━━━ Status ━━━
@@ -315,37 +314,7 @@ _volume_score() {
fi
}
# Stage 2: TMDB vote_average × 10 (0-40)
_rating_score_s2() {
local v="$1"
if (( v >= 80 )); then echo 40
elif (( v >= 75 )); then echo 32
elif (( v >= 70 )); then echo 25
elif (( v >= 65 )); then echo 18
elif (( v >= 60 )); then echo 12
else echo 5
fi
}
# Stage 2: vote count (0-20)
_votes_score() {
local c="$1"
if (( c >= 10000 )); then echo 20
elif (( c >= 5000 )); then echo 15
elif (( c >= 1000 )); then echo 10
elif (( c >= 200 )); then echo 5
else echo 2
fi
}
# Stage 2: seed breadth (0-40)
_breadth_score() {
local seeds="$1"
if (( seeds >= 3 )); then echo 40
elif (( seeds == 2 )); then echo 25
else echo 10
fi
}
# _rating_score_s2(), _votes_score(), _breadth_score() — provided by common.sh
# ==============================================================================================
# ━━━ Fetch Emby Series Library ━━━
@@ -636,9 +605,6 @@ done < <(echo "$SONARR_SERIES_JSON" | jq -r '.[] |
log "${#SONARR_TVDB[@]} series in Sonarr | ${#EMBY_TMDB_IDS[@]} series in Emby"
_in_sonarr() { [[ "${SONARR_TVDB["$1"]+x}" || "${SONARR_TMDB["$2"]+x}" ]]; }
_in_emby() { [[ "${EMBY_TMDB_IDS["$1"]+x}" || "${EMBY_TVDB_IDS["$2"]+x}" ]]; }
# ==============================================================================================
# ━━━ Score Stage 2 Candidates ━━━
# ==============================================================================================
Arrs_Stack/radarr_cleanup.sh
+17 -185
View File
@@ -64,7 +64,7 @@
#
# acquire_lock "wait" — large scans take time, wait for previous run to finish
# jq + curl validation — exits if either tool missing
# DOCKER_TIMEOUT — container checks protected against daemon hangs
# ARR_DOCKER_TIMEOUT — container checks protected against daemon hangs (script-local, not common.sh's DOCKER_TIMEOUT)
# notify_emby_scan() — triggers Emby clean after deletion
# platform_require_cmd — notify script validated before use
# Silent by default — orphans/junk warn(), clean library logs silently
@@ -120,37 +120,12 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../load_config.sh"
# ── Special flag pre-processing ───────────────────────────────────────────────────────────────
I_KNOW=false
SKIP_AGE_CHECK=false
FILTERED_ARGS=()
for arg in "$@"; do
case "$arg" in
--i-know-what-im-doing) I_KNOW=true ;;
--skip-age-check) SKIP_AGE_CHECK=true ;;
*) FILTERED_ARGS+=("$arg") ;;
esac
done
parse_destructive_flags "$@"
parse_args "${FILTERED_ARGS[@]}"
# ── Nuclear mode warning ──────────────────────────────────────────────────────────────────────
if [[ "$I_KNOW" == true ]] && [[ "$SKIP_AGE_CHECK" == true ]] && [[ "$DRY_RUN" != true ]]; then
echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "⚠️ WARNING — NUCLEAR MODE ACTIVE"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo " Flags: --i-know-what-im-doing --skip-age-check"
echo " Age check: BYPASSED — deletes on first pass"
echo " Size threshold: BYPASSED — no GB limit"
echo " Data recovery: NOT POSSIBLE after deletion"
echo ""
echo " Review --dry-run output before proceeding."
echo " You have 10 seconds to cancel (Ctrl+C)..."
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
sleep 10
echo " Proceeding..."
echo ""
fi
nuclear_mode_warning
# ==============================================================================================
# ━━━ Setup ━━━
@@ -191,15 +166,11 @@ if [[ -z "${RADARR_URL:-}" ]] || [[ -z "${RADARR_API_KEY:-}" ]]; then
exit 0
fi
DOCKER_TIMEOUT=15
ARR_DOCKER_TIMEOUT=15
RADARR_CONTAINER="Radarr"
# Build path map from MY_ID's Radarr path map
declare -A ARR_PATH_MAP
local_path_map_var="${MY_ID}_RADARR_PATH_MAP"
eval "for key in \"\${!${local_path_map_var}[@]}\"; do
ARR_PATH_MAP[\"\$key\"]=\"\${${local_path_map_var}[\$key]}\"
done"
build_arr_path_map "RADARR"
require_var RADARR_URL
require_var RADARR_API_KEY
@@ -247,90 +218,13 @@ fi
echo ""
echo "━━━ $ICON_SHIELD Safety Checks ━━━"
CONTAINER_RUNNING=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
'{{.State.Running}}' "$RADARR_CONTAINER" 2>/dev/null)
if [[ "$CONTAINER_RUNNING" != "true" ]]; then
error "$RADARR_CONTAINER is not running — aborting"
notify "Radarr cleanup aborted on $(hostname) — container not running" \
"Radarr Cleanup" "warning"
exit 1
fi
CONTAINER_HEALTH=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
'{{.State.Health.Status}}' "$RADARR_CONTAINER" 2>/dev/null)
case "$CONTAINER_HEALTH" in
healthy) info "$RADARR_CONTAINER is healthy" ;;
"") info "$RADARR_CONTAINER has no health check — proceeding" ;;
starting)
error "$RADARR_CONTAINER is still starting — aborting"
notify "Radarr cleanup aborted on $(hostname) — container still starting" \
"Radarr Cleanup" "warning"
exit 1 ;;
unhealthy)
error "$RADARR_CONTAINER is unhealthy — aborting"
notify "Radarr cleanup aborted on $(hostname) — container unhealthy" \
"Radarr Cleanup" "warning"
exit 1 ;;
*) warn "$RADARR_CONTAINER health: $CONTAINER_HEALTH — proceeding with caution" ;;
esac
info "Safety layer 1 passed — container healthy"
check_container_health "$RADARR_CONTAINER" "$ARR_DOCKER_TIMEOUT" "Radarr Cleanup"
# ==============================================================================================
# ── HELPER FUNCTIONS ──────────────────────────────────────────────────────────────────────────
# ==============================================================================================
radarr_api() {
local endpoint="$1"
local response http_code body
response=$(curl -sf \
--max-time 30 \
-H "X-Api-Key: $RADARR_API_KEY" \
-w "\n%{http_code}" \
"${RADARR_URL}/api/v3/${endpoint}" 2>/dev/null)
http_code=$(echo "$response" | tail -1)
body=$(echo "$response" | head -n -1)
if [[ "$http_code" != "200" ]]; then
error "Radarr API HTTP $http_code for: $endpoint"
return 1
fi
echo "$body"
}
is_video_file() {
local ext="${1##*.}"
ext="${ext,,}"
for valid_ext in "${RADARR_EXTENSIONS[@]}"; do
[[ "$ext" == "$valid_ext" ]] && return 0
done
return 1
}
is_protected_file() {
local filename
filename=$(basename "$1")
for pattern in "${RADARR_PROTECTED_PATTERNS[@]}"; do
# shellcheck disable=SC2254
case "$filename" in
$pattern) return 0 ;;
esac
done
return 1
}
format_bytes() {
local bytes=$1
if (( bytes > 1073741824 )); then
awk "BEGIN {printf \"%.1fGB\", $bytes / 1073741824}"
elif (( bytes > 1048576 )); then
awk "BEGIN {printf \"%.1fMB\", $bytes / 1048576}"
else
echo "${bytes}B"
fi
}
# check_container_health(), arr_api(), has_extension(), matches_pattern_list(), format_bytes() — common.sh
# ==============================================================================================
# ━━━ Pre-flight: Radarr Import Scan ━━━
@@ -340,7 +234,7 @@ echo "━━━ $ICON_SYNC Pre-flight: Radarr Import Scan ━━━"
# Fetch root folders from Radarr API and translate container paths to host paths
mapfile -t SCAN_ROOTS < <(
radarr_api "rootfolder" | \
arr_api "$RADARR_URL" "$RADARR_API_KEY" "v3" "rootfolder" "Radarr" | \
jq -r '.[].path' 2>/dev/null | \
while IFS= read -r cp; do translate_path "$cp"; done
)
@@ -357,36 +251,7 @@ info "Scan targets (${#SCAN_ROOTS[@]}): ${SCAN_ROOTS[*]}"
info "Triggering ProcessMonitoredDownloads pre-flight"
SCAN_PAYLOAD='{"name": "ProcessMonitoredDownloads"}'
SCAN_RESPONSE=$(curl -sf --max-time 30 -X POST \
-H "X-Api-Key: $RADARR_API_KEY" \
-H "Content-Type: application/json" \
-d "$SCAN_PAYLOAD" \
"${RADARR_URL}/api/v3/command" 2>/dev/null)
SCAN_CMD_ID=$(echo "$SCAN_RESPONSE" | jq -r '.id // empty' 2>/dev/null)
if [[ -z "$SCAN_CMD_ID" ]]; then
warn "Could not trigger import scan — proceeding without pre-flight"
else
info "Import scan queued (command ID: $SCAN_CMD_ID) — waiting for completion..."
POLL_TIMEOUT=${RADARR_IMPORT_SCAN_TIMEOUT:-600}
POLLED=0
while [[ "$POLLED" -lt "$POLL_TIMEOUT" ]]; do
SCAN_STATUS=$(curl -sf --max-time 10 \
-H "X-Api-Key: $RADARR_API_KEY" \
"${RADARR_URL}/api/v3/command/${SCAN_CMD_ID}" 2>/dev/null | \
jq -r '.status // empty' 2>/dev/null)
case "$SCAN_STATUS" in
completed) info "Import scan complete ✅"; break ;;
failed) warn "Import scan reported failed — proceeding anyway"; break ;;
esac
sleep 10
(( POLLED += 10 ))
[[ $(( POLLED % 60 )) -eq 0 ]] && log " Still scanning... (${POLLED}s elapsed)"
done
[[ "$POLLED" -ge "$POLL_TIMEOUT" ]] && \
warn "Import scan timed out after ${POLL_TIMEOUT}s — proceeding anyway"
fi
trigger_and_await_command "$RADARR_URL" "$RADARR_API_KEY" "v3" "$SCAN_PAYLOAD" "${RADARR_IMPORT_SCAN_TIMEOUT:-600}"
# ==============================================================================================
# ━━━ Fetch Radarr Tracked Files ━━━
@@ -406,7 +271,7 @@ check_arr_version "$RADARR_URL" "$RADARR_API_KEY" "v3" "$RADARR_VERSION_MAJOR" "
info "Querying Radarr API..."
# Fetch all movies
MOVIES_RESPONSE=$(radarr_api "movie") || {
MOVIES_RESPONSE=$(arr_api "$RADARR_URL" "$RADARR_API_KEY" "v3" "movie" "Radarr") || {
error "Failed to fetch movies from Radarr"
notify "Radarr cleanup failed on $(hostname) — could not fetch movies" \
"Radarr Cleanup" "warning"
@@ -435,7 +300,7 @@ while IFS= read -r movie_id; do
(( MOVIE_INDEX++ ))
[[ $(( MOVIE_INDEX % 100 )) -eq 0 ]] && \
log "Fetching files: $MOVIE_INDEX/$MOVIE_COUNT movies..."
MOVIE_FILES=$(radarr_api "moviefile?movieId=${movie_id}" 2>/dev/null)
MOVIE_FILES=$(arr_api "$RADARR_URL" "$RADARR_API_KEY" "v3" "moviefile?movieId=${movie_id}" "Radarr" 2>/dev/null)
if [[ -n "$MOVIE_FILES" ]]; then
while IFS= read -r api_path; do
[[ -z "$api_path" ]] && continue
@@ -467,25 +332,7 @@ fi
info "$MOVIE_COUNT movies | $TRACKED_COUNT tracked movie files"
# Safety Layer 6 — percentage drop vs last known count
if [[ -f "$RADARR_TRACKED_COUNT_FILE" ]]; then
LAST_COUNT=$(cat "$RADARR_TRACKED_COUNT_FILE" 2>/dev/null || echo 0)
if [[ "$LAST_COUNT" -gt 0 ]]; then
PCT=$(awk "BEGIN {printf \"%d\", ($TRACKED_COUNT / $LAST_COUNT) * 100}")
if [[ "$PCT" -lt "$RADARR_MIN_TRACKED_PCT" ]]; then
error "Tracked count dropped to ${PCT}% of last run ($TRACKED_COUNT vs $LAST_COUNT)"
error "Suggests API issue — aborting to prevent mass deletion"
error "If expected (large library removal) delete: $RADARR_TRACKED_COUNT_FILE"
notify "Radarr cleanup aborted on $(hostname) — tracked count dropped to ${PCT}%" \
"Radarr Cleanup" "warning"
exit 1
fi
info "Tracked count: ${PCT}% of last run ($TRACKED_COUNT vs $LAST_COUNT) ✅"
fi
else
info "No previous count on record — first run, saving baseline"
fi
echo "$TRACKED_COUNT" > "$RADARR_TRACKED_COUNT_FILE"
check_tracked_count_floor "$TRACKED_COUNT" "$RADARR_TRACKED_COUNT_FILE" "$RADARR_MIN_TRACKED_PCT" "Radarr Cleanup"
# ==============================================================================================
# ━━━ Scan Movies Root ━━━
@@ -504,7 +351,6 @@ JUNK_BYTES=0
AGE_SECONDS=$(( RADARR_ORPHAN_AGE * 86400 ))
NOW=$(date +%s)
MAX_DELETE_BYTES=$(awk "BEGIN {printf \"%d\", $RADARR_MAX_DELETE_GB * 1073741824}")
while IFS= read -r filepath; do
[[ -z "$filepath" ]] && continue
@@ -514,7 +360,7 @@ while IFS= read -r filepath; do
continue
fi
if is_protected_file "$filepath"; then
if matches_pattern_list "$filepath" "${RADARR_PROTECTED_PATTERNS[@]}"; then
log "$ICON_PROTECTED PROTECTED: $filepath"
(( PROTECTED_COUNT++ ))
continue
@@ -522,7 +368,7 @@ while IFS= read -r filepath; do
FILE_SIZE=$(stat -c%s "$filepath" 2>/dev/null || echo 0)
if is_video_file "$filepath"; then
if has_extension "$filepath" "${RADARR_EXTENSIONS[@]}"; then
FILE_MTIME=$(stat -c %Y "$filepath" 2>/dev/null || echo 0)
FILE_AGE=$(( NOW - FILE_MTIME ))
@@ -553,33 +399,19 @@ TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT ))
# ==============================================================================================
# ━━━ Safety Layer 7 — Deletion Size Threshold ━━━
# ==============================================================================================
if [[ "$TOTAL_DELETE_BYTES" -gt "$MAX_DELETE_BYTES" ]]; then
TOTAL_HUMAN=$(awk "BEGIN {printf \"%.1fGB\", $TOTAL_DELETE_BYTES / 1073741824}")
if [[ "$I_KNOW" != true ]]; then
echo ""
error "Deletion would exceed ${RADARR_MAX_DELETE_GB}GB — $TOTAL_HUMAN would be deleted"
error "Review ORPHAN lines above carefully before proceeding"
error "Rerun with: --i-know-what-im-doing"
error "To also bypass age check: add --skip-age-check"
notify "Radarr cleanup halted on $(hostname) — ${TOTAL_HUMAN} requires --i-know-what-im-doing" \
"Radarr Cleanup" "warning"
exit 1
else
warn "OVERRIDE — deletion is $TOTAL_HUMAN — proceeding with --i-know-what-im-doing"
fi
fi
check_delete_size_threshold "$TOTAL_DELETE_BYTES" "$RADARR_MAX_DELETE_GB" "Radarr Cleanup"
# ── Execute Deletions ─────────────────────────────────────────────────────────────────────────
if [[ "$DRY_RUN" == false ]]; then
while IFS= read -r filepath; do
[[ -z "$filepath" ]] && continue
[[ -n "${TRACKED_MAP[$filepath]:-}" ]] && continue
is_protected_file "$filepath" && continue
matches_pattern_list "$filepath" "${RADARR_PROTECTED_PATTERNS[@]}" && continue
FILE_MTIME=$(stat -c %Y "$filepath" 2>/dev/null || echo 0)
FILE_AGE=$(( NOW - FILE_MTIME ))
if is_video_file "$filepath"; then
if has_extension "$filepath" "${RADARR_EXTENSIONS[@]}"; then
[[ "$FILE_AGE" -lt "$AGE_SECONDS" ]] && \
[[ "$SKIP_AGE_CHECK" != true ]] && continue
fi
Arrs_Stack/sonarr_cleanup.sh
+17 -185
View File
@@ -64,7 +64,7 @@
#
# acquire_lock "wait" — large scans take time, wait for previous run to finish
# jq + curl validation — exits if either tool missing
# DOCKER_TIMEOUT — container checks protected against daemon hangs
# ARR_DOCKER_TIMEOUT — container checks protected against daemon hangs (script-local, not common.sh's DOCKER_TIMEOUT)
# notify_emby_scan() — triggers Emby clean after deletion
# platform_require_cmd — notify script validated before use
# Silent by default — orphans/junk warn(), clean library logs silently
@@ -120,37 +120,12 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../load_config.sh"
# ── Special flag pre-processing ───────────────────────────────────────────────────────────────
I_KNOW=false
SKIP_AGE_CHECK=false
FILTERED_ARGS=()
for arg in "$@"; do
case "$arg" in
--i-know-what-im-doing) I_KNOW=true ;;
--skip-age-check) SKIP_AGE_CHECK=true ;;
*) FILTERED_ARGS+=("$arg") ;;
esac
done
parse_destructive_flags "$@"
parse_args "${FILTERED_ARGS[@]}"
# ── Nuclear mode warning ──────────────────────────────────────────────────────────────────────
if [[ "$I_KNOW" == true ]] && [[ "$SKIP_AGE_CHECK" == true ]] && [[ "$DRY_RUN" != true ]]; then
echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "⚠️ WARNING — NUCLEAR MODE ACTIVE"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo " Flags: --i-know-what-im-doing --skip-age-check"
echo " Age check: BYPASSED — deletes on first pass"
echo " Size threshold: BYPASSED — no GB limit"
echo " Data recovery: NOT POSSIBLE after deletion"
echo ""
echo " Review --dry-run output before proceeding."
echo " You have 10 seconds to cancel (Ctrl+C)..."
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
sleep 10
echo " Proceeding..."
echo ""
fi
nuclear_mode_warning
# ==============================================================================================
# ━━━ Setup ━━━
@@ -191,15 +166,11 @@ if [[ -z "${SONARR_URL:-}" ]] || [[ -z "${SONARR_API_KEY:-}" ]]; then
exit 0
fi
DOCKER_TIMEOUT=15
ARR_DOCKER_TIMEOUT=15
SONARR_CONTAINER="Sonarr"
# Build path map from MY_ID's Sonarr path map
declare -A ARR_PATH_MAP
local_path_map_var="${MY_ID}_SONARR_PATH_MAP"
eval "for key in \"\${!${local_path_map_var}[@]}\"; do
ARR_PATH_MAP[\"\$key\"]=\"\${${local_path_map_var}[\$key]}\"
done"
build_arr_path_map "SONARR"
require_var SONARR_URL
require_var SONARR_API_KEY
@@ -247,90 +218,13 @@ fi
echo ""
echo "━━━ $ICON_SHIELD Safety Checks ━━━"
CONTAINER_RUNNING=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
'{{.State.Running}}' "$SONARR_CONTAINER" 2>/dev/null)
if [[ "$CONTAINER_RUNNING" != "true" ]]; then
error "$SONARR_CONTAINER is not running — aborting"
notify "Sonarr cleanup aborted on $(hostname) — container not running" \
"Sonarr Cleanup" "warning"
exit 1
fi
CONTAINER_HEALTH=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
'{{.State.Health.Status}}' "$SONARR_CONTAINER" 2>/dev/null)
case "$CONTAINER_HEALTH" in
healthy) info "$SONARR_CONTAINER is healthy" ;;
"") info "$SONARR_CONTAINER has no health check — proceeding" ;;
starting)
error "$SONARR_CONTAINER is still starting — aborting"
notify "Sonarr cleanup aborted on $(hostname) — container still starting" \
"Sonarr Cleanup" "warning"
exit 1 ;;
unhealthy)
error "$SONARR_CONTAINER is unhealthy — aborting"
notify "Sonarr cleanup aborted on $(hostname) — container unhealthy" \
"Sonarr Cleanup" "warning"
exit 1 ;;
*) warn "$SONARR_CONTAINER health: $CONTAINER_HEALTH — proceeding with caution" ;;
esac
info "Safety layer 1 passed — container healthy"
check_container_health "$SONARR_CONTAINER" "$ARR_DOCKER_TIMEOUT" "Sonarr Cleanup"
# ==============================================================================================
# ── HELPER FUNCTIONS ──────────────────────────────────────────────────────────────────────────
# ==============================================================================================
sonarr_api() {
local endpoint="$1"
local response http_code body
response=$(curl -sf \
--max-time 30 \
-H "X-Api-Key: $SONARR_API_KEY" \
-w "\n%{http_code}" \
"${SONARR_URL}/api/v3/${endpoint}" 2>/dev/null)
http_code=$(echo "$response" | tail -1)
body=$(echo "$response" | head -n -1)
if [[ "$http_code" != "200" ]]; then
error "Sonarr API HTTP $http_code for: $endpoint"
return 1
fi
echo "$body"
}
is_video_file() {
local ext="${1##*.}"
ext="${ext,,}"
for valid_ext in "${SONARR_EXTENSIONS[@]}"; do
[[ "$ext" == "$valid_ext" ]] && return 0
done
return 1
}
is_protected_file() {
local filename
filename=$(basename "$1")
for pattern in "${SONARR_PROTECTED_PATTERNS[@]}"; do
# shellcheck disable=SC2254
case "$filename" in
$pattern) return 0 ;;
esac
done
return 1
}
format_bytes() {
local bytes=$1
if (( bytes > 1073741824 )); then
awk "BEGIN {printf \"%.1fGB\", $bytes / 1073741824}"
elif (( bytes > 1048576 )); then
awk "BEGIN {printf \"%.1fMB\", $bytes / 1048576}"
else
echo "${bytes}B"
fi
}
# check_container_health(), arr_api(), has_extension(), matches_pattern_list(), format_bytes() — common.sh
# ==============================================================================================
# ━━━ Pre-flight: Sonarr Import Scan ━━━
@@ -340,7 +234,7 @@ echo "━━━ $ICON_SYNC Pre-flight: Sonarr Import Scan ━━━"
# Fetch root folders from Sonarr API and translate container paths to host paths
mapfile -t SCAN_ROOTS < <(
sonarr_api "rootfolder" | \
arr_api "$SONARR_URL" "$SONARR_API_KEY" "v3" "rootfolder" "Sonarr" | \
jq -r '.[].path' 2>/dev/null | \
while IFS= read -r cp; do translate_path "$cp"; done
)
@@ -357,36 +251,7 @@ info "Scan targets (${#SCAN_ROOTS[@]}): ${SCAN_ROOTS[*]}"
info "Triggering ProcessMonitoredDownloads pre-flight"
SCAN_PAYLOAD='{"name": "ProcessMonitoredDownloads"}'
SCAN_RESPONSE=$(curl -sf --max-time 30 -X POST \
-H "X-Api-Key: $SONARR_API_KEY" \
-H "Content-Type: application/json" \
-d "$SCAN_PAYLOAD" \
"${SONARR_URL}/api/v3/command" 2>/dev/null)
SCAN_CMD_ID=$(echo "$SCAN_RESPONSE" | jq -r '.id // empty' 2>/dev/null)
if [[ -z "$SCAN_CMD_ID" ]]; then
warn "Could not trigger import scan — proceeding without pre-flight"
else
info "Import scan queued (command ID: $SCAN_CMD_ID) — waiting for completion..."
POLL_TIMEOUT=${SONARR_IMPORT_SCAN_TIMEOUT:-600}
POLLED=0
while [[ "$POLLED" -lt "$POLL_TIMEOUT" ]]; do
SCAN_STATUS=$(curl -sf --max-time 10 \
-H "X-Api-Key: $SONARR_API_KEY" \
"${SONARR_URL}/api/v3/command/${SCAN_CMD_ID}" 2>/dev/null | \
jq -r '.status // empty' 2>/dev/null)
case "$SCAN_STATUS" in
completed) info "Import scan complete ✅"; break ;;
failed) warn "Import scan reported failed — proceeding anyway"; break ;;
esac
sleep 10
(( POLLED += 10 ))
[[ $(( POLLED % 60 )) -eq 0 ]] && log " Still scanning... (${POLLED}s elapsed)"
done
[[ "$POLLED" -ge "$POLL_TIMEOUT" ]] && \
warn "Import scan timed out after ${POLL_TIMEOUT}s — proceeding anyway"
fi
trigger_and_await_command "$SONARR_URL" "$SONARR_API_KEY" "v3" "$SCAN_PAYLOAD" "${SONARR_IMPORT_SCAN_TIMEOUT:-600}"
# ==============================================================================================
# ━━━ Fetch Sonarr Tracked Files ━━━
@@ -406,7 +271,7 @@ check_arr_version "$SONARR_URL" "$SONARR_API_KEY" "v3" "$SONARR_VERSION_MAJOR" "
info "Querying Sonarr API..."
# Fetch all series
SERIES_RESPONSE=$(sonarr_api "series") || {
SERIES_RESPONSE=$(arr_api "$SONARR_URL" "$SONARR_API_KEY" "v3" "series" "Sonarr") || {
error "Failed to fetch series from Sonarr"
notify "Sonarr cleanup failed on $(hostname) — could not fetch series" \
"Sonarr Cleanup" "warning"
@@ -435,7 +300,7 @@ while IFS= read -r series_id; do
(( SERIES_INDEX++ ))
[[ $(( SERIES_INDEX % 50 )) -eq 0 ]] && \
log "Fetching files: $SERIES_INDEX/$SERIES_COUNT series..."
SERIES_FILES=$(sonarr_api "episodefile?seriesId=${series_id}" 2>/dev/null)
SERIES_FILES=$(arr_api "$SONARR_URL" "$SONARR_API_KEY" "v3" "episodefile?seriesId=${series_id}" "Sonarr" 2>/dev/null)
if [[ -n "$SERIES_FILES" ]]; then
while IFS= read -r api_path; do
[[ -z "$api_path" ]] && continue
@@ -466,25 +331,7 @@ fi
info "$SERIES_COUNT series | $TRACKED_COUNT tracked episode files"
# Safety Layer 6 — percentage drop vs last known count
if [[ -f "$SONARR_TRACKED_COUNT_FILE" ]]; then
LAST_COUNT=$(cat "$SONARR_TRACKED_COUNT_FILE" 2>/dev/null || echo 0)
if [[ "$LAST_COUNT" -gt 0 ]]; then
PCT=$(awk "BEGIN {printf \"%d\", ($TRACKED_COUNT / $LAST_COUNT) * 100}")
if [[ "$PCT" -lt "$SONARR_MIN_TRACKED_PCT" ]]; then
error "Tracked count dropped to ${PCT}% of last run ($TRACKED_COUNT vs $LAST_COUNT)"
error "Suggests API issue — aborting to prevent mass deletion"
error "If expected (large library removal) delete: $SONARR_TRACKED_COUNT_FILE"
notify "Sonarr cleanup aborted on $(hostname) — tracked count dropped to ${PCT}%" \
"Sonarr Cleanup" "warning"
exit 1
fi
info "Tracked count: ${PCT}% of last run ($TRACKED_COUNT vs $LAST_COUNT) ✅"
fi
else
info "No previous count on record — first run, saving baseline"
fi
echo "$TRACKED_COUNT" > "$SONARR_TRACKED_COUNT_FILE"
check_tracked_count_floor "$TRACKED_COUNT" "$SONARR_TRACKED_COUNT_FILE" "$SONARR_MIN_TRACKED_PCT" "Sonarr Cleanup"
# ==============================================================================================
# ━━━ Scan TV Root ━━━
@@ -503,7 +350,6 @@ JUNK_BYTES=0
AGE_SECONDS=$(( SONARR_ORPHAN_AGE * 86400 ))
NOW=$(date +%s)
MAX_DELETE_BYTES=$(awk "BEGIN {printf \"%d\", $SONARR_MAX_DELETE_GB * 1073741824}")
while IFS= read -r filepath; do
[[ -z "$filepath" ]] && continue
@@ -513,7 +359,7 @@ while IFS= read -r filepath; do
continue
fi
if is_protected_file "$filepath"; then
if matches_pattern_list "$filepath" "${SONARR_PROTECTED_PATTERNS[@]}"; then
log "$ICON_PROTECTED PROTECTED: $filepath"
(( PROTECTED_COUNT++ ))
continue
@@ -521,7 +367,7 @@ while IFS= read -r filepath; do
FILE_SIZE=$(stat -c%s "$filepath" 2>/dev/null || echo 0)
if is_video_file "$filepath"; then
if has_extension "$filepath" "${SONARR_EXTENSIONS[@]}"; then
FILE_MTIME=$(stat -c %Y "$filepath" 2>/dev/null || echo 0)
FILE_AGE=$(( NOW - FILE_MTIME ))
@@ -552,33 +398,19 @@ TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT ))
# ==============================================================================================
# ━━━ Safety Layer 7 — Deletion Size Threshold ━━━
# ==============================================================================================
if [[ "$TOTAL_DELETE_BYTES" -gt "$MAX_DELETE_BYTES" ]]; then
TOTAL_HUMAN=$(awk "BEGIN {printf \"%.1fGB\", $TOTAL_DELETE_BYTES / 1073741824}")
if [[ "$I_KNOW" != true ]]; then
echo ""
error "Deletion would exceed ${SONARR_MAX_DELETE_GB}GB — $TOTAL_HUMAN would be deleted"
error "Review ORPHAN lines above carefully before proceeding"
error "Rerun with: --i-know-what-im-doing"
error "To also bypass age check: add --skip-age-check"
notify "Sonarr cleanup halted on $(hostname) — ${TOTAL_HUMAN} requires --i-know-what-im-doing" \
"Sonarr Cleanup" "warning"
exit 1
else
warn "OVERRIDE — deletion is $TOTAL_HUMAN — proceeding with --i-know-what-im-doing"
fi
fi
check_delete_size_threshold "$TOTAL_DELETE_BYTES" "$SONARR_MAX_DELETE_GB" "Sonarr Cleanup"
# ── Execute Deletions ─────────────────────────────────────────────────────────────────────────
if [[ "$DRY_RUN" == false ]]; then
while IFS= read -r filepath; do
[[ -z "$filepath" ]] && continue
[[ -n "${TRACKED_MAP[$filepath]:-}" ]] && continue
is_protected_file "$filepath" && continue
matches_pattern_list "$filepath" "${SONARR_PROTECTED_PATTERNS[@]}" && continue
FILE_MTIME=$(stat -c %Y "$filepath" 2>/dev/null || echo 0)
FILE_AGE=$(( NOW - FILE_MTIME ))
if is_video_file "$filepath"; then
if has_extension "$filepath" "${SONARR_EXTENSIONS[@]}"; then
[[ "$FILE_AGE" -lt "$AGE_SECONDS" ]] && \
[[ "$SKIP_AGE_CHECK" != true ]] && continue
fi
Arrs_Stack/upgrade_webhook_handler.sh
+1 -7
View File
@@ -103,13 +103,7 @@ case "$ARR_TYPE" in
esac
# ── Remote node list ──────────────────────────────────────────────────────────────────────────
declare -a REMOTE_NODES=()
for _hv in HOST1 HOST2 HOST3 HOST4 HOST5 HOST6 HOST7 HOST8; do
[[ "$_hv" == "$MY_ID" ]] && continue
[[ -z "${!_hv:-}" ]] && continue
REMOTE_NODES+=("$_hv")
done
unset _hv
discover_remote_nodes
if [[ "${#REMOTE_NODES[@]}" -eq 0 ]]; then
echo "No remote nodes configured — nothing to push"
Configurations/master.conf
+9
View File
@@ -802,6 +802,9 @@
# Crash loop threshold
WATCHDOG_CRASH_LIMIT=5 # RestartCount above this = critical crash loop
# Required-container strikes — consecutive down-checks before docker_watchdog.sh attempts a restart
WATCHDOG_REQUIRED_STRIKE_LIMIT=2
# Startup grace period — skip restarts while system is still booting
WATCHDOG_STARTUP_GRACE=600 # seconds after boot before watchdog acts on failures
@@ -891,6 +894,9 @@
# Seconds of warning broadcast to logged-in users before server_reboot.sh reboots.
# Gives users time to save work — 300s = 5 minutes.
REBOOT_SLEEP=300
# Seconds to wait for graceful VM shutdown (ACPI signal via virsh) before libvirt
# stops it anyway — reboot takes priority.
REBOOT_VM_WAIT=30
# ━━━ Mover ━━━
# Seconds to wait before mover_stop.sh sends SIGTERM to the mover process.
@@ -1085,6 +1091,7 @@
LIDARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run
# protects against API returning partial data on a bad day
LIDARR_TRACKED_COUNT_FILE="$DATA_DIR/lidarr_tracked.count"
LIDARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan
LIDARR_EXTENSIONS=("flac" "mp3" "m4a" "wav" "aac" "ogg" "opus" "wma")
LIDARR_PROTECTED_PATTERNS=(
# Metadata
@@ -1153,6 +1160,7 @@
SONARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run
# protects against API returning partial data on a bad day
SONARR_TRACKED_COUNT_FILE="$DATA_DIR/sonarr_tracked.count"
SONARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan
SONARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "ts" "wmv" "mov")
SONARR_PROTECTED_PATTERNS=(
# Subtitles
@@ -1179,6 +1187,7 @@
RADARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run
# protects against API returning partial data on a bad day
RADARR_TRACKED_COUNT_FILE="$DATA_DIR/radarr_tracked.count"
RADARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan
RADARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "wmv" "mov")
RADARR_PROTECTED_PATTERNS=(
# Subtitles
Deployment/master.conf.template
+9
View File
@@ -797,6 +797,9 @@
# Crash loop threshold
WATCHDOG_CRASH_LIMIT=5 # RestartCount above this = critical crash loop
# Required-container strikes — consecutive down-checks before docker_watchdog.sh attempts a restart
WATCHDOG_REQUIRED_STRIKE_LIMIT=2
# Startup grace period — skip restarts while system is still booting
WATCHDOG_STARTUP_GRACE=600 # seconds after boot before watchdog acts on failures
@@ -886,6 +889,9 @@
# Seconds of warning broadcast to logged-in users before server_reboot.sh reboots.
# Gives users time to save work — 300s = 5 minutes.
REBOOT_SLEEP=300
# Seconds to wait for graceful VM shutdown (ACPI signal via virsh) before libvirt
# stops it anyway — reboot takes priority.
REBOOT_VM_WAIT=30
# ━━━ Mover ━━━
# Seconds to wait before mover_stop.sh sends SIGTERM to the mover process.
@@ -1080,6 +1086,7 @@
LIDARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run
# protects against API returning partial data on a bad day
LIDARR_TRACKED_COUNT_FILE="$DATA_DIR/lidarr_tracked.count"
LIDARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan
LIDARR_EXTENSIONS=("flac" "mp3" "m4a" "wav" "aac" "ogg" "opus" "wma")
LIDARR_PROTECTED_PATTERNS=(
# Metadata
@@ -1144,6 +1151,7 @@
SONARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run
# protects against API returning partial data on a bad day
SONARR_TRACKED_COUNT_FILE="$DATA_DIR/sonarr_tracked.count"
SONARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan
SONARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "ts" "wmv" "mov")
SONARR_PROTECTED_PATTERNS=(
# Subtitles
@@ -1168,6 +1176,7 @@
RADARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run
# protects against API returning partial data on a bad day
RADARR_TRACKED_COUNT_FILE="$DATA_DIR/radarr_tracked.count"
RADARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan
RADARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "wmv" "mov")
RADARR_PROTECTED_PATTERNS=(
# Subtitles
Docker_Essentials/docker_container_stop.sh
-2
View File
@@ -186,9 +186,7 @@ FAILED=()
for container in "${RUNNING[@]}"; do
[[ -z "$container" ]] && continue
local c_start
c_start=$(date +%s)
local c_image
c_image=$(docker inspect --format '{{.Config.Image}}' "$container" 2>/dev/null || echo "unknown")
log "━━━ $ICON_CONTAINERS $container ($c_image) ━━━"
Docker_Essentials/docker_daily_restart.sh
+2 -67
View File
@@ -159,71 +159,7 @@ fi
# docker_cmd, verify_running, retry_docker — defined in common.sh
# Builds a dependency-safe restart order from DAILY_RESTART_CONTAINERS.
# Containers that are dependencies of others restart first.
# Returns ordered list in ORDERED_RESTART array.
build_restart_order() {
ORDERED_RESTART=()
local remaining=("${DAILY_RESTART_CONTAINERS[@]}")
local placed=()
# First pass — add dependency containers that appear in our list
for container in "${remaining[@]}"; do
[[ -z "$container" ]] && continue
local is_dependency=false
# Check if this container is a dependency of any other in our list
for dep_string in "${WATCHDOG_DEPENDENCIES[@]}"; do
if [[ "$dep_string" == *"$container"* ]]; then
is_dependency=true
break
fi
done
# Also check associative array format
for dependent in "${!WATCHDOG_DEPENDENCIES[@]}"; do
if [[ "${WATCHDOG_DEPENDENCIES[$dependent]}" == *"$container"* ]]; then
is_dependency=true
break
fi
done
if [[ "$is_dependency" == true ]]; then
# Check not already placed
local already=false
for p in "${placed[@]}"; do [[ "$p" == "$container" ]] && already=true && break; done
if [[ "$already" == false ]]; then
ORDERED_RESTART+=("$container")
placed+=("$container")
fi
fi
done
# Second pass — add remaining containers (dependents and independents)
for container in "${remaining[@]}"; do
[[ -z "$container" ]] && continue
local already=false
for p in "${placed[@]}"; do [[ "$p" == "$container" ]] && already=true && break; done
if [[ "$already" == false ]]; then
ORDERED_RESTART+=("$container")
placed+=("$container")
fi
done
}
# Checks if a container is a dependent of the previously restarted container.
# If so, waits CONTAINER_DELAY before restarting to allow dependency to settle.
# Usage: check_dependency_delay "$container" "$last_restarted"
check_dependency_delay() {
local container="$1"
local last="$2"
[[ -z "$last" ]] && return
local deps="${WATCHDOG_DEPENDENCIES[$container]:-}"
if [[ -n "$deps" ]] && [[ "$deps" == *"$last"* ]]; then
echo " Waiting ${CONTAINER_DELAY}s — $container depends on $last..."
sleep "$CONTAINER_DELAY"
fi
}
# build_restart_order() / check_dependency_delay() — provided by common.sh
# ==============================================================================================
# ━━━ Daily Restart ━━━
@@ -241,8 +177,7 @@ RESTARTED=()
SKIPPED=()
# Build dependency-safe restart order
build_restart_order
log "$ICON_GEAR Restart order: ${ORDERED_RESTART[*]}"
build_restart_order DAILY_RESTART_CONTAINERS
LAST_RESTARTED=""
Docker_Essentials/docker_weekly_restart.sh
+2 -55
View File
@@ -158,58 +158,7 @@ fi
# docker_cmd, retry_docker, verify_running — defined in common.sh
# Builds a dependency-safe restart order from WEEKLY_RESTART_CONTAINERS.
# Containers that are dependencies of others restart first.
build_restart_order() {
ORDERED_RESTART=()
local remaining=("${WEEKLY_RESTART_CONTAINERS[@]}")
local placed=()
# First pass — add dependency containers that appear in our list
for container in "${remaining[@]}"; do
[[ -z "$container" ]] && continue
local is_dependency=false
for dependent in "${!WATCHDOG_DEPENDENCIES[@]}"; do
if [[ "${WATCHDOG_DEPENDENCIES[$dependent]}" == *"$container"* ]]; then
is_dependency=true
break
fi
done
if [[ "$is_dependency" == true ]]; then
local already=false
for p in "${placed[@]}"; do [[ "$p" == "$container" ]] && already=true && break; done
if [[ "$already" == false ]]; then
ORDERED_RESTART+=("$container")
placed+=("$container")
fi
fi
done
# Second pass — add remaining containers (dependents and independents)
for container in "${remaining[@]}"; do
[[ -z "$container" ]] && continue
local already=false
for p in "${placed[@]}"; do [[ "$p" == "$container" ]] && already=true && break; done
if [[ "$already" == false ]]; then
ORDERED_RESTART+=("$container")
placed+=("$container")
fi
done
log "Restart order: ${ORDERED_RESTART[*]}"
}
# Waits CONTAINER_DELAY if this container depends on the last restarted one.
check_dependency_delay() {
local container="$1"
local last="$2"
[[ -z "$last" ]] && return
local deps="${WATCHDOG_DEPENDENCIES[$container]:-}"
if [[ -n "$deps" ]] && [[ "$deps" == *"$last"* ]]; then
log "Waiting ${CONTAINER_DELAY}s — $container depends on $last..."
sleep "$CONTAINER_DELAY"
fi
}
# build_restart_order() / check_dependency_delay() — provided by common.sh
# ==============================================================================================
# ━━━ Weekly Restart ━━━
@@ -226,15 +175,13 @@ RESTARTED=()
SKIPPED=()
# Build dependency-safe restart order
build_restart_order
build_restart_order WEEKLY_RESTART_CONTAINERS
LAST_RESTARTED=""
for container in "${ORDERED_RESTART[@]}"; do
[[ -z "$container" ]] && continue
local c_start
c_start=$(date +%s)
local c_image
c_image=$(docker inspect --format '{{.Config.Image}}' "$container" 2>/dev/null || echo "unknown")
log "━━━ $ICON_CONTAINERS $container ($c_image) ━━━"
Docker_Essentials/downloaders_reset.sh
+6 -4
View File
@@ -602,11 +602,13 @@ if [[ -n "$QBIT_URL" ]] && [[ -n "$QBIT_USERNAME" ]]; then
# Age check — must be old enough
[[ "$AGE_DAYS" -lt "$QBIT_FAILSAFE_MIN_DAYS" ]] && ((SKIPPED++)) && continue
# Ratio check — if configured
# Ratio check — if configured. awk handles the fractional comparison;
# bash's [[ -lt ]] only does integers and would treat e.g. 1.4 and 1.5
# as equal once truncated. Missing/empty ratio defaults to 0 (protected).
if [[ "$QBIT_FAILSAFE_MIN_RATIO" != "0" ]]; then
RATIO_INT="${RATIO%.*}"
MIN_RATIO_INT="${QBIT_FAILSAFE_MIN_RATIO%.*}"
[[ "$RATIO_INT" -lt "$MIN_RATIO_INT" ]] && ((SKIPPED++)) && continue
if (( $(awk "BEGIN {print (${RATIO:-0} < $QBIT_FAILSAFE_MIN_RATIO) ? 1 : 0}") )); then
((SKIPPED++)) && continue
fi
fi
if [[ "$DRY_RUN" == true ]]; then
Fallback/fallback.sh
+8 -1
View File
@@ -1000,12 +1000,19 @@ while [[ "$FALLBACK_RUNNING" == true ]]; do
fi
elif [[ "$INTERNET_UP" == false ]]; then
# Lost internet during fallback — enter DARK
# Lost internet during fallback — enter DARK — same actions as NO_INTERNET
echo ""
echo "━━━ $ICON_NET Entering DARK — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
warn "Lost internet during fallback — entering DARK state"
state_set state "DARK"
local_ddns_stop
# Stop containers configured to stop on internet loss
read -r -a stop_on_no_net <<< "$(get_stop_on_no_net)"
for container in "${stop_on_no_net[@]}"; do
[[ -n "$container" ]] && local_stop "$container"
done
notify "DARK state on $(hostname) — lost internet during fallback" \
"Fallback" "warning"
Kernel/decision_engine.sh
+1
View File
@@ -65,6 +65,7 @@ score_candidate() {
local popularity_score="${2:-0}"
local recency_score="${3:-0}"
local quality_score="${4:-0}"
local TOTAL_SCORE
TOTAL_SCORE=$(( \
user_score + \
Monitors/cert_monitor.sh
+12 -31
View File
@@ -159,43 +159,24 @@ fi
check_cert() {
local domain="$1"
local port="${2:-443}"
_CERT_DAYS=""
_CERT_EXPIRY=""
local expiry_str
expiry_str=$(echo | timeout "$CERT_TIMEOUT" openssl s_client \
-connect "${domain}:${port}" \
-servername "$domain" \
2>/dev/null | openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2)
if [[ -z "$expiry_str" ]]; then
error "$ICON_CERT $domain — could not retrieve certificate (unreachable or no TLS)"
if ! check_cert_expiry "$domain" "$port" "$CERT_TIMEOUT"; then
if [[ -n "$_CERT_EXPIRY_RAW" ]]; then
error "$ICON_CERT $domain — could not parse expiry date: $_CERT_EXPIRY_RAW"
else
error "$ICON_CERT $domain — could not retrieve certificate (unreachable or no TLS)"
fi
return 3
fi
local expiry_epoch
expiry_epoch=$(date -d "$expiry_str" +%s 2>/dev/null)
if [[ -z "$expiry_epoch" ]]; then
error "$ICON_CERT $domain — could not parse expiry date: $expiry_str"
return 3
fi
local now days_remaining expiry_display
now=$(date +%s)
days_remaining=$(( (expiry_epoch - now) / 86400 ))
expiry_display=$(date -d "$expiry_str" '+%Y-%m-%d' 2>/dev/null)
_CERT_DAYS=$days_remaining
_CERT_EXPIRY=$expiry_display
if [[ "$days_remaining" -le "$CERT_CRIT_DAYS" ]]; then
error "$ICON_CERT $domain — CRITICAL: ${days_remaining} days remaining (expires $expiry_display)"
if [[ "$_CERT_DAYS" -le "$CERT_CRIT_DAYS" ]]; then
error "$ICON_CERT $domain — CRITICAL: ${_CERT_DAYS} days remaining (expires $_CERT_EXPIRY)"
return 2
elif [[ "$days_remaining" -le "$CERT_WARN_DAYS" ]]; then
warn "$ICON_CERT $domain — WARNING: ${days_remaining} days remaining (expires $expiry_display)"
elif [[ "$_CERT_DAYS" -le "$CERT_WARN_DAYS" ]]; then
warn "$ICON_CERT $domain — WARNING: ${_CERT_DAYS} days remaining (expires $_CERT_EXPIRY)"
return 1
else
log "$ICON_CERT $domain — OK: ${days_remaining} days remaining (expires $expiry_display)"
log "$ICON_CERT $domain — OK: ${_CERT_DAYS} days remaining (expires $_CERT_EXPIRY)"
return 0
fi
}
@@ -285,7 +266,7 @@ fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
# ── Write JSON status cache ───────────────────────────────────────────────────
_CERT_CACHE_FILE="$SCRIPTS_DIR/State_Files/cert_status.json"
_CERT_CACHE_FILE="$STATE_DIR/cert_status.json"
{
printf '{"checked_at":%d,"host":"%s","warn_days":%d,"crit_days":%d,"dry_run":%s,"domains":[\n' \
"$(date +%s)" "$MY_ID" "$CERT_WARN_DAYS" "$CERT_CRIT_DAYS" \
Monitors/emby_session_report.sh
+1 -1
View File
@@ -316,7 +316,7 @@ echo ""
echo "━━━ $ICON_RAM Transcode Status ━━━"
if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
RAMDISK_USED_KB=$(df "$RAMDISK_PATH" --output=used 2>/dev/null | tail -1 | tr -d ' ')
RAMDISK_USED_GB=$(awk "BEGIN {printf \"%.2f\", ${RAMDISK_USED_KB:-0} / 1048576}")
RAMDISK_USED_GB=$(kb_to_gb "$RAMDISK_USED_KB")
SYMLINK=$(readlink "$TRANSCODE_LINK" 2>/dev/null || echo "unknown")
echo " $ICON_RAM Ramdisk usage: ${RAMDISK_USED_GB}GB / ${RAMDISK_SIZE:-8G}"
echo " $ICON_LINK Symlink target: $SYMLINK"
Monitors/smart_health.sh
+3 -23
View File
@@ -156,11 +156,7 @@ if [[ "$SHOW_STATUS" == true ]]; then
for drive in /dev/sd? /dev/nvme?; do
[[ ! -e "$drive" ]] && continue
drive_name=$(basename "$drive")
ignored=false
for ignore in "${SMART_IGNORE_DRIVES[@]}"; do
[[ "$drive_name" == "$ignore" ]] && ignored=true && break
done
if [[ "$ignored" == true ]]; then
if is_in_list "$drive_name" "${SMART_IGNORE_DRIVES[@]}"; then
echo " $ICON_WARN $drive — ignored"
else
echo " $ICON_SMART $drive — would check"
@@ -201,18 +197,7 @@ get_drive_temp() {
echo "${temp:-}"
}
# Detect if a drive is SSD/NVMe (rotational=0)
is_ssd() {
local drive="$1"
local dev_name
dev_name=$(basename "$drive" | sed 's/nvme[0-9]/nvme0/')
local rotational="/sys/block/$(basename "$drive")/queue/rotational"
[[ -f "$rotational" ]] && [[ "$(cat "$rotational" 2>/dev/null)" == "0" ]] && return 0
# NVMe is always SSD
[[ "$drive" == *nvme* ]] && return 0
return 1
}
# is_ssd() — provided by common.sh
# ==============================================================================================
# ━━━ SMART Health Check ━━━
# ==============================================================================================
@@ -232,12 +217,7 @@ for drive in /dev/sd? /dev/nvme?; do
drive_name=$(basename "$drive")
# Check ignore list
ignored=false
for ignore in "${SMART_IGNORE_DRIVES[@]}"; do
[[ "$drive_name" == "$ignore" ]] && ignored=true && break
done
if [[ "$ignored" == true ]]; then
if is_in_list "$drive_name" "${SMART_IGNORE_DRIVES[@]}"; then
log "$drive_name — ignored (SMART_IGNORE_DRIVES)"
DRIVES_SKIP+=("$drive_name")
continue
Monitors/weekly_health_digest.sh
+4 -8
View File
@@ -270,7 +270,7 @@ fi
# ── Transcode Ramdisk ─────────────────────────────────────────────────────────────────────────
if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
RAMDISK_USED_KB=$(df "$RAMDISK_PATH" --output=used 2>/dev/null | tail -1 | tr -d ' ')
RAMDISK_USED_GB=$(awk "BEGIN {printf \"%.2f\", ${RAMDISK_USED_KB:-0} / 1048576}")
RAMDISK_USED_GB=$(kb_to_gb "$RAMDISK_USED_KB")
SYMLINK_TARGET=$(readlink "$TRANSCODE_LINK" 2>/dev/null || echo "unknown")
DIGEST_LINES+=("$ICON_RAM Transcodes: ${RAMDISK_USED_GB}GB used → $SYMLINK_TARGET")
@@ -309,7 +309,7 @@ if [[ -f "${BANDWIDTH_LOG:-}" ]] && [[ -s "$BANDWIDTH_LOG" ]]; then
YESTERDAY=$(date -d "yesterday" '+%Y-%m-%d')
YESTERDAY_BYTES=$(awk -F'|' -v d="$YESTERDAY" '$1==d{sum+=$6} END{print sum+0}' \
"$BANDWIDTH_LOG")
YESTERDAY_GB=$(awk "BEGIN {printf \"%.2f\", ${YESTERDAY_BYTES:-0} / 1073741824}")
YESTERDAY_GB=$(bytes_to_gb "$YESTERDAY_BYTES")
YESTERDAY_LARGE=$(awk -F'|' -v d="$YESTERDAY" '$1==d && $7=="LARGE"' \
"$BANDWIDTH_LOG" | wc -l)
@@ -329,12 +329,8 @@ if [[ ${#CERT_MONITOR_DOMAINS[@]} -gt 0 ]] && command -v openssl >/dev/null 2>&1
CERT_ISSUES=()
for domain in "${CERT_MONITOR_DOMAINS[@]}"; do
[[ -z "$domain" ]] && continue
expiry_str=$(echo | timeout "${CERT_TIMEOUT:-10}" openssl s_client \
-connect "${domain}:443" -servername "$domain" \
2>/dev/null | openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2)
if [[ -n "$expiry_str" ]]; then
expiry_epoch=$(date -d "$expiry_str" +%s 2>/dev/null)
days_remaining=$(( (expiry_epoch - $(date +%s)) / 86400 ))
if check_cert_expiry "$domain" 443 "${CERT_TIMEOUT:-10}"; then
days_remaining="$_CERT_DAYS"
if [[ "$days_remaining" -le "${CERT_CRIT_DAYS:-7}" ]]; then
CERT_ISSUES+=("$domain: ${days_remaining}d CRITICAL")
SHOULD_SEND=true
Monitors/zfs_memory_snapshot.sh
+5 -5
View File
@@ -261,9 +261,9 @@ else
ARC_SIZE=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats)
ARC_META_USED=$(awk '/^arc_meta_used / {print $3}' /proc/spl/kstat/zfs/arcstats)
ARC_MAX_GB=$(awk "BEGIN {printf \"%.1f\", $ARC_MAX / 1073741824}")
ARC_CUR_GB=$(awk "BEGIN {printf \"%.1f\", $ARC_SIZE / 1073741824}")
ARC_META_GB=$(awk "BEGIN {printf \"%.1f\", $ARC_META_USED / 1073741824}")
ARC_MAX_GB=$(bytes_to_gb "$ARC_MAX" 1)
ARC_CUR_GB=$(bytes_to_gb "$ARC_SIZE" 1)
ARC_META_GB=$(bytes_to_gb "$ARC_META_USED" 1)
ARC_PCT=$(awk "BEGIN {printf \"%.1f\", $ARC_SIZE * 100 / $ARC_MAX}")
ARC_PCT_INT=$(printf "%.0f" "$ARC_PCT")
@@ -287,8 +287,8 @@ else
META_MISSES=$(awk '/^demand_metadata_misses / {print $3}' \
/proc/spl/kstat/zfs/arcstats 2>/dev/null || echo 0)
MRU_GB=$(awk "BEGIN {printf \"%.2f\", $META_MRU_GHOST / 1073741824}")
MFU_GB=$(awk "BEGIN {printf \"%.2f\", $META_MFU_GHOST / 1073741824}")
MRU_GB=$(bytes_to_gb "$META_MRU_GHOST")
MFU_GB=$(bytes_to_gb "$META_MFU_GHOST")
echo " $ICON_ZFS MRU Ghost: ${MRU_GB}GB"
echo " $ICON_ZFS MFU Ghost: ${MFU_GB}GB"
Orchestrators/transcode_management.sh
+12 -36
View File
@@ -88,8 +88,9 @@
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
source "$SCRIPT_DIR/../load_config.sh"
source "$ECOSYSTEM_ROOT/load_config.sh"
parse_args "$@"
@@ -158,17 +159,6 @@ if [[ "$SHOW_STATUS" == true ]]; then
exit 0
fi
# ==============================================================================================
# ━━━ Validate Child Scripts ━━━
# ==============================================================================================
for _entry in "${TRANSCODE_MANAGEMENT_SCRIPTS[@]}"; do
_script_path="$SCRIPT_DIR/../$_entry"
if [[ ! -f "$_script_path" ]]; then
error "$(basename "$_entry") not found: $_script_path"
exit 1
fi
done
# ==============================================================================================
# ━━━ Pre-run State Snapshot ━━━
# ==============================================================================================
@@ -188,36 +178,22 @@ fi
# ==============================================================================================
# transcode_manager.sh writes to TRANSCODE_DAILY_LOG after each run — no flag here
# suppresses that; it owns the log write for this cycle regardless of position ✅
DRY_FLAG=""
[[ "$DRY_RUN" == true ]] && DRY_FLAG="--dry-run"
WORST_EXIT=0
PASS_COUNT=0
FAIL_NAMES=()
JOB_PASS=()
JOB_FAIL=()
for _entry in "${TRANSCODE_MANAGEMENT_SCRIPTS[@]}"; do
_script_path="$SCRIPT_DIR/../$_entry"
_script_name=$(basename "$_entry" .sh)
_start=$(date +%s)
bash "$_script_path" $DRY_FLAG
_exit=$?
log "$_script_name: $(format_duration $(( $(date +%s) - _start ))) (exit $_exit)"
if [[ "$_exit" -ne 0 ]]; then
WORST_EXIT=1
FAIL_NAMES+=("$_script_name")
else
PASS_COUNT=$(( PASS_COUNT + 1 ))
fi
[[ -z "$_entry" ]] && continue
run_orch_child "$_entry"
done
# ==============================================================================================
# ━━━ Summary — minimal one-liner by default (7-min cadence — keep it quiet when healthy) ━━━
# ==============================================================================================
if [[ "$WORST_EXIT" -eq 0 ]]; then
echo "$ICON_SUCCESS Transcode cycle — $PASS_COUNT/${#TRANSCODE_MANAGEMENT_SCRIPTS[@]} passed"
if [[ "${#JOB_FAIL[@]}" -eq 0 ]]; then
echo "$ICON_SUCCESS Transcode cycle — ${#JOB_PASS[@]}/${#TRANSCODE_MANAGEMENT_SCRIPTS[@]} passed"
else
error "Transcode cycle — failed: ${FAIL_NAMES[*]}"
error "Transcode cycle — failed: ${JOB_FAIL[*]}"
if [[ "$DRY_RUN" != true ]]; then
notify "Transcode management failure on $(hostname) ($MY_ID) — ${FAIL_NAMES[*]}" \
notify "Transcode management failure on $(hostname) ($MY_ID) — ${JOB_FAIL[*]}" \
"Transcode Management" "warning"
fi
fi
@@ -225,5 +201,5 @@ fi
# ==============================================================================================
# ━━━ Exit ━━━
# ==============================================================================================
# Return worst exit code — caller knows if any script failed
exit "$WORST_EXIT"
[[ "${#JOB_FAIL[@]}" -gt 0 ]] && exit 1
exit 0
Orchestrators/watchdog_orchestrator.sh
+10 -38
View File
@@ -177,40 +177,12 @@ log "Startup grace: past — uptime $(format_duration $UPTIME_SECONDS)"
# ━━━ Run Watchdog Cycle ━━━
# ==============================================================================================
CYCLE_START=$(date +%s)
PASS=()
FAIL=()
run_watchdog() {
local name="$1" script="$2"
if [[ ! -f "$script" ]]; then
error "$name — not found: $script"
FAIL+=("$name:missing")
return 1
fi
[[ ! -x "$script" ]] && chmod +x "$script"
local extra_args=()
[[ "$DRY_RUN" == true ]] && extra_args+=("--dry-run")
[[ "$ENABLE_LOGGING" == true ]] && extra_args+=("--log")
local _ws
_ws=$(date +%s)
log "$ICON_START $name"
if bash "$script" "${extra_args[@]}"; then
log "$ICON_DONE $name — done in $(format_duration $(( $(date +%s) - _ws )))"
PASS+=("$name")
return 0
else
error "$name — non-zero exit ($(format_duration $(( $(date +%s) - _ws ))))"
FAIL+=("$name")
return 1
fi
}
JOB_PASS=()
JOB_FAIL=()
for _entry in "${WATCHDOG_ORCHESTRATOR_SCRIPTS[@]}"; do
run_watchdog "$(_watchdog_display_name "$_entry")" "$ECOSYSTEM_ROOT/$_entry"
[[ -z "$_entry" ]] && continue
run_orch_child "$_entry"
done
CYCLE_END=$(date +%s)
@@ -236,19 +208,19 @@ fi
# ==============================================================================================
# ━━━ Summary — minimal one-liner by default, full breakdown on failure or --log ━━━
# ==============================================================================================
if [[ "${#FAIL[@]}" -gt 0 || "$ENABLE_LOGGING" == true ]]; then
if [[ "${#JOB_FAIL[@]}" -gt 0 || "$ENABLE_LOGGING" == true ]]; then
echo ""
echo "━━━━━ $ICON_SUMMARY WATCHDOG CYCLE — $MY_ID — $(date '+%H:%M:%S') ━━━━━"
for p in "${PASS[@]}"; do log " $ICON_DONE $p"; done
for f in "${FAIL[@]}"; do error " $ICON_ERROR $f"; done
for p in "${JOB_PASS[@]}"; do log " $ICON_DONE $p"; done
for f in "${JOB_FAIL[@]}"; do error " $ICON_ERROR $f"; done
echo "$ICON_TIME Duration: $(format_duration $DURATION)"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
else
echo "$ICON_DONE Watchdog cycle — ${#PASS[@]}/${#WATCHDOG_ORCHESTRATOR_SCRIPTS[@]} passed ($(format_duration $DURATION))"
echo "$ICON_DONE Watchdog cycle — ${#JOB_PASS[@]}/${#WATCHDOG_ORCHESTRATOR_SCRIPTS[@]} passed ($(format_duration $DURATION))"
fi
if [[ "${#FAIL[@]}" -gt 0 ]]; then
notify "Watchdog cycle failure on $(hostname) ($MY_ID) — ${FAIL[*]}" \
if [[ "${#JOB_FAIL[@]}" -gt 0 ]]; then
notify "Watchdog cycle failure on $(hostname) ($MY_ID) — ${JOB_FAIL[*]}" \
"Watchdog Orchestrator" "warning"
exit 1
fi
Partnership/partnership_manager.sh
+31 -66
View File
@@ -249,22 +249,7 @@ fi
detect_hosts
# ── Derive owner and mirror from PARTNERSHIP_OWNER_HOST ───────────────────────────────────────
OWNER_ID="${PARTNERSHIP_OWNER_HOST:-HOST1}" # e.g. "HOST1"
MIRROR_ID=$( [[ "$OWNER_ID" == "HOST1" ]] && echo "HOST2" || echo "HOST1" )
OWNER="${!OWNER_ID}" # hostname string
MIRROR="${!MIRROR_ID}"
# SSH_KEY (set by detect_hosts) is this server's own private key.
# The remote accepts it because this server's PUBLIC key was installed there via ssh_setup.sh.
# With sparse checkout, each server only has its own host{N}.conf — the other server's
# key path is never available here. Use SSH_KEY for all outbound SSH regardless of mode.
MIRROR_SSH_KEY="$SSH_KEY"
OWNER_SSH_KEY="$SSH_KEY"
AM_OWNER=false
AM_MIRROR=false
[[ "$MY_ID" == "$OWNER_ID" ]] && AM_OWNER=true
[[ "$MY_ID" == "$MIRROR_ID" ]] && AM_MIRROR=true
partnership_resolve_roles
# State files
LOCAL_STATE_FILE="${STATE_DIR}/partnership_${LOCAL_SERVER_NAME}.db"
@@ -355,7 +340,7 @@ push_state_to_remote() {
warn "DRY RUN — would push state file to remote"
return 0
fi
timeout "$SSH_TIMEOUT" scp -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" \
timeout "$SSH_TIMEOUT" scp -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes \
"$local_file" "root@${remote_ip}:${local_file}" 2>/dev/null && \
echo "State file pushed to remote ✅" || \
warn "Could not push state file to remote — will propagate on next sync"
@@ -364,7 +349,7 @@ push_state_to_remote() {
read_remote_state() {
local remote_ip="$1" ssh_key="$2" remote_file="$3"
timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
-o ConnectTimeout="$SSH_TIMEOUT" root@"$remote_ip" \
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$remote_ip" \
"cat '$remote_file' 2>/dev/null" 2>/dev/null
}
@@ -524,34 +509,9 @@ gather_partner_fallback_containers() {
done
}
# Read a scalar var from the mirror's own config via SSH.
# Sources load_config.sh + detect_hosts() on the remote so HOST* aliasing works.
read_remote_conf_var() {
local mirror_ip="$1" var_name="$2"
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \
"source '$SCRIPT_DIR/../load_config.sh' 2>/dev/null
detect_hosts 2>/dev/null
printf '%s' \"\${${var_name}:-}\"" 2>/dev/null
}
# read_remote_conf_var() / read_remote_conf_array() — provided by common.sh
# Read an array var from the mirror's own config via SSH — one element per line.
read_remote_conf_array() {
local mirror_ip="$1" var_name="$2"
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \
"source '$SCRIPT_DIR/../load_config.sh' 2>/dev/null
detect_hosts 2>/dev/null
printf '%s\n' \"\${${var_name}[@]:-}\"" 2>/dev/null
}
# Returns just the short name portion: "unRAID-Gmer4Lfe" → "Gmer4Lfe"
derive_short_name() {
local hostname="$1"
local short="${hostname,,}"
[[ "$short" == unraid-* ]] && short="${short:7}"
echo "${short^}"
}
# derive_short_name() — provided by common.sh
# Start this server's own parked containers after partnership ends.
start_own_stack() {
@@ -644,7 +604,7 @@ cleanup_owner_containers_on_mirror() {
local container_list
container_list=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
-o ConnectTimeout="$SSH_TIMEOUT" root@"$mirror_ip" \
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \
"docker ps -a --format '{{.Names}}' 2>/dev/null | grep -i -- '-${owner_short}$'" 2>/dev/null)
if [[ -z "$container_list" ]]; then
@@ -658,12 +618,12 @@ cleanup_owner_containers_on_mirror() {
# Collect appdata paths before removal
local appdata_paths
appdata_paths=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
-o ConnectTimeout="$SSH_TIMEOUT" root@"$mirror_ip" \
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \
"docker inspect --format '{{range .HostConfig.Binds}}{{println .}}{{end}}' '$container' 2>/dev/null \
| awk -F: '{print \$1}' | grep '^/mnt/.*/appdata'" 2>/dev/null)
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
-o ConnectTimeout="$SSH_TIMEOUT" root@"$mirror_ip" \
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \
"docker stop '$container' >/dev/null 2>&1
docker rm '$container' >/dev/null 2>&1 && echo removed" 2>/dev/null | \
grep -q removed && \
@@ -674,7 +634,7 @@ cleanup_owner_containers_on_mirror() {
while IFS= read -r path; do
[[ -z "$path" ]] && continue
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
-o ConnectTimeout="$SSH_TIMEOUT" root@"$mirror_ip" \
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \
"rm -rf '$path' && echo removed" 2>/dev/null | grep -q removed && \
echo " Appdata removed on $MIRROR: $path ✅" || \
warn " Failed to remove appdata on $MIRROR: $path"
@@ -707,7 +667,7 @@ start_mirror_own_stack() {
continue
fi
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
-o ConnectTimeout="$SSH_TIMEOUT" root@"$mirror_ip" \
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \
"docker start '$container' >/dev/null 2>&1 && echo started" 2>/dev/null | \
grep -q started && \
echo "$container started on $MIRROR ✅" || \
@@ -881,7 +841,7 @@ check_both_healthy() {
[[ -z "$mirror_ip" ]] && { error "Cannot resolve $MIRROR Tailscale IP"; return 1; }
timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" \
-o ConnectTimeout="$SSH_TIMEOUT" root@"$mirror_ip" \
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \
"mountpoint -q '$REMOTE_STORAGE_PATH' && timeout 10 docker ps" >/dev/null 2>&1 || {
error "Mirror $MIRROR not healthy"
return 1
@@ -913,7 +873,8 @@ do_final_sync() {
warn "Final sync complete — mirror has current state ✅"
}
# Safe master.conf modification with error handling
# Safe master.conf modification with error handling — appends the key if not already present,
# since sed -i returns 0 whether or not it matched anything.
update_master_conf() {
local key="$1" value="$2"
local conf="$CONF_DIR/master.conf"
@@ -921,12 +882,22 @@ update_master_conf() {
error "master.conf not found at $conf"
return 1
fi
if sed -i "s|^[[:space:]]*${key}=.*| ${key}=${value}|" "$conf" 2>/dev/null; then
echo "master.conf updated: ${key}=${value}"
return 0
if grep -q "^[[:space:]]*${key}=" "$conf" 2>/dev/null; then
if sed -i "s|^[[:space:]]*${key}=.*| ${key}=${value}|" "$conf" 2>/dev/null; then
echo "master.conf updated: ${key}=${value}"
return 0
else
error "Failed to update master.conf: ${key}=${value}"
return 1
fi
else
error "Failed to update master.conf: ${key}=${value}"
return 1
if echo " ${key}=${value}" >> "$conf" 2>/dev/null; then
echo "master.conf updated: ${key}=${value} (appended)"
return 0
else
error "Failed to append to master.conf: ${key}=${value}"
return 1
fi
fi
}
@@ -1162,13 +1133,7 @@ if [[ "$MODE" == "onboard" ]]; then
echo ""
echo "Enabling partnership in master.conf..."
if [[ "$DRY_RUN" == false ]]; then
_master_conf="$CONF_DIR/master.conf"
if grep -q "^[[:space:]]*PARTNERSHIP_ENABLED=" "$_master_conf" 2>/dev/null; then
sed -i "s|^[[:space:]]*PARTNERSHIP_ENABLED=.*|PARTNERSHIP_ENABLED=true|" "$_master_conf"
else
echo "PARTNERSHIP_ENABLED=true" >> "$_master_conf"
fi
echo "PARTNERSHIP_ENABLED=true in master.conf ✅"
update_master_conf "PARTNERSHIP_ENABLED" "true"
platform_push_conf | while IFS= read -r line; do log "$line"; done
else
warn "DRY RUN — would set PARTNERSHIP_ENABLED=true in master.conf and push"
@@ -1435,9 +1400,9 @@ if false; then
# Update remote master.conf
timeout "$SSH_TIMEOUT" ssh -i "$NEW_MIRROR_SSH_KEY" \
-o ConnectTimeout="$SSH_TIMEOUT" root@"$NEW_MIRROR_IP" \
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$NEW_MIRROR_IP" \
"sed -i 's|^[[:space:]]*PARTNERSHIP_OWNER_HOST=.*| PARTNERSHIP_OWNER_HOST=\"$NEW_OWNER_ID\"|' \
'$SCRIPT_DIR/../master.conf'" 2>/dev/null && \
'$SCRIPT_DIR/../Configurations/master.conf'" 2>/dev/null && \
echo "Remote master.conf updated ✅" || \
error "Failed to update remote master.conf — update manually"
else
Partnership/partnership_offboard.sh
+1 -11
View File
@@ -108,17 +108,7 @@ fi
detect_hosts
OWNER_ID="${PARTNERSHIP_OWNER_HOST:-HOST1}"
MIRROR_ID=$( [[ "$OWNER_ID" == "HOST1" ]] && echo "HOST2" || echo "HOST1" )
OWNER="${!OWNER_ID}"
MIRROR="${!MIRROR_ID}"
MIRROR_SSH_KEY="$SSH_KEY"
OWNER_SSH_KEY="$SSH_KEY"
AM_OWNER=false
AM_MIRROR=false
[[ "$MY_ID" == "$OWNER_ID" ]] && AM_OWNER=true
[[ "$MY_ID" == "$MIRROR_ID" ]] && AM_MIRROR=true
partnership_resolve_roles
LOCAL_STATE_FILE="${STATE_DIR}/partnership_${LOCAL_SERVER_NAME}.db"
REMOTE_STATE_FILE="${STATE_DIR}/partnership_${REMOTE_SERVER_NAME}.db"
Partnership/partnership_onboard.sh
+8 -37
View File
@@ -223,20 +223,7 @@ fi
detect_hosts
OWNER_ID="${PARTNERSHIP_OWNER_HOST:-HOST1}"
MIRROR_ID=$( [[ "$OWNER_ID" == "HOST1" ]] && echo "HOST2" || echo "HOST1" )
OWNER="${!OWNER_ID}"
MIRROR="${!MIRROR_ID}"
# SSH_KEY (set by detect_hosts) is this server's own private key.
# The remote accepts it because this server's PUBLIC key was installed there via ssh_setup.sh.
# HOST{N}_SSH_KEY lives in host{N}.conf — with sparse checkout, the other server's
# conf is never present here. Always use SSH_KEY (local private key) for outbound SSH.
MIRROR_SSH_KEY="$SSH_KEY"
AM_OWNER=false
AM_MIRROR=false
[[ "$MY_ID" == "$OWNER_ID" ]] && AM_OWNER=true
[[ "$MY_ID" == "$MIRROR_ID" ]] && AM_MIRROR=true
partnership_resolve_roles
EXTRA_FLAGS=()
[[ "$DRY_RUN" == true ]] && EXTRA_FLAGS+=("--dry-run")
@@ -250,11 +237,7 @@ write_onboard_phase() {
local key="${target_id}_PHASE${phase}_DONE"
local state_file="$(platform_setup_db_path)"
[[ "$DRY_RUN" == true ]] && { warn "DRY RUN — would write ${key}=true"; return 0; }
if grep -q "^${key}=" "$state_file" 2>/dev/null; then
sed -i "s|^${key}=.*|${key}=true|" "$state_file"
else
echo "${key}=true" >> "$state_file"
fi
set_state_var "$state_file" "$key" "true"
platform_push_setup_state
}
@@ -282,13 +265,7 @@ stop_mirror_stack() {
local config_var="$1" label="$2"
local -a to_stop=()
mapfile -t to_stop < <(
timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" \
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$MIRROR_IP" \
"source '$SCRIPTS_ROOT/load_config.sh' 2>/dev/null
detect_hosts 2>/dev/null
printf '%s\n' \"\${${config_var}[@]:-}\"" 2>/dev/null | grep -v '^$'
)
mapfile -t to_stop < <(read_remote_conf_array "$MIRROR_IP" "$config_var" | grep -v '^$')
if [[ ${#to_stop[@]} -eq 0 ]]; then
log "No $label containers to stop on $MIRROR — skipping"
@@ -302,7 +279,7 @@ stop_mirror_stack() {
continue
fi
timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" \
-o ConnectTimeout="$SSH_TIMEOUT" root@"$MIRROR_IP" \
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$MIRROR_IP" \
"docker stop '$container' 2>/dev/null
docker rm '$container' 2>/dev/null && echo removed" 2>/dev/null | \
grep -q removed && \
@@ -338,11 +315,7 @@ if [[ "$AM_MIRROR" == true ]]; then
if [[ -n "$OWNER_IP" ]]; then
# Read OWNER's SCRIPTS_DIR via platform probe command — don't assume same path as mirror
_probe_cmd=$(platform_scripts_dir_probe_cmd)
OWNER_SCRIPTS_DIR=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$OWNER_IP" \
"$_probe_cmd" 2>/dev/null | tr -d '[:space:]')
OWNER_SCRIPTS_DIR="${OWNER_SCRIPTS_DIR:-$SCRIPTS_DIR}"
OWNER_SCRIPTS_DIR=$(resolve_remote_scripts_dir "$OWNER_IP")
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would SSH to $OWNER ($OWNER_IP) and trigger Phase 2"
@@ -444,11 +417,9 @@ elif [[ "$PHASE1_ONLY" == true ]]; then
echo " Then click 'Push Conf' in the Partnership tab."
# Write key-ready flag so UI can show the manual-install state
[[ "$DRY_RUN" == false ]] && {
local kflag="${MIRROR_ID}_KEY_READY"
local _setup_f="$(platform_setup_db_path)"
grep -q "^${kflag}=" "$_setup_f" 2>/dev/null \
&& sed -i "s|^${kflag}=.*|${kflag}=true|" "$_setup_f" \
|| echo "${kflag}=true" >> "$_setup_f"
kflag="${MIRROR_ID}_KEY_READY"
_setup_f="$(platform_setup_db_path)"
set_state_var "$_setup_f" "$kflag" "true"
}
fi
STEP_SSH_OK=false
Partnership/partnership_transfer.sh
+3 -16
View File
@@ -128,17 +128,7 @@ fi
detect_hosts
OWNER_ID="${PARTNERSHIP_OWNER_HOST:-HOST1}"
MIRROR_ID=$( [[ "$OWNER_ID" == "HOST1" ]] && echo "HOST2" || echo "HOST1" )
OWNER="${!OWNER_ID}"
MIRROR="${!MIRROR_ID}"
MIRROR_SSH_KEY="$SSH_KEY"
OWNER_SSH_KEY="$SSH_KEY"
AM_OWNER=false
AM_MIRROR=false
[[ "$MY_ID" == "$OWNER_ID" ]] && AM_OWNER=true
[[ "$MY_ID" == "$MIRROR_ID" ]] && AM_MIRROR=true
partnership_resolve_roles
LOCAL_STATE_FILE="${STATE_DIR}/partnership_${LOCAL_SERVER_NAME}.db"
REMOTE_STATE_FILE="${STATE_DIR}/partnership_${REMOTE_SERVER_NAME}.db"
@@ -286,14 +276,11 @@ if [[ "$DRY_RUN" == false ]]; then
# Push updated master.conf to new owner so both servers agree immediately.
# master.conf is shared — host-specific credentials live in host*.conf.
_probe_cmd=$(platform_scripts_dir_probe_cmd)
_REMOTE_SD=$(ssh -i "$SSH_KEY" -o ConnectTimeout=5 -o StrictHostKeyChecking=no \
"root@${MIRROR_IP}" "$_probe_cmd" \
2>/dev/null | tr -d '[:space:]')
_REMOTE_SD="${_REMOTE_SD:-$SCRIPTS_DIR}"
_REMOTE_SD=$(resolve_remote_scripts_dir "$MIRROR_IP" "$SSH_KEY" "no")
scp -i "$SSH_KEY" \
-o ConnectTimeout="$SSH_TIMEOUT" \
-o StrictHostKeyChecking=no \
-o BatchMode=yes \
"$SCRIPTS_ROOT/Configurations/master.conf" \
"root@${MIRROR_IP}:${_REMOTE_SD}/Configurations/master.conf" 2>/dev/null && \
echo "master.conf pushed to $NEW_OWNER ✅" || \
Partnership/share_setup.sh
+1 -1
View File
@@ -52,7 +52,7 @@ MIRROR_IP=$(resolve_tailscale_ip "$MIRROR")
[[ -z "$MIRROR_IP" ]] && { error "Cannot resolve $MIRROR Tailscale IP — is Tailscale running?"; exit 1; }
_ssh() { ssh -i "$SSH_KEY" -o ConnectTimeout=10 -o BatchMode=yes -o StrictHostKeyChecking=no root@"$MIRROR_IP" "$@"; }
_scp() { scp -i "$SSH_KEY" -o ConnectTimeout=10 -o StrictHostKeyChecking=no "$@"; }
_scp() { scp -i "$SSH_KEY" -o ConnectTimeout=10 -o BatchMode=yes -o StrictHostKeyChecking=no "$@"; }
# ── Detect remote appdata pool ────────────────────────────────────────────────────────────────
Partnership/ssh_setup.sh
+4 -3
View File
@@ -74,9 +74,10 @@ acquire_lock
detect_hosts
# ── Derive key name from hostname ─────────────────────────────────────────────────────────────
# Strip unraid- prefix (case-insensitive) if present → gmer4lfe_rsync_automation
SHORT_NAME="${LOCAL_SERVER_NAME,,}"
[[ "${SHORT_NAME}" == unraid-* ]] && SHORT_NAME="${SHORT_NAME:7}"
# derive_short_name() title-cases the result (for display elsewhere) — lowercase it here,
# same as before, since this feeds a filename → gmer4lfe_rsync_automation
SHORT_NAME="$(derive_short_name "$LOCAL_SERVER_NAME")"
SHORT_NAME="${SHORT_NAME,,}"
SSH_KEY_NAME="${SHORT_NAME}_rsync_automation"
SSH_KEY_PATH="/root/.ssh/${SSH_KEY_NAME}"
SSH_PUB_PATH="${SSH_KEY_PATH}.pub"
Plugin/unraid/Partnership/containers.sh
+7 -7
View File
@@ -220,7 +220,7 @@ deploy_container_from_xml() {
log "Deploying $name..."
if [[ "$DRY_RUN" == false ]]; then
timeout "$SSH_TIMEOUT" scp -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" \
timeout "$SSH_TIMEOUT" scp -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes \
"$xml_file" "root@${remote_ip}:${TEMPLATES_DIR}/${xml_name}" 2>/dev/null || {
rm -f "$_gpu_tmp"
warn " SCP failed for $xml_name — skipping $name"
@@ -291,9 +291,9 @@ deploy_container_from_xml() {
local remote_script="/tmp/deploy_${name//[^a-zA-Z0-9_]/_}.sh"
if timeout "$SSH_TIMEOUT" scp -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" \
if timeout "$SSH_TIMEOUT" scp -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes \
"$tmp_script" "root@${remote_ip}:${remote_script}" 2>/dev/null && \
timeout 120 ssh -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" root@"$remote_ip" \
timeout 120 ssh -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$remote_ip" \
"bash '$remote_script' 2>&1; rc=\$?; rm -f '$remote_script'; exit \$rc" 2>/dev/null | \
grep -q "deployed:${name}"; then
echo " $name deployed ✅"
@@ -386,7 +386,7 @@ cleanup_deployed_stack_on_remote() {
| awk -F: '{print \$1}' | grep '^/mnt/.*/appdata'" 2>/dev/null)
timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
-o ConnectTimeout="$SSH_TIMEOUT" root@"$remote_ip" \
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$remote_ip" \
"docker stop '$cname' >/dev/null 2>&1
docker rm '$cname' >/dev/null 2>&1 && echo removed" 2>/dev/null | \
grep -q removed && \
@@ -396,7 +396,7 @@ cleanup_deployed_stack_on_remote() {
while IFS= read -r path; do
[[ -z "$path" ]] && continue
timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
-o ConnectTimeout="$SSH_TIMEOUT" root@"$remote_ip" \
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$remote_ip" \
"rm -rf '$path' && echo removed" 2>/dev/null | grep -q removed && \
echo " Appdata removed on $MIRROR: $path ✅" || \
warn " Failed to remove appdata on $MIRROR: $path"
@@ -504,7 +504,7 @@ reconfigure_webui() {
local template
template=$(timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
-o ConnectTimeout="$SSH_TIMEOUT" root@"$remote_ip" \
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$remote_ip" \
"grep -rl '<WebUI>' '$TEMPLATES_DIR/' 2>/dev/null | \
xargs grep -l '\"$container\"' 2>/dev/null | head -1" 2>/dev/null)
@@ -514,7 +514,7 @@ reconfigure_webui() {
fi
timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
-o ConnectTimeout="$SSH_TIMEOUT" root@"$remote_ip" \
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$remote_ip" \
"sed -i 's|<WebUI>.*</WebUI>|<WebUI>http://${target_ip}:${port}/</WebUI>|g' '$template'" \
2>/dev/null && \
echo "$container → http://${target_ip}:${port}/ ✅" || {
Rsync/rsync.sh
+7 -2
View File
@@ -56,7 +56,7 @@
#
# Remote Health Pre-flights
# check_connectivity() — Tailscale IP reachable before any SSH
# check_remote_rootfs() — aborts if remote rootfs exceeds ROOTFS_WARN_PCT
# check_remote_rootfs() — aborts if remote rootfs exceeds ROOTFS_WARN
# check_remote_share() — aborts if target directory missing or empty on remote
# check_remote_disks() — verifies all backing disks online on remote
#
@@ -98,7 +98,7 @@
# SLEEP
# Default seconds between retry attempts. (default: 60)
#
# ROOTFS_WARN_PCT
# ROOTFS_WARN
# Abort threshold for remote rootfs percentage full. (default: 75)
#
# PROFILES["profile_KEY"]
@@ -223,6 +223,11 @@ VV_LAST_LOG="/tmp/unraid_locks/rsync_${PROFILE_NAME}.last.log"
exec 1> >(tee -a "$VV_LIVE_LOG") 2>&1
# ── Load profile settings ─────────────────────────────────────────────────────────────────────
# NOTE: DEFAULT_RSYNC_OPTS and PROFILE_RSYNC_OPTS are bash arrays defined in master.conf —
# any "$BW_LIMIT" inside them is expanded once, when master.conf is sourced, before this
# override runs. So this recomputed $BW_LIMIT only affects the log line below and --status;
# it does NOT change the actual rsync --bwlimit unless a profile also bakes its own
# --bwlimit="${PROFILE_BW_LIMIT[name]}" directly into that profile's PROFILE_RSYNC_OPTS entry.
BW_LIMIT=${PROFILE_BW_LIMIT[$PROFILE_NAME]:-$BW_LIMIT}
RETRY_COUNT=${PROFILE_RETRY_COUNT[$PROFILE_NAME]:-$RETRY_COUNT}
SLEEP=${PROFILE_SLEEP[$PROFILE_NAME]:-$SLEEP}
System_Essentials/conf_cache_restore.sh
+2 -2
View File
@@ -53,7 +53,7 @@ parse_args "$@"
detect_hosts
require_partnership
RAM_CACHE="/tmp/.cache/vv/d"
RAM_CACHE="$CONF_RAM_CACHE_DIR"
SAVE_DIR="$PERSISTENT_CONF_CACHE"
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
@@ -67,7 +67,7 @@ restored=0
for conf in "$SAVE_DIR"/host*.conf; do
[[ -f "$conf" ]] || continue
base="$(basename "$conf")"
[[ "${base,,}" == "${MY_ID,,}.conf" ]] && continue
is_own_conf_file "$base" && continue
if [[ -f "$RAM_CACHE/$base" ]]; then
log "$base already in RAM cache (conf_sync succeeded) — skipping"
System_Essentials/conf_cache_save.sh
+2 -2
View File
@@ -47,7 +47,7 @@ parse_args "$@"
detect_hosts
require_partnership
RAM_CACHE="/tmp/.cache/vv/d"
RAM_CACHE="$CONF_RAM_CACHE_DIR"
SAVE_DIR="$PERSISTENT_CONF_CACHE"
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be written"
@@ -61,7 +61,7 @@ saved=0
for conf in "$RAM_CACHE"/host*.conf; do
[[ -f "$conf" ]] || continue
base="$(basename "$conf")"
[[ "${base,,}" == "${MY_ID,,}.conf" ]] && continue
is_own_conf_file "$base" && continue
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would save $base → $SAVE_DIR/"
System_Essentials/conf_sync.sh
+1 -1
View File
@@ -81,7 +81,7 @@ if [[ "${CONF_SYNC_ENABLED:-true}" == false ]]; then
exit 0
fi
CACHE_DIR="/tmp/.cache/vv/d"
CACHE_DIR="$CONF_RAM_CACHE_DIR"
MY_CONF="$SCRIPTS_DIR/Configurations/${MY_ID,,}.conf"
SSH_TIMEOUT=10
Tools/container_data_export.sh
+6 -71
View File
@@ -146,17 +146,7 @@ cleanup_on_exit() {
warn "Removing partial archive: $ARCHIVE_PATH"
rm -f "$ARCHIVE_PATH" 2>/dev/null
fi
# Always restart container if it was running
if [[ "$CONTAINER_WAS_RUNNING" == true && "$DRY_RUN" == false ]]; then
local status
status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \
"$CONTAINER_NAME" 2>/dev/null)
if [[ "$status" != "true" ]]; then
warn "Restarting $CONTAINER_NAME (cleanup)..."
timeout "$DOCKER_TIMEOUT" docker start "$CONTAINER_NAME" >/dev/null 2>&1 || \
error "Failed to restart $CONTAINER_NAME — start it manually"
fi
fi
container_force_restart_if_needed "$CONTAINER_NAME" "$CONTAINER_WAS_RUNNING"
}
trap cleanup_on_exit EXIT
@@ -166,35 +156,10 @@ trap cleanup_on_exit EXIT
echo ""
echo "━━━ $ICON_STOP Stop Container ━━━"
STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \
"$CONTAINER_NAME" 2>/dev/null)
case "$STATUS" in
true)
CONTAINER_WAS_RUNNING=true
log "Stopping $CONTAINER_NAME for clean export..."
if [[ "$DRY_RUN" == false ]]; then
if timeout "$DOCKER_TIMEOUT" docker stop "$CONTAINER_NAME" >/dev/null 2>&1; then
echo "$CONTAINER_NAME stopped ✅"
else
error "Failed to stop $CONTAINER_NAME — aborting export"
exit 1
fi
else
warn "DRY RUN — would stop $CONTAINER_NAME"
fi
;;
false)
echo "$CONTAINER_NAME is not running — archiving as-is (was stopped state respected)"
;;
"")
error "$CONTAINER_NAME not found — check container name"
exit 1
;;
*)
warn "$CONTAINER_NAME status: $STATUS — proceeding with caution"
;;
esac
container_stop_for_maintenance "$CONTAINER_NAME" CONTAINER_WAS_RUNNING \
"Stopping $CONTAINER_NAME for clean export..." log || exit 1
[[ "$CONTAINER_WAS_RUNNING" == false ]] && \
echo "$CONTAINER_NAME is not running — archiving as-is (was stopped state respected)"
# ==============================================================================================
# ━━━ Archive ━━━
@@ -249,37 +214,7 @@ END=$(date +%s)
echo ""
echo "━━━ $ICON_START Restart Container ━━━"
RESTART_OK=false
if [[ "$CONTAINER_WAS_RUNNING" == true ]]; then
log "Restarting $CONTAINER_NAME..."
if [[ "$DRY_RUN" == false ]]; then
if timeout "$DOCKER_TIMEOUT" docker start "$CONTAINER_NAME" >/dev/null 2>&1; then
# Brief settle then verify
sleep 3
POST_STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
'{{.State.Running}}' "$CONTAINER_NAME" 2>/dev/null)
if [[ "$POST_STATUS" == "true" ]]; then
echo "$CONTAINER_NAME restarted and running ✅"
RESTART_OK=true
else
error "$CONTAINER_NAME started but crashed immediately — check container logs"
notify "$CONTAINER_NAME failed to stay running after export on $(hostname)" \
"Container Export" "warning"
fi
else
error "Failed to restart $CONTAINER_NAME — start it manually"
notify "$CONTAINER_NAME failed to restart after export on $(hostname)" \
"Container Export" "warning"
fi
else
warn "DRY RUN — would restart $CONTAINER_NAME"
RESTART_OK=true
fi
else
echo "$CONTAINER_NAME was not running — leaving stopped (state respected) ✅"
RESTART_OK=true
fi
container_restart_after_maintenance "$CONTAINER_NAME" "$CONTAINER_WAS_RUNNING" 3 "Container Export"
# Clear trap — clean exit, cleanup_on_exit no longer needed
trap - EXIT
Tools/emby_database_repair.sh
+7 -71
View File
@@ -171,16 +171,7 @@ EMBY_WAS_RUNNING=false
cleanup_on_exit() {
local exit_code=$?
if [[ "$EMBY_WAS_RUNNING" == true && "$DRY_RUN" == false ]]; then
local status
status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \
"$EMBY_CONTAINER" 2>/dev/null)
if [[ "$status" != "true" ]]; then
warn "Restarting $EMBY_CONTAINER (cleanup)..."
timeout "$DOCKER_TIMEOUT" docker start "$EMBY_CONTAINER" >/dev/null 2>&1 || \
error "Failed to restart $EMBY_CONTAINER — start it manually"
fi
fi
container_force_restart_if_needed "$EMBY_CONTAINER" "$EMBY_WAS_RUNNING"
}
trap cleanup_on_exit EXIT
@@ -191,36 +182,10 @@ trap cleanup_on_exit EXIT
echo ""
echo "━━━ $ICON_STOP Stop Emby ━━━"
STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \
"$EMBY_CONTAINER" 2>/dev/null)
case "$STATUS" in
true)
EMBY_WAS_RUNNING=true
warn "Stopping $EMBY_CONTAINER — active sessions will be interrupted"
if [[ "$DRY_RUN" == false ]]; then
if timeout "$DOCKER_TIMEOUT" docker stop "$EMBY_CONTAINER" >/dev/null 2>&1; then
echo "$EMBY_CONTAINER stopped ✅"
sleep 3 # let file handles release
else
error "Failed to stop $EMBY_CONTAINER — aborting"
exit 1
fi
else
warn "DRY RUN — would stop $EMBY_CONTAINER"
fi
;;
false)
log "$EMBY_CONTAINER is not running — proceeding with checks"
;;
"")
error "$EMBY_CONTAINER not found — check container name"
exit 1
;;
*)
warn "$EMBY_CONTAINER status: $STATUS — proceeding with caution"
;;
esac
container_stop_for_maintenance "$EMBY_CONTAINER" EMBY_WAS_RUNNING \
"Stopping $EMBY_CONTAINER — active sessions will be interrupted" warn 3 || exit 1
[[ "$EMBY_WAS_RUNNING" == false ]] && \
log "$EMBY_CONTAINER is not running — proceeding with checks"
# ==============================================================================================
# ━━━ Database Integrity Check ━━━
@@ -301,37 +266,8 @@ END=$(date +%s)
echo ""
echo "━━━ $ICON_START Restart Emby ━━━"
RESTART_OK=false
if [[ "$EMBY_WAS_RUNNING" == true ]]; then
if [[ "$DRY_RUN" == false ]]; then
log "Restarting $EMBY_CONTAINER..."
if timeout "$DOCKER_TIMEOUT" docker start "$EMBY_CONTAINER" >/dev/null 2>&1; then
sleep 5 # Emby takes longer to initialise than most containers
POST_STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
'{{.State.Running}}' "$EMBY_CONTAINER" 2>/dev/null)
if [[ "$POST_STATUS" == "true" ]]; then
echo "$EMBY_CONTAINER restarted and running ✅"
RESTART_OK=true
else
error "$EMBY_CONTAINER started but crashed — database may be corrupt"
error "Check Docker logs: docker logs $EMBY_CONTAINER"
notify "$EMBY_CONTAINER crashed on restart — possible database corruption on $(hostname)" \
"Emby DB Repair" "warning"
fi
else
error "Failed to restart $EMBY_CONTAINER — start it manually"
notify "$EMBY_CONTAINER failed to restart after integrity check on $(hostname)" \
"Emby DB Repair" "warning"
fi
else
warn "DRY RUN — would restart $EMBY_CONTAINER"
RESTART_OK=true
fi
else
echo "$EMBY_CONTAINER was not running — leaving stopped (state respected) ✅"
RESTART_OK=true
fi
container_restart_after_maintenance "$EMBY_CONTAINER" "$EMBY_WAS_RUNNING" 5 "Emby DB Repair"
[[ "$RESTART_OK" == false ]] && error "Check Docker logs: docker logs $EMBY_CONTAINER"
# Clear EXIT trap — clean exit
trap - EXIT
Tools/smart_long_test.sh
+3 -10
View File
@@ -138,11 +138,8 @@ _build_drive_list() {
[[ ! -e "$drive" ]] && continue
local drive_name
drive_name=$(basename "$drive")
local ignored=false
for ignore in "${SMART_IGNORE_DRIVES[@]:-}"; do
[[ "$drive_name" == "$ignore" ]] && ignored=true && break
done
[[ "$ignored" == true ]] && { log "Skipping $drive_name (SMART_IGNORE_DRIVES)"; continue; }
is_in_list "$drive_name" "${SMART_IGNORE_DRIVES[@]:-}" && \
{ log "Skipping $drive_name (SMART_IGNORE_DRIVES)"; continue; }
if ! smartctl -i "$drive" 2>/dev/null | grep -q "SMART support is: Enabled"; then
log "Skipping $drive_name — SMART not enabled"
continue
@@ -166,11 +163,7 @@ if [[ "$SHOW_STATUS" == true ]]; then
drive_name=$(basename "$drive")
local_result=$(smartctl -l selftest "$drive" 2>/dev/null | \
grep -m1 "Extended" | awk '{print $NF}')
ignored=false
for ignore in "${SMART_IGNORE_DRIVES[@]:-}"; do
[[ "$drive_name" == "$ignore" ]] && ignored=true && break
done
if [[ "$ignored" == true ]]; then
if is_in_list "$drive_name" "${SMART_IGNORE_DRIVES[@]:-}"; then
echo " $ICON_WARN $drive_name — ignored"
else
echo " $ICON_SMART $drive_name — last extended: ${local_result:-no result}"
Transcodes/transcode_cleanup.sh
+2 -10
View File
@@ -240,15 +240,7 @@ cleanup_location() {
# Format bytes freed
local freed_human
if (( bytes_freed > 1073741824 )); then
freed_human=$(awk "BEGIN {printf \"%.1fGB\", $bytes_freed / 1073741824}")
elif (( bytes_freed > 1048576 )); then
freed_human=$(awk "BEGIN {printf \"%.1fMB\", $bytes_freed / 1048576}")
elif (( bytes_freed > 0 )); then
freed_human="${bytes_freed}B"
else
freed_human="0B"
fi
freed_human=$(format_bytes "$bytes_freed")
log "$label — removed $files_removed ($freed_human) | active(fresh): $files_too_young | streaming(lsof): $files_streaming | protected(old+open): $files_active | skipped: $files_skipped | failed: $files_failed"
@@ -299,7 +291,7 @@ fi
# Post-cleanup — check if ramdisk recovered enough to flip symlink back to ramdisk
if [[ "$DRY_RUN" == false ]] && mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
RAMDISK_USED_KB=$(df "$RAMDISK_PATH" --output=used 2>/dev/null | tail -1 | tr -d ' ')
RAMDISK_USED_GB=$(awk "BEGIN {printf \"%.2f\", ${RAMDISK_USED_KB:-0} / 1048576}")
RAMDISK_USED_GB=$(kb_to_gb "$RAMDISK_USED_KB")
LOW_RECOVERED=$(awk "BEGIN {print ($RAMDISK_USED_GB < $RAMDISK_LOW_GB) ? 1 : 0}")
CURRENT_TARGET=$(grep "^current_target=" "$STATE_FILE" 2>/dev/null | cut -d= -f2)
Transcodes/transcode_manager.sh
+3 -6
View File
@@ -234,18 +234,15 @@ fix_permissions() {
}
get_ramdisk_used_gb() {
df "$RAMDISK_PATH" --output=used 2>/dev/null | \
tail -1 | tr -d ' ' | awk '{printf "%.2f", $1/1048576}'
kb_to_gb "$(df "$RAMDISK_PATH" --output=used 2>/dev/null | tail -1 | tr -d ' ')"
}
get_ramdisk_avail_gb() {
df "$RAMDISK_PATH" --output=avail 2>/dev/null | \
tail -1 | tr -d ' ' | awk '{printf "%.2f", $1/1048576}'
kb_to_gb "$(df "$RAMDISK_PATH" --output=avail 2>/dev/null | tail -1 | tr -d ' ')"
}
get_ssd_free_gb() {
df "$TRANSCODE_SSD" --output=avail 2>/dev/null | \
tail -1 | tr -d ' ' | awk '{printf "%.2f", $1/1048576}'
kb_to_gb "$(df "$TRANSCODE_SSD" --output=avail 2>/dev/null | tail -1 | tr -d ' ')"
}
get_flip_count() {
Watchdogs/System/network_watchdog.sh
+3 -10
View File
@@ -125,16 +125,9 @@ log "$ICON_NET DDNS: ${DDNS_DOMAIN:-not configured} → ${DDNS_CONTAINER:-no c
touch "${NETWORK_WATCHDOG_NPM_STATE_FILE}" 2>/dev/null
# ━━━ Strike helpers ━━━
get_strikes() { grep "^${1}:" "${2}" 2>/dev/null | cut -d: -f2 || echo "0"; }
set_strikes() {
local key="$1" count="$2" file="$3"
if grep -q "^${key}:" "$file" 2>/dev/null; then
sed -i "s|^${key}:.*|${key}:${count}|" "$file"
else
echo "${key}:${count}" >> "$file"
fi
}
# ━━━ Strike helpers — wrap common.sh's wd_state_get()/wd_state_set() ━━━
get_strikes() { wd_state_get "$1" "$2"; }
set_strikes() { wd_state_set "$1" "$2" "$3"; }
# ==============================================================================================
# ━━━ Status ━━━
Watchdogs/System/storage_watchdog.sh
+3 -10
View File
@@ -164,16 +164,9 @@ log "$ICON_DISK Paths: ${WATCHDOG_APPDATA_PATHS[*]:-none configured}"
touch "$STORAGE_WATCHDOG_STATE_FILE" 2>/dev/null
touch "$WATCHDOG_APPDATA_GROWTH_FILE" 2>/dev/null
# ━━━ Strike helpers ━━━
get_strikes() { grep "^${1}:" "${2}" 2>/dev/null | cut -d: -f2 || echo "0"; }
set_strikes() {
local key="$1" count="$2" file="$3"
if grep -q "^${key}:" "$file" 2>/dev/null; then
sed -i "s|^${key}:.*|${key}:${count}|" "$file"
else
echo "${key}:${count}" >> "$file"
fi
}
# ━━━ Strike helpers — wrap common.sh's wd_state_get()/wd_state_set() ━━━
get_strikes() { wd_state_get "$1" "$2"; }
set_strikes() { wd_state_set "$1" "$2" "$3"; }
# ==============================================================================================
# ━━━ Status ━━━
Watchdogs/docker_watchdog.sh
+9 -10
View File
@@ -194,6 +194,9 @@
# WATCHDOG_CONTAINER_RESTART_LIMIT / WATCHDOG_CONTAINER_RESTART_WINDOW
# Restart loop protection: attempt limit and rolling window in hours
#
# WATCHDOG_REQUIRED_STRIKE_LIMIT
# Consecutive down-checks on a required container before a restart is attempted (default: 2)
#
# WATCHDOG_BATCH_NOTIFY
# Collect cycle events and send as one notification (default: true)
#
@@ -337,19 +340,15 @@ fi
# ── HELPER FUNCTIONS ──────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Get strike count for a container from state file
# Get strike count for a container from state file — wraps common.sh's wd_state_get()
get_strikes() {
grep "^${1}:" "${2}" 2>/dev/null | cut -d: -f2 || echo "0"
wd_state_get "$1" "$2"
}
# Set strike count for a container in state file
# Set strike count for a container in state file — wraps common.sh's wd_state_set()
set_strikes() {
local container="$1" count="$2" file="$3"
if grep -q "^${container}:" "$file" 2>/dev/null; then
sed -i "s/^${container}:.*/${container}:${count}/" "$file"
else
echo "${container}:${count}" >> "$file"
fi
wd_state_set "$container" "$count" "$file"
}
# Check if container is on the persistent skip list
@@ -763,10 +762,10 @@ CYCLE_START=$(date +%s)
STRIKES=$(get_strikes "$container" "$WATCHDOG_STATE_FILE")
STRIKES=$(( STRIKES + 1 ))
set_strikes "$container" "$STRIKES" "$WATCHDOG_STATE_FILE"
warn "$container — not running, exit ${LAST_EXIT} (strike $STRIKES/$SYS_WATCHDOG_STRIKE_LIMIT)"
warn "$container — not running, exit ${LAST_EXIT} (strike $STRIKES/${WATCHDOG_REQUIRED_STRIKE_LIMIT:-2})"
((T1_WARNINGS++))
if [[ "$STRIKES" -ge "$SYS_WATCHDOG_STRIKE_LIMIT" ]]; then
if [[ "$STRIKES" -ge "${WATCHDOG_REQUIRED_STRIKE_LIMIT:-2}" ]]; then
result=0
safe_restart "$container" "required container down (exit ${LAST_EXIT})" || result=$?
case $result in
Watchdogs/resource_watchdog.sh
+12 -12
View File
@@ -165,27 +165,25 @@ trap "_release_all_locks; _rw_trap_restart_stopped" EXIT
# ==============================================================================================
# rm_state_get/set use : separator for RM-internal state
# rm_state_get_eq/set_eq use = separator for docker_watchdog coordination flags
# Both wrap common.sh's wd_state_get()/wd_state_set() — file is implicit here (this
# script's own state file), unlike the generic helper which always takes it as an argument.
rm_state_get() {
grep -E "^${1}:" "$RW_STATE_FILE" 2>/dev/null | cut -d: -f2-
wd_state_get "$1" "$RW_STATE_FILE"
}
rm_state_set() {
local key="$1" val="$2"
grep -vE "^${key}:" "$RW_STATE_FILE" 2>/dev/null > "${RW_STATE_FILE}.tmp"
echo "${key}:${val}" >> "${RW_STATE_FILE}.tmp"
mv "${RW_STATE_FILE}.tmp" "$RW_STATE_FILE"
wd_state_set "$key" "$val" "$RW_STATE_FILE"
}
rm_state_get_eq() {
grep -E "^${1}=" "$RW_STATE_FILE" 2>/dev/null | cut -d= -f2-
wd_state_get "$1" "$RW_STATE_FILE" "="
}
rm_state_set_eq() {
local key="$1" val="$2"
grep -vE "^${key}=" "$RW_STATE_FILE" 2>/dev/null > "${RW_STATE_FILE}.tmp"
echo "${key}=${val}" >> "${RW_STATE_FILE}.tmp"
mv "${RW_STATE_FILE}.tmp" "$RW_STATE_FILE"
wd_state_set "$key" "$val" "$RW_STATE_FILE" "="
}
# ==============================================================================================
@@ -376,7 +374,8 @@ unpause_containers() {
}
# Stop a list of containers — returns comma-separated list of actually-stopped containers
stop_containers() {
# Named rw_ to avoid colliding with common.sh's stop_containers() (different signature/semantics)
rw_stop_containers() {
local actually_stopped=()
for container in "$@"; do
[[ -z "$container" ]] && continue
@@ -407,7 +406,8 @@ stop_containers() {
}
# Start a comma-separated list of containers (only those RM stopped)
start_containers() {
# Named rw_ to avoid colliding with common.sh's start_containers() (different signature/semantics)
rw_start_containers() {
local IFS=','
for container in $1; do
[[ -z "$container" ]] && continue
@@ -468,7 +468,7 @@ apply_level_3() {
if [[ ${#RW_STOP_CONTAINERS[@]} -gt 0 ]]; then
local newly_stopped
newly_stopped=$(stop_containers "${RW_STOP_CONTAINERS[@]}")
newly_stopped=$(rw_stop_containers "${RW_STOP_CONTAINERS[@]}")
if [[ -n "$newly_stopped" ]]; then
if [[ -n "$STOPPED_LIST" ]]; then
STOPPED_LIST="${STOPPED_LIST},${newly_stopped}"
@@ -490,7 +490,7 @@ apply_level_3() {
restore_level_3() {
echo "Restoring from level 3 — starting stopped containers"
if [[ -n "$STOPPED_LIST" ]]; then
start_containers "$STOPPED_LIST"
rw_start_containers "$STOPPED_LIST"
STOPPED_LIST=""
fi
rm_state_set_eq "mem_shutdown_active" "false"
Watchdogs/stability_watchdog.sh
+6 -8
View File
@@ -192,15 +192,15 @@ fi
# ── STATE HELPERS ─────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Wrap common.sh's wd_state_get()/wd_state_set() — file is implicit here (this script's
# own state file), unlike the generic helper which always takes it as an argument.
get_strikes() {
grep -E "^${1}:" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | cut -d':' -f2
wd_state_get "$1" "$SYS_WATCHDOG_STATE_FILE"
}
set_strikes() {
local key="$1" count="$2"
grep -vE "^${key}:" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null > "${SYS_WATCHDOG_STATE_FILE}.tmp"
echo "${key}:${count}" >> "${SYS_WATCHDOG_STATE_FILE}.tmp"
mv "${SYS_WATCHDOG_STATE_FILE}.tmp" "$SYS_WATCHDOG_STATE_FILE"
wd_state_set "$key" "$count" "$SYS_WATCHDOG_STATE_FILE"
}
increment_strikes() {
@@ -218,14 +218,12 @@ reset_strikes() {
}
get_state_val() {
grep -E "^${1}=" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | cut -d'=' -f2
wd_state_get "$1" "$SYS_WATCHDOG_STATE_FILE" "="
}
set_state_val() {
local key="$1" val="$2"
grep -vE "^${key}=" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null > "${SYS_WATCHDOG_STATE_FILE}.tmp"
echo "${key}=${val}" >> "${SYS_WATCHDOG_STATE_FILE}.tmp"
mv "${SYS_WATCHDOG_STATE_FILE}.tmp" "$SYS_WATCHDOG_STATE_FILE"
wd_state_set "$key" "$val" "$SYS_WATCHDOG_STATE_FILE" "="
}
purge_old_reboots() {
common.sh
+653 -107
View File
@@ -49,8 +49,7 @@
# check_arr_version() added — API version safety gate before arr operations
# check_api() added — pre-flight API reachability check
# ping_remote(), ping_internet() — non-fatal ping for fallback use
# check_local_array(), check_remote_array() — array health checks
# check_remote_docker() — Docker daemon health check
# check_remote_array() — array health check
# get_unraid_temp_thresholds() — reads thresholds from dynamix.cfg
# check_local_disk_temps() — pre-rsync temp check with exit codes 0/1/2
# stop/start local containers added alongside existing remote variants
@@ -79,15 +78,12 @@
# Called by lidarr/sonarr/radarr_cleanup.sh when files are deleted
#
#
# Three new safety functions added:
# Two new safety functions added:
# check_os_version_parity() — refuses remote ops on version mismatch
# reads OS version via platform_get_os_version() / platform_os_version_probe_cmd()
# major mismatch → abort | minor mismatch → configurable warn/abort
# check_remote_docker_daemon() — verifies remote Docker daemon before
# issuing any remote container commands — strike system → skip/retry/exit
# validate_unraid_cmd() — verifies unRAID-specific commands exist and
# produce expected output before use — notifies and exits calling script
# if command changed or disappeared after upgrade — other scripts unaffected
#
# ── VERSION ───────────────────────────────────────────────────────────────────────────────────
# Current: v3.5
@@ -130,6 +126,7 @@ ICON_SYNC="🔄" # transfer section header
ICON_RUN="🚀" # sync starting / rsync attempt
ICON_RETRY="🔁" # retry attempt
ICON_DONE="🏁" # transfer complete
ICON_GIT="🌱" # git pull/push section header
# Summary
ICON_SUMMARY="📋" # summary section header
@@ -269,6 +266,33 @@ format_duration() {
else echo "${rem}s"; fi
}
# Tiered human-readable size — picks B/MB/GB based on magnitude.
# Usage: format_bytes 5368709120 → 5.0GB
format_bytes() {
local bytes=${1:-0}
if (( bytes > 1073741824 )); then
awk "BEGIN {printf \"%.1fGB\", $bytes / 1073741824}"
elif (( bytes > 1048576 )); then
awk "BEGIN {printf \"%.1fMB\", $bytes / 1048576}"
else
echo "${bytes}B"
fi
}
# Fixed-precision bytes → GB (no tiering) — for values always compared against a GB threshold.
# Usage: bytes_to_gb 5368709120 [decimals=2] → 5.00
bytes_to_gb() {
local bytes=${1:-0} decimals=${2:-2}
awk "BEGIN {printf \"%.${decimals}f\", $bytes / 1073741824}"
}
# Fixed-precision KB → GB (no tiering) — df --output=used/avail reports in KB.
# Usage: kb_to_gb 5242880 [decimals=2] → 5.00
kb_to_gb() {
local kb=${1:-0} decimals=${2:-2}
awk "BEGIN {printf \"%.${decimals}f\", $kb / 1048576}"
}
# ==============================================================================================
# ── ARG PARSER ────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
@@ -355,6 +379,17 @@ validate_int() {
log "$name validated: $value"
}
# Membership check against a list of values (e.g. an ignore list).
# Usage: is_in_list "$needle" "${HAYSTACK_ARRAY[@]}"
is_in_list() {
local needle="$1"; shift
local item
for item in "$@"; do
[[ "$needle" == "$item" ]] && return 0
done
return 1
}
# ==============================================================================================
# ── HOST DETECTION ────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
@@ -506,6 +541,7 @@ detect_hosts() {
SYS_WATCHDOG_CHECK_MDSTAT SYS_WATCHDOG_CHECK_NETWORK
SYS_WATCHDOG_CHECK_SSHD SYS_WATCHDOG_CHECK_RUNAWAY
)
local _wd_var
for _wd_var in "${_wd_checks[@]}"; do
local _wd_src="${MY_ID}_${_wd_var}"
# Only alias if the host-specific var is set — preserves master.conf defaults
@@ -625,6 +661,86 @@ detect_hosts() {
log "$ICON_HOST Host: $LOCAL_SERVER_NAME → $REMOTE_SERVER_NAME"
}
# Derives owner/mirror roles from PARTNERSHIP_OWNER_HOST — used by every Partnership/*.sh
# script. Requires detect_hosts() to have already run (needs MY_ID, SSH_KEY).
# Sets: OWNER_ID MIRROR_ID OWNER MIRROR MIRROR_SSH_KEY OWNER_SSH_KEY AM_OWNER AM_MIRROR
# Does NOT set any *_STATE_FILE vars — callers that need those set them after calling this,
# since not every Partnership script needs the same set (transfer/onboard need fewer than
# manager/offboard).
partnership_resolve_roles() {
OWNER_ID="${PARTNERSHIP_OWNER_HOST:-HOST1}"
MIRROR_ID=$( [[ "$OWNER_ID" == "HOST1" ]] && echo "HOST2" || echo "HOST1" )
OWNER="${!OWNER_ID}"
MIRROR="${!MIRROR_ID}"
# SSH_KEY (set by detect_hosts) is this server's own private key. The remote accepts it
# because this server's PUBLIC key was installed there via ssh_setup.sh. With sparse
# checkout, each server only has its own host{N}.conf — the other server's key path is
# never available here. Use SSH_KEY for all outbound SSH regardless of mode.
MIRROR_SSH_KEY="$SSH_KEY"
OWNER_SSH_KEY="$SSH_KEY"
AM_OWNER=false
AM_MIRROR=false
[[ "$MY_ID" == "$OWNER_ID" ]] && AM_OWNER=true
[[ "$MY_ID" == "$MIRROR_ID" ]] && AM_MIRROR=true
}
# Read a scalar/array var from a partner's own config via SSH — the partner sources its
# own load_config.sh + detect_hosts() remotely so the value reflects ITS host*.conf, not
# ours. Requires $SCRIPT_DIR to be set by the caller (one level above the repo root, same
# convention every Partnership/*.sh script already uses for its own SCRIPT_DIR).
# Usage: read_remote_conf_var "$mirror_ip" "VAR_NAME"
# read_remote_conf_array "$mirror_ip" "ARRAY_NAME" # one element per line
read_remote_conf_var() {
local mirror_ip="$1" var_name="$2"
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \
"source '$SCRIPT_DIR/../load_config.sh' 2>/dev/null
detect_hosts 2>/dev/null
printf '%s' \"\${${var_name}:-}\"" 2>/dev/null
}
read_remote_conf_array() {
local mirror_ip="$1" var_name="$2"
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \
"source '$SCRIPT_DIR/../load_config.sh' 2>/dev/null
detect_hosts 2>/dev/null
printf '%s\n' \"\${${var_name}[@]:-}\"" 2>/dev/null
}
# Probes a remote host's SCRIPTS_DIR (in case it differs from ours — e.g. one host runs
# internal storage mode, the other flash), falling back to our own $SCRIPTS_DIR if the
# probe fails or the remote isn't reachable. strict_host_key defaults to "yes" (matches
# existing behavior everywhere except partnership_transfer.sh, which passes "no" since
# it may be contacting a mirror for the first time during an ownership transfer).
# Usage: resolve_remote_scripts_dir "$ip" ["$ssh_key"] ["no"|"yes"]
resolve_remote_scripts_dir() {
local ip="$1" ssh_key="${2:-$SSH_KEY}" strict_host_key="${3:-yes}"
local probe_cmd result
local -a opts=(-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes)
[[ "$strict_host_key" == "no" ]] && opts+=(-o StrictHostKeyChecking=no)
probe_cmd=$(platform_scripts_dir_probe_cmd)
result=$(timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" "${opts[@]}" root@"$ip" "$probe_cmd" \
2>/dev/null | tr -d '[:space:]')
echo "${result:-$SCRIPTS_DIR}"
}
# ==============================================================================================
# ── MULTI-NODE DISCOVERY ──────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Scans HOST1..HOST8 for every configured node other than this one. Requires detect_hosts()
# to have already run (needs MY_ID). Sets the global REMOTE_NODES array — empty if none found.
discover_remote_nodes() {
REMOTE_NODES=()
local _hv
for _hv in HOST1 HOST2 HOST3 HOST4 HOST5 HOST6 HOST7 HOST8; do
[[ "$_hv" == "$MY_ID" ]] && continue
[[ -z "${!_hv:-}" ]] && continue
REMOTE_NODES+=("$_hv")
done
}
# ==============================================================================================
# ── REMOTE IP RESOLUTION ──────────────────────────────────────────────────────────────────────
# ==============================================================================================
@@ -669,6 +785,34 @@ resolve_tailscale_ip() {
echo "$ip"
}
# Strips the "unraid-" prefix (case-insensitive) and title-cases what remains.
# Usage: derive_short_name "unRAID-Gmer4Lfe" → "Gmer4lfe"
derive_short_name() {
local hostname="$1"
local short="${hostname,,}"
[[ "$short" == unraid-* ]] && short="${short:7}"
echo "${short^}"
}
# True if the given host*.conf basename belongs to this host (case-insensitive).
# Usage: is_own_conf_file "$(basename "$conf")"
is_own_conf_file() {
[[ "${1,,}" == "${MY_ID,,}.conf" ]]
}
# Sets key=value in a flat state file (setup.db style) — updates in place if the
# key exists, appends if not. No indentation handling — for master.conf use
# partnership_manager.sh's update_master_conf() instead.
# Usage: set_state_var "$state_file" "$key" "$value"
set_state_var() {
local file="$1" key="$2" value="$3"
if grep -q "^${key}=" "$file" 2>/dev/null; then
sed -i "s|^${key}=.*|${key}=${value}|" "$file"
else
echo "${key}=${value}" >> "$file"
fi
}
# ==============================================================================================
# ── CONNECTIVITY CHECKS ───────────────────────────────────────────────────────────────────────
# ==============================================================================================
@@ -755,26 +899,6 @@ is_vm_manager_enabled() {
# ── LOCAL HEALTH CHECKS ───────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Verifies local storage is mounted and has shares.
# Non-fatal — returns status for caller to decide.
# Used by fallback before starting remote containers locally.
check_local_array() {
log "Checking local array..."
if ! platform_storage_healthy; then
error "$ICON_DISK Local array is not started — storage not mounted"
return 1
fi
local _storage_path file_count
_storage_path=$(platform_storage_path)
file_count=$(ls "$_storage_path" 2>/dev/null | wc -l)
if [[ "$file_count" -eq 0 ]]; then
error "$ICON_DISK Local array appears empty — shares may not be available"
return 1
fi
log "Local array is healthy"
return 0
}
# ==============================================================================================
# ── REMOTE HEALTH CHECKS ─────────────────────────────────────────────────────────────────────
# ==============================================================================================
@@ -796,19 +920,6 @@ check_remote_array() {
return 0
}
# Verifies remote Docker daemon is responding.
# Non-fatal — returns status. A hung daemon means container commands silently fail.
check_remote_docker() {
log "Checking remote Docker daemon on $REMOTE_SERVER_NAME..."
if ! ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \
"timeout 10 docker ps" >/dev/null 2>&1; then
error "$ICON_CONTAINERS Remote Docker daemon not responding on $REMOTE_SERVER_NAME"
return 1
fi
log "Remote Docker daemon is healthy"
return 0
}
# Aborts if remote rootfs (/) usage is at or above ROOTFS_WARN threshold.
# Fatal — exits the calling script.
# When remote array is down rsync writes land on rootfs and fill it rapidly.
@@ -861,9 +972,14 @@ check_remote_share() {
# Returns 0 (true) if device is non-rotational (SSD/NVMe), 1 (false) if HDD.
# Checks /sys/block/<device>/queue/rotational — 0=SSD, 1=HDD.
is_ssd() {
local base
base=$(basename "$1")
[[ "$(cat "/sys/block/$base/queue/rotational" 2>/dev/null)" == "0" ]]
local drive="$1"
local base rotational
base=$(basename "$drive")
rotational="/sys/block/$base/queue/rotational"
[[ -f "$rotational" ]] && [[ "$(cat "$rotational" 2>/dev/null)" == "0" ]] && return 0
# NVMe is always SSD — some NVMe controllers don't expose rotational correctly
[[ "$drive" == *nvme* ]] && return 0
return 1
}
# Sets globals: UNRAID_DISK_HOT UNRAID_DISK_MAX UNRAID_SSD_HOT UNRAID_SSD_MAX
@@ -878,6 +994,37 @@ get_unraid_temp_thresholds() {
}
# Checks local disk temperatures before rsync.
# Fetches a domain's TLS cert expiry via openssl s_client + x509 and computes days
# remaining. Pure data — no logging, no thresholds — callers classify warn/crit
# themselves against their own CERT_WARN_DAYS/CERT_CRIT_DAYS and log however they like.
#
# Sets _CERT_DAYS, _CERT_EXPIRY (YYYY-MM-DD) on success. Sets _CERT_EXPIRY_RAW to the
# unparsed openssl date string only when parsing failed (empty when unreachable) so a
# caller that wants a more specific error message can distinguish the two failure modes.
#
# Usage: check_cert_expiry "$domain" [port=443] [timeout=$CERT_TIMEOUT or 10]
# Returns: 0 = fetched and parsed | 1 = unreachable or unparseable
check_cert_expiry() {
local domain="$1" port="${2:-443}" cert_timeout="${3:-${CERT_TIMEOUT:-10}}"
_CERT_DAYS=""
_CERT_EXPIRY=""
_CERT_EXPIRY_RAW=""
local expiry_str
expiry_str=$(echo | timeout "$cert_timeout" openssl s_client \
-connect "${domain}:${port}" -servername "$domain" \
2>/dev/null | openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2)
[[ -z "$expiry_str" ]] && return 1
local expiry_epoch
expiry_epoch=$(date -d "$expiry_str" +%s 2>/dev/null)
if [[ -z "$expiry_epoch" ]]; then
_CERT_EXPIRY_RAW="$expiry_str"
return 1
fi
_CERT_DAYS=$(( (expiry_epoch - $(date +%s)) / 86400 ))
_CERT_EXPIRY=$(date -d "$expiry_str" '+%Y-%m-%d' 2>/dev/null)
return 0
}
# Reads temps and rotational flag from /var/local/emhttp/disks.ini.
# Uses unRAID's own thresholds from dynamix.cfg via get_unraid_temp_thresholds().
#
@@ -1305,6 +1452,62 @@ start_local_containers() {
done
}
# Builds a dependency-safe restart order — dependency containers (anything named as a
# value in WATCHDOG_DEPENDENCIES) restart first, then everything else in original order.
# Sets ORDERED_RESTART. Usage: build_restart_order CONTAINERS_ARRAY_NAME (bare name, e.g.
# build_restart_order DAILY_RESTART_CONTAINERS — not "${DAILY_RESTART_CONTAINERS[@]}").
build_restart_order() {
local -n _bro_containers="$1"
ORDERED_RESTART=()
local remaining=("${_bro_containers[@]}")
local placed=()
# First pass — add dependency containers that appear in our list
for container in "${remaining[@]}"; do
[[ -z "$container" ]] && continue
local is_dependency=false
for dependent in "${!WATCHDOG_DEPENDENCIES[@]}"; do
if [[ "${WATCHDOG_DEPENDENCIES[$dependent]}" == *"$container"* ]]; then
is_dependency=true
break
fi
done
if [[ "$is_dependency" == true ]]; then
local already=false
for p in "${placed[@]}"; do [[ "$p" == "$container" ]] && already=true && break; done
if [[ "$already" == false ]]; then
ORDERED_RESTART+=("$container")
placed+=("$container")
fi
fi
done
# Second pass — add remaining containers (dependents and independents)
for container in "${remaining[@]}"; do
[[ -z "$container" ]] && continue
local already=false
for p in "${placed[@]}"; do [[ "$p" == "$container" ]] && already=true && break; done
if [[ "$already" == false ]]; then
ORDERED_RESTART+=("$container")
placed+=("$container")
fi
done
log "Restart order: ${ORDERED_RESTART[*]}"
}
# Waits CONTAINER_DELAY if this container depends on the last restarted one.
# Usage: check_dependency_delay "$container" "$last_restarted"
check_dependency_delay() {
local container="$1" last="$2"
[[ -z "$last" ]] && return
local deps="${WATCHDOG_DEPENDENCIES[$container]:-}"
if [[ -n "$deps" ]] && [[ "$deps" == *"$last"* ]]; then
log "Waiting ${CONTAINER_DELAY}s — $container depends on $last..."
sleep "$CONTAINER_DELAY"
fi
}
# ==============================================================================================
# ── RSYNC OPTIONS ─────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
@@ -1380,12 +1583,6 @@ _lock_file() {
echo "$LOCK_DIR/${1:-$(_lock_name)}.lock"
}
# Internal — release lock on exit (registered via EXIT trap)
_release_on_exit() {
local lockfile="$1"
[[ -f "$lockfile" ]] && rm -f "$lockfile"
}
# Acquire exclusive lock for this script.
# Usage: acquire_lock [strict|wait|continuous]
acquire_lock() {
@@ -1524,17 +1721,30 @@ acquire_rsync_lock() {
log "$ICON_LOCK rsync lock acquired: profile '$profile' (PID $$, active: $(( current_count + 1 ))/$RSYNC_MAX_CONCURRENT)"
}
# Internal — release rsync lock and decrement global counter on exit
_release_rsync_on_exit() {
local profile_lock="$1"
[[ -f "$profile_lock" ]] && rm -f "$profile_lock"
if [[ -f "$RSYNC_COUNT_FILE" ]]; then
local count
count=$(cat "$RSYNC_COUNT_FILE" 2>/dev/null || echo 1)
count=$(( count - 1 ))
[[ "$count" -lt 0 ]] && count=0
echo "$count" > "$RSYNC_COUNT_FILE"
fi
# ==============================================================================================
# ── FLAT STATE-FILE KEY/VALUE HELPERS ─────────────────────────────────────────────────────────
# ==============================================================================================
# Generic get/set for flat "key<sep>value" state files (one entry per line) — the pattern
# every watchdog's strike-tracking and state-file logic was independently reimplementing.
# grep -v + tempfile-swap on write, not sed -i in place — avoids sed treating a key containing
# regex metacharacters (container names, etc.) as part of the substitution pattern.
#
# Usage: wd_state_get "$key" "$file" [sep=:]
# wd_state_set "$key" "$value" "$file" [sep=:]
#
# Callers with a fixed state file and/or separator (e.g. stability_watchdog.sh's
# get_strikes/get_state_val, resource_watchdog.sh's rm_state_get/rm_state_get_eq) should
# keep their own thin same-named wrapper around these rather than changing call sites.
wd_state_get() {
local key="$1" file="$2" sep="${3:-:}"
grep -E "^${key}${sep}" "$file" 2>/dev/null | cut -d"$sep" -f2-
}
wd_state_set() {
local key="$1" value="$2" file="$3" sep="${4:-:}"
grep -vE "^${key}${sep}" "$file" 2>/dev/null > "${file}.tmp"
echo "${key}${sep}${value}" >> "${file}.tmp"
mv "${file}.tmp" "$file"
}
# ==============================================================================================
@@ -1632,6 +1842,348 @@ translate_path() {
fi
}
# ==============================================================================================
# ── CONTAINER STOP/RESTART FOR MAINTENANCE ────────────────────────────────────────────────────
# ==============================================================================================
# The "stop a container for a maintenance window, guarantee it comes back" shape shared by
# container_data_export.sh and emby_database_repair.sh. Callers still register their own EXIT
# trap (they may need extra cleanup, e.g. removing a partial archive) — the trap should call
# container_force_restart_if_needed() for the actual "still down? bring it back" part.
# Usage: container_stop_for_maintenance "$container" was_running_var_name \
# ["pre-stop reason message"] [log|warn] [post_stop_sleep=0]
# The reason message (if given) prints via the chosen level (default: log) right before the
# stop attempt, only when the container was actually running — callers use this for their
# own context ("stopping for clean export" vs "active sessions will be interrupted").
# Sets the named var true/false. Returns 1 (caller should exit) if the container doesn't
# exist or fails to stop; 0 otherwise (including the "wasn't running" case — caller logs
# its own context-specific message for that, since wording/visibility differs per caller).
container_stop_for_maintenance() {
local container="$1"
local -n _csm_was_running="$2"
local pre_stop_msg="${3:-}" pre_stop_level="${4:-log}" post_stop_sleep="${5:-0}"
_csm_was_running=false
local status
status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
case "$status" in
true)
_csm_was_running=true
if [[ -n "$pre_stop_msg" ]]; then
if [[ "$pre_stop_level" == "warn" ]]; then warn "$pre_stop_msg"; else log "$pre_stop_msg"; fi
fi
if [[ "$DRY_RUN" == false ]]; then
if timeout "$DOCKER_TIMEOUT" docker stop "$container" >/dev/null 2>&1; then
echo "$container stopped ✅"
[[ "$post_stop_sleep" -gt 0 ]] && sleep "$post_stop_sleep"
else
error "Failed to stop $container — aborting"
return 1
fi
else
warn "DRY RUN — would stop $container"
fi
;;
false) : ;; # not running — caller logs its own context-specific message
"")
error "$container not found — check container name"
return 1
;;
*)
warn "$container status: $status — proceeding with caution"
;;
esac
return 0
}
# Force-restart from inside a script's own EXIT trap if the container is still down after
# a crash mid-maintenance. No-op if it wasn't running before, in dry-run, or already up.
# Usage: container_force_restart_if_needed "$container" "$was_running"
container_force_restart_if_needed() {
local container="$1" was_running="$2"
[[ "$was_running" == true && "$DRY_RUN" == false ]] || return 0
local status
status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
if [[ "$status" != "true" ]]; then
warn "Restarting $container (cleanup)..."
timeout "$DOCKER_TIMEOUT" docker start "$container" >/dev/null 2>&1 || \
error "Failed to restart $container — start it manually"
fi
}
# Normal-path restart after maintenance completes — verifies it stayed up, notifies on failure.
# Sets RESTART_OK true/false.
# Usage: container_restart_after_maintenance "$container" "$was_running" [settle_sleep=3] [notify_label]
container_restart_after_maintenance() {
local container="$1" was_running="$2" settle_sleep="${3:-3}" notify_label="${4:-Container Maintenance}"
RESTART_OK=false
if [[ "$was_running" != true ]]; then
echo "$container was not running — leaving stopped (state respected) ✅"
RESTART_OK=true
return 0
fi
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would restart $container"
RESTART_OK=true
return 0
fi
log "Restarting $container..."
if timeout "$DOCKER_TIMEOUT" docker start "$container" >/dev/null 2>&1; then
sleep "$settle_sleep"
local post_status
post_status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
if [[ "$post_status" == "true" ]]; then
echo "$container restarted and running ✅"
RESTART_OK=true
else
error "$container started but crashed — check container logs"
notify "$container failed to stay running after maintenance on $(hostname)" \
"$notify_label" "warning"
fi
else
error "Failed to restart $container — start it manually"
notify "$container failed to restart after maintenance on $(hostname)" \
"$notify_label" "warning"
fi
}
# ==============================================================================================
# ── ARR CLEANUP SAFETY GATES ──────────────────────────────────────────────────────────────────
# ==============================================================================================
# Shared by lidarr/radarr/sonarr_cleanup.sh — these scripts DELETE files, so every function
# here is a direct byte-for-byte port of what was independently duplicated three times, not
# a rewrite. Callers should behave identically to before this consolidation.
# Safety Layer 1 — container running, not starting/unhealthy. Exits 1 (with notify) on failure.
# Usage: check_container_health "$LIDARR_CONTAINER" "$ARR_DOCKER_TIMEOUT" "Lidarr Cleanup"
check_container_health() {
local container="$1" docker_timeout="$2" notify_label="$3"
local running
running=$(timeout "$docker_timeout" docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
if [[ "$running" != "true" ]]; then
error "$container is not running — aborting"
notify "$notify_label aborted on $(hostname) — container not running" "$notify_label" "warning"
exit 1
fi
local health
health=$(timeout "$docker_timeout" docker inspect -f '{{.State.Health.Status}}' "$container" 2>/dev/null)
case "$health" in
healthy) info "$container is healthy" ;;
"") info "$container has no health check — proceeding" ;;
starting)
error "$container is still starting — aborting"
notify "$notify_label aborted on $(hostname) — container still starting" "$notify_label" "warning"
exit 1 ;;
unhealthy)
error "$container is unhealthy — aborting"
notify "$notify_label aborted on $(hostname) — container unhealthy" "$notify_label" "warning"
exit 1 ;;
*) warn "$container health: $health — proceeding with caution" ;;
esac
info "Safety layer 1 passed — container healthy"
}
# Safety Layer 6 — abort if tracked count dropped below min_pct of the last known count
# (protects against a misconfigured path / partial API response silently wiping the library).
# Writes the new baseline only if the check passes. Exits 1 (with notify) on failure.
# Usage: check_tracked_count_floor "$TRACKED_COUNT" "$LIDARR_TRACKED_COUNT_FILE" "$LIDARR_MIN_TRACKED_PCT" "Lidarr Cleanup"
check_tracked_count_floor() {
local tracked_count="$1" baseline_file="$2" min_pct="$3" notify_label="$4"
if [[ -f "$baseline_file" ]]; then
local last_count pct
last_count=$(cat "$baseline_file" 2>/dev/null || echo 0)
if [[ "$last_count" -gt 0 ]]; then
pct=$(awk "BEGIN {printf \"%d\", ($tracked_count / $last_count) * 100}")
if [[ "$pct" -lt "$min_pct" ]]; then
error "Tracked count dropped to ${pct}% of last run ($tracked_count vs $last_count)"
error "Suggests API issue — aborting to prevent mass deletion"
error "If expected (large library removal) delete: $baseline_file"
notify "$notify_label aborted on $(hostname) — tracked count dropped to ${pct}%" \
"$notify_label" "warning"
exit 1
fi
info "Tracked count: ${pct}% of last run ($tracked_count vs $last_count) ✅"
fi
else
info "No previous count on record — first run, saving baseline"
fi
echo "$tracked_count" > "$baseline_file"
}
# Filters --i-know-what-im-doing / --skip-age-check out of "$@" before parse_args sees them
# (both are cleanup-script-specific, not part of the shared arg parser).
# Sets I_KNOW, SKIP_AGE_CHECK, FILTERED_ARGS — call parse_args "${FILTERED_ARGS[@]}" after.
parse_destructive_flags() {
I_KNOW=false
SKIP_AGE_CHECK=false
FILTERED_ARGS=()
local arg
for arg in "$@"; do
case "$arg" in
--i-know-what-im-doing) I_KNOW=true ;;
--skip-age-check) SKIP_AGE_CHECK=true ;;
*) FILTERED_ARGS+=("$arg") ;;
esac
done
}
# Prints the nuclear-mode banner and sleeps 10s (Ctrl+C window) when both destructive flags
# are set and this isn't a dry run. Call after parse_args. No-op otherwise.
nuclear_mode_warning() {
if [[ "$I_KNOW" == true ]] && [[ "$SKIP_AGE_CHECK" == true ]] && [[ "$DRY_RUN" != true ]]; then
echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "⚠️ WARNING — NUCLEAR MODE ACTIVE"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo " Flags: --i-know-what-im-doing --skip-age-check"
echo " Age check: BYPASSED — deletes on first pass"
echo " Size threshold: BYPASSED — no GB limit"
echo " Data recovery: NOT POSSIBLE after deletion"
echo ""
echo " Review --dry-run output before proceeding."
echo " You have 10 seconds to cancel (Ctrl+C)..."
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
sleep 10
echo " Proceeding..."
echo ""
fi
}
# Safety Layer 7 — abort if total deletion size exceeds max_gb and --i-know-what-im-doing
# wasn't passed. Sets TOTAL_HUMAN. Exits 1 (with notify) on failure; warns and continues
# if I_KNOW is true (set by parse_destructive_flags).
# Usage: check_delete_size_threshold "$TOTAL_DELETE_BYTES" "$LIDARR_MAX_DELETE_GB" "Lidarr Cleanup"
check_delete_size_threshold() {
local total_bytes="$1" max_gb="$2" notify_label="$3"
local max_bytes
max_bytes=$(awk "BEGIN {printf \"%d\", $max_gb * 1073741824}")
if [[ "$total_bytes" -gt "$max_bytes" ]]; then
TOTAL_HUMAN=$(awk "BEGIN {printf \"%.1fGB\", $total_bytes / 1073741824}")
if [[ "$I_KNOW" != true ]]; then
echo ""
error "Deletion would exceed ${max_gb}GB — $TOTAL_HUMAN would be deleted"
error "Review ORPHAN lines above carefully before proceeding"
error "Rerun with: --i-know-what-im-doing"
error "To also bypass age check: add --skip-age-check"
notify "$notify_label halted on $(hostname) — ${TOTAL_HUMAN} requires --i-know-what-im-doing" \
"$notify_label" "warning"
exit 1
else
warn "OVERRIDE — deletion is $TOTAL_HUMAN — proceeding with --i-know-what-im-doing"
fi
fi
}
# True if filepath's extension (case-insensitive) matches one of the given extensions.
# Usage: has_extension "$filepath" "${LIDARR_EXTENSIONS[@]}"
has_extension() {
local filepath="$1"; shift
local ext="${filepath##*.}"
ext="${ext,,}"
local valid_ext
for valid_ext in "$@"; do
[[ "$ext" == "$valid_ext" ]] && return 0
done
return 1
}
# True if filepath's basename matches one of the given glob patterns.
# Usage: matches_pattern_list "$filepath" "${LIDARR_PROTECTED_PATTERNS[@]}"
matches_pattern_list() {
local filepath="$1"; shift
local filename
filename=$(basename "$filepath")
local pattern
for pattern in "$@"; do
# shellcheck disable=SC2254
case "$filename" in
$pattern) return 0 ;;
esac
done
return 1
}
# Builds the global ARR_PATH_MAP associative array (container path → host path) from
# ${MY_ID}_<ARR_TYPE>_PATH_MAP, for translate_path() to use. Usage: build_arr_path_map "LIDARR"
build_arr_path_map() {
local arr_type="$1"
declare -gA ARR_PATH_MAP=()
local _bapm_var="${MY_ID}_${arr_type}_PATH_MAP"
eval "for key in \"\${!${_bapm_var}[@]}\"; do
ARR_PATH_MAP[\"\$key\"]=\"\${${_bapm_var}[\$key]}\"
done"
}
# Generic arr API GET wrapper with HTTP status check.
# Usage: arr_api "$RADARR_URL" "$RADARR_API_KEY" "v3" "movie" "Radarr"
arr_api() {
local base_url="$1" api_key="$2" api_version="$3" endpoint="$4" label="${5:-Arr}"
local response http_code body
response=$(curl -sf \
--max-time 30 \
-H "X-Api-Key: $api_key" \
-w "\n%{http_code}" \
"${base_url}/api/${api_version}/${endpoint}" 2>/dev/null)
http_code=$(echo "$response" | tail -1)
body=$(echo "$response" | head -n -1)
if [[ "$http_code" != "200" ]]; then
error "$label API HTTP $http_code for: $endpoint"
return 1
fi
echo "$body"
}
# Triggers an arr "command" endpoint with the given JSON payload, then polls every 10s until
# completed/failed/timeout, logging progress every 60s. Best-effort pre-flight — never treats
# an unreachable/timed-out scan as fatal, matching original per-script behavior.
# Usage: trigger_and_await_command "$LIDARR_URL" "$LIDARR_API_KEY" "v1" \
# '{"name": "DownloadedAlbumsScan"}' "${LIDARR_IMPORT_SCAN_TIMEOUT:-600}"
trigger_and_await_command() {
local base_url="$1" api_key="$2" api_version="$3" payload="$4" poll_timeout="${5:-600}"
local scan_response scan_cmd_id
scan_response=$(curl -sf --max-time 30 -X POST \
-H "X-Api-Key: $api_key" \
-H "Content-Type: application/json" \
-d "$payload" \
"${base_url}/api/${api_version}/command" 2>/dev/null)
scan_cmd_id=$(echo "$scan_response" | jq -r '.id // empty' 2>/dev/null)
if [[ -z "$scan_cmd_id" ]]; then
warn "Could not trigger import scan — proceeding without pre-flight"
return 0
fi
info "Import scan queued (command ID: $scan_cmd_id) — waiting for completion..."
local polled=0 scan_status
while [[ "$polled" -lt "$poll_timeout" ]]; do
scan_status=$(curl -sf --max-time 10 \
-H "X-Api-Key: $api_key" \
"${base_url}/api/${api_version}/command/${scan_cmd_id}" 2>/dev/null | \
jq -r '.status // empty' 2>/dev/null)
case "$scan_status" in
completed) info "Import scan complete ✅"; return 0 ;;
failed) warn "Import scan reported failed — proceeding anyway"; return 0 ;;
esac
sleep 10
(( polled += 10 ))
[[ $(( polled % 60 )) -eq 0 ]] && log " Still scanning... (${polled}s elapsed)"
done
warn "Import scan timed out after ${poll_timeout}s — proceeding anyway"
return 0
}
# ==============================================================================================
# ── ARR VERSION CHECK ─────────────────────────────────────────────────────────────────────────
# ==============================================================================================
@@ -1692,6 +2244,48 @@ check_arr_version() {
fi
}
# ==============================================================================================
# ── TMDB DISCOVERY SCORING ────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Shared by playback_aware_radarr_discovery.sh and playback_aware_sonarr_discovery.sh — stage 2
# candidate scoring against TMDB rating/vote/seed-breadth data. Byte-for-byte identical between
# the two before this consolidation.
# Formats a TMDB vote_average×10 integer back to one decimal place (e.g. 78 → 7.8).
_fmt_rating() { local v="${1:-0}"; echo "${v::-1}.${v: -1}" 2>/dev/null || echo "$v"; }
# Stage 2: TMDB vote_average × 10 as integer (0-40)
_rating_score_s2() {
local v="$1"
if (( v >= 80 )); then echo 40
elif (( v >= 75 )); then echo 32
elif (( v >= 70 )); then echo 25
elif (( v >= 65 )); then echo 18
elif (( v >= 60 )); then echo 12
else echo 5
fi
}
# Stage 2: vote count (0-20)
_votes_score() {
local c="$1"
if (( c >= 10000 )); then echo 20
elif (( c >= 5000 )); then echo 15
elif (( c >= 1000 )); then echo 10
elif (( c >= 200 )); then echo 5
else echo 2
fi
}
# Stage 2: seed breadth (0-40)
_breadth_score() {
local seeds="$1"
if (( seeds >= 3 )); then echo 40
elif (( seeds == 2 )); then echo 25
else echo 10
fi
}
# ==============================================================================================
# ── EMBY LIBRARY SCAN ─────────────────────────────────────────────────────────────────────────
# ==============================================================================================
@@ -1911,54 +2505,6 @@ check_remote_docker_daemon() {
return 1
}
# ==============================================================================================
# ── UNRAID COMMAND VALIDATION ─────────────────────────────────────────────────────────────────
# ==============================================================================================
# Verifies a unRAID-specific command exists and produces expected output before use.
# Protects against commands moving, changing format, or disappearing after upgrades.
#
# If validation fails — notifies and returns 1. Caller exits its own section only.
# Other scripts that pass validation are unaffected.
#
# Usage:
# validate_unraid_cmd # "/usr/local/emhttp/plugins/dynamix/scripts/notify" # "--help" # "Usage" # "unRAID notify script"
#
# Arguments:
# $1 = full path to command
# $2 = test argument to pass (use "" for no argument)
# $3 = expected string in output (use "" to skip output check)
# $4 = human readable name for error messages
validate_unraid_cmd() {
local cmd_path="$1"
local test_arg="$2"
local expected_output="$3"
local cmd_name="${4:-$1}"
# Check command exists and is executable
if [[ ! -x "$cmd_path" ]]; then
error "unRAID command not found or not executable: $cmd_path"
error "$cmd_name may have moved or been removed — check after recent unRAID upgrade"
notify "$cmd_name not found on $(hostname) at $cmd_path — check after unRAID upgrade" "Command Validation" "warning"
return 1
fi
# Check output matches expected pattern if provided
if [[ -n "$expected_output" ]]; then
local actual_output
actual_output=$(timeout 10 "$cmd_path" $test_arg 2>&1 || true)
if ! echo "$actual_output" | grep -q "$expected_output"; then
error "$cmd_name output format changed — expected '$expected_output' not found"
error "Command may have changed after unRAID upgrade — review $cmd_path"
notify "$cmd_name output format changed on $(hostname) — may need script update after unRAID upgrade" "Command Validation" "warning"
return 1
fi
fi
log "$cmd_name validated: $cmd_path ✅"
return 0
}
# ==============================================================================================
# ── STATUS DISPLAY ────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
load_config.sh
+2 -1
View File
@@ -135,7 +135,8 @@
CONF_DIR="${SCRIPTS_DIR}/Configurations"
LOG_DIR="/var/log/varaverk"
VV_CACHE_DIR="/tmp/vv_cache"
export CONF_DIR LOG_DIR VV_CACHE_DIR
CONF_RAM_CACHE_DIR="/tmp/.cache/vv/d" # tmpfs — cleared every reboot, repopulated by conf_sync.sh
export CONF_DIR LOG_DIR VV_CACHE_DIR CONF_RAM_CACHE_DIR
# ━━━ Cleanup ━━━
unset _conf _host_confs_loaded _adapter LOAD_CONFIG_DIR