Fix dead/incorrect vars and consolidate duplicated logic into common.sh
Codebase-wide audit pass: fixed real bugs (SSH hangs missing BatchMode, local-outside-function no-ops, variable name collisions, a truncated ratio calc, wrong state-dir path, DARK vs NO_INTERNET drift, and more), then pulled logic that was duplicated across multiple scripts — arr cleanup safety gates, docker restart ordering, container maintenance stop/restart, watchdog state-file helpers, partnership role resolution, cert expiry checks, remote node discovery, and TMDB discovery scoring — into common.sh so each now has a single implementation.
This commit is contained in:
+4
-10
@@ -206,13 +206,7 @@ declare -A _ID_TYPE=([lidarr]="string" [sonarr]="int" [radarr]="int")
|
|||||||
ARR_TYPES=(lidarr sonarr radarr)
|
ARR_TYPES=(lidarr sonarr radarr)
|
||||||
|
|
||||||
# ── Remote node discovery ──────────────────────────────────────────────────────────────────────
|
# ── Remote node discovery ──────────────────────────────────────────────────────────────────────
|
||||||
REMOTE_NODES=()
|
discover_remote_nodes
|
||||||
for _hv in HOST1 HOST2 HOST3 HOST4 HOST5 HOST6 HOST7 HOST8; do
|
|
||||||
[[ "$_hv" == "$MY_ID" ]] && continue
|
|
||||||
[[ -z "${!_hv:-}" ]] && continue
|
|
||||||
REMOTE_NODES+=("$_hv")
|
|
||||||
done
|
|
||||||
unset _hv
|
|
||||||
|
|
||||||
if [[ "${#REMOTE_NODES[@]}" -eq 0 ]]; then
|
if [[ "${#REMOTE_NODES[@]}" -eq 0 ]]; then
|
||||||
warn "No remote nodes defined in master.conf — nothing to sync"
|
warn "No remote nodes defined in master.conf — nothing to sync"
|
||||||
@@ -478,7 +472,7 @@ if [[ "$SHOW_STATUS" == true ]]; then
|
|||||||
echo ""
|
echo ""
|
||||||
echo " Arr Port URL"
|
echo " Arr Port URL"
|
||||||
for _arr in "${ARR_TYPES[@]}"; do
|
for _arr in "${ARR_TYPES[@]}"; do
|
||||||
local _url="" _key=""
|
_url=""
|
||||||
case "$_arr" in
|
case "$_arr" in
|
||||||
lidarr) _url="${LIDARR_URL:-not configured}" ;;
|
lidarr) _url="${LIDARR_URL:-not configured}" ;;
|
||||||
sonarr) _url="${SONARR_URL:-not configured}" ;;
|
sonarr) _url="${SONARR_URL:-not configured}" ;;
|
||||||
@@ -992,8 +986,8 @@ _sync_arr() {
|
|||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
local _n_local=${#to_add_local[@]:-}; _n_local=${_n_local:-0}
|
local _n_local=${#to_add_local[@]}
|
||||||
local _n_remote=${#to_add_remote[@]:-}; _n_remote=${_n_remote:-0}
|
local _n_remote=${#to_add_remote[@]}
|
||||||
log " $node_name: +${_n_local} local | +${_n_remote} remote | $total_skipped blocklisted"
|
log " $node_name: +${_n_local} local | +${_n_remote} remote | $total_skipped blocklisted"
|
||||||
|
|
||||||
unset remote_ids to_add_local to_add_remote _n_local _n_remote
|
unset remote_ids to_add_local to_add_remote _n_local _n_remote
|
||||||
|
|||||||
+17
-187
@@ -60,7 +60,7 @@
|
|||||||
#
|
#
|
||||||
# acquire_lock "wait" — large scans take time, wait for previous run to finish
|
# acquire_lock "wait" — large scans take time, wait for previous run to finish
|
||||||
# jq + curl validation — exits if either tool missing
|
# jq + curl validation — exits if either tool missing
|
||||||
# DOCKER_TIMEOUT — container checks protected against daemon hangs
|
# ARR_DOCKER_TIMEOUT — container checks protected against daemon hangs (script-local, not common.sh's DOCKER_TIMEOUT)
|
||||||
# Duplicate detection — temp file of tracked paths, grep before delete
|
# Duplicate detection — temp file of tracked paths, grep before delete
|
||||||
# platform_require_cmd — notify script validated before use
|
# platform_require_cmd — notify script validated before use
|
||||||
# Silent by default — orphans/junk warn(), clean library logs silently
|
# Silent by default — orphans/junk warn(), clean library logs silently
|
||||||
@@ -120,37 +120,12 @@ source "$SCRIPT_DIR/../load_config.sh"
|
|||||||
# ── Special flag pre-processing ───────────────────────────────────────────────────────────────
|
# ── Special flag pre-processing ───────────────────────────────────────────────────────────────
|
||||||
# Filter --i-know-what-im-doing and --skip-age-check before parse_args
|
# Filter --i-know-what-im-doing and --skip-age-check before parse_args
|
||||||
# to avoid unknown flag errors — both are handled separately below.
|
# to avoid unknown flag errors — both are handled separately below.
|
||||||
I_KNOW=false
|
parse_destructive_flags "$@"
|
||||||
SKIP_AGE_CHECK=false
|
|
||||||
FILTERED_ARGS=()
|
|
||||||
for arg in "$@"; do
|
|
||||||
case "$arg" in
|
|
||||||
--i-know-what-im-doing) I_KNOW=true ;;
|
|
||||||
--skip-age-check) SKIP_AGE_CHECK=true ;;
|
|
||||||
*) FILTERED_ARGS+=("$arg") ;;
|
|
||||||
esac
|
|
||||||
done
|
|
||||||
|
|
||||||
parse_args "${FILTERED_ARGS[@]}"
|
parse_args "${FILTERED_ARGS[@]}"
|
||||||
|
|
||||||
# ── Nuclear mode warning ──────────────────────────────────────────────────────────────────────
|
# ── Nuclear mode warning ──────────────────────────────────────────────────────────────────────
|
||||||
if [[ "$I_KNOW" == true ]] && [[ "$SKIP_AGE_CHECK" == true ]] && [[ "$DRY_RUN" != true ]]; then
|
nuclear_mode_warning
|
||||||
echo ""
|
|
||||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
||||||
echo "⚠️ WARNING — NUCLEAR MODE ACTIVE"
|
|
||||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
||||||
echo " Flags: --i-know-what-im-doing --skip-age-check"
|
|
||||||
echo " Age check: BYPASSED — deletes on first pass"
|
|
||||||
echo " Size threshold: BYPASSED — no GB limit"
|
|
||||||
echo " Data recovery: NOT POSSIBLE after deletion"
|
|
||||||
echo ""
|
|
||||||
echo " Review --dry-run output before proceeding."
|
|
||||||
echo " You have 10 seconds to cancel (Ctrl+C)..."
|
|
||||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
||||||
sleep 10
|
|
||||||
echo " Proceeding..."
|
|
||||||
echo ""
|
|
||||||
fi
|
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Setup ━━━
|
# ━━━ Setup ━━━
|
||||||
@@ -194,15 +169,11 @@ if [[ -z "${LIDARR_URL:-}" ]] || [[ -z "${LIDARR_API_KEY:-}" ]]; then
|
|||||||
exit 0
|
exit 0
|
||||||
fi
|
fi
|
||||||
|
|
||||||
DOCKER_TIMEOUT=15
|
ARR_DOCKER_TIMEOUT=15
|
||||||
LIDARR_CONTAINER="Lidarr" # container name on HOST1
|
LIDARR_CONTAINER="Lidarr" # container name on HOST1
|
||||||
|
|
||||||
# Build path map from MY_ID's Lidarr path map
|
# Build path map from MY_ID's Lidarr path map
|
||||||
declare -A ARR_PATH_MAP
|
build_arr_path_map "LIDARR"
|
||||||
local_path_map_var="${MY_ID}_LIDARR_PATH_MAP"
|
|
||||||
eval "for key in \"\${!${local_path_map_var}[@]}\"; do
|
|
||||||
ARR_PATH_MAP[\"\$key\"]=\"\${${local_path_map_var}[\$key]}\"
|
|
||||||
done"
|
|
||||||
|
|
||||||
# Validate required vars — detect_hosts() should have set these
|
# Validate required vars — detect_hosts() should have set these
|
||||||
require_var LIDARR_URL
|
require_var LIDARR_URL
|
||||||
@@ -251,83 +222,13 @@ fi
|
|||||||
echo ""
|
echo ""
|
||||||
echo "━━━ $ICON_SHIELD Safety Checks ━━━"
|
echo "━━━ $ICON_SHIELD Safety Checks ━━━"
|
||||||
|
|
||||||
CONTAINER_RUNNING=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
|
check_container_health "$LIDARR_CONTAINER" "$ARR_DOCKER_TIMEOUT" "Lidarr Cleanup"
|
||||||
'{{.State.Running}}' "$LIDARR_CONTAINER" 2>/dev/null)
|
|
||||||
if [[ "$CONTAINER_RUNNING" != "true" ]]; then
|
|
||||||
error "$LIDARR_CONTAINER is not running — aborting"
|
|
||||||
notify "Lidarr cleanup aborted on $(hostname) — container not running" \
|
|
||||||
"Lidarr Cleanup" "warning"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
CONTAINER_HEALTH=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
|
|
||||||
'{{.State.Health.Status}}' "$LIDARR_CONTAINER" 2>/dev/null)
|
|
||||||
case "$CONTAINER_HEALTH" in
|
|
||||||
healthy) info "$LIDARR_CONTAINER is healthy" ;;
|
|
||||||
"") info "$LIDARR_CONTAINER has no health check — proceeding" ;;
|
|
||||||
starting)
|
|
||||||
error "$LIDARR_CONTAINER is still starting — aborting"
|
|
||||||
notify "Lidarr cleanup aborted on $(hostname) — container still starting" \
|
|
||||||
"Lidarr Cleanup" "warning"
|
|
||||||
exit 1 ;;
|
|
||||||
unhealthy)
|
|
||||||
error "$LIDARR_CONTAINER is unhealthy — aborting"
|
|
||||||
notify "Lidarr cleanup aborted on $(hostname) — container unhealthy" \
|
|
||||||
"Lidarr Cleanup" "warning"
|
|
||||||
exit 1 ;;
|
|
||||||
*) warn "$LIDARR_CONTAINER health: $CONTAINER_HEALTH — proceeding with caution" ;;
|
|
||||||
esac
|
|
||||||
|
|
||||||
info "Safety layer 1 passed — container healthy"
|
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ── HELPER FUNCTIONS ──────────────────────────────────────────────────────────────────────────
|
# ── HELPER FUNCTIONS ──────────────────────────────────────────────────────────────────────────
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
|
|
||||||
# Lidarr API call with HTTP status check
|
# check_container_health(), arr_api(), has_extension(), matches_pattern_list() — common.sh
|
||||||
# Usage: lidarr_api "artist" | lidarr_api "trackFile?artistId=123"
|
|
||||||
lidarr_api() {
|
|
||||||
local endpoint="$1"
|
|
||||||
local response http_code body
|
|
||||||
|
|
||||||
response=$(curl -sf \
|
|
||||||
--max-time 30 \
|
|
||||||
-H "X-Api-Key: $LIDARR_API_KEY" \
|
|
||||||
-w "\n%{http_code}" \
|
|
||||||
"${LIDARR_URL}/api/v1/${endpoint}" 2>/dev/null)
|
|
||||||
|
|
||||||
http_code=$(echo "$response" | tail -1)
|
|
||||||
body=$(echo "$response" | head -n -1)
|
|
||||||
|
|
||||||
if [[ "$http_code" != "200" ]]; then
|
|
||||||
error "Lidarr API HTTP $http_code for: $endpoint"
|
|
||||||
return 1
|
|
||||||
fi
|
|
||||||
echo "$body"
|
|
||||||
}
|
|
||||||
|
|
||||||
# Check if a file extension is a tracked music format
|
|
||||||
is_music_file() {
|
|
||||||
local ext="${1##*.}"
|
|
||||||
ext="${ext,,}"
|
|
||||||
for valid_ext in "${LIDARR_EXTENSIONS[@]}"; do
|
|
||||||
[[ "$ext" == "$valid_ext" ]] && return 0
|
|
||||||
done
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|
||||||
# Check if a file matches any protected pattern
|
|
||||||
is_protected_file() {
|
|
||||||
local filename
|
|
||||||
filename=$(basename "$1")
|
|
||||||
for pattern in "${LIDARR_PROTECTED_PATTERNS[@]}"; do
|
|
||||||
# shellcheck disable=SC2254
|
|
||||||
case "$filename" in
|
|
||||||
$pattern) return 0 ;;
|
|
||||||
esac
|
|
||||||
done
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Pre-flight: Lidarr Import Scan ━━━
|
# ━━━ Pre-flight: Lidarr Import Scan ━━━
|
||||||
@@ -353,36 +254,7 @@ else
|
|||||||
SCAN_PAYLOAD='{"name": "DownloadedAlbumsScan"}'
|
SCAN_PAYLOAD='{"name": "DownloadedAlbumsScan"}'
|
||||||
fi
|
fi
|
||||||
|
|
||||||
SCAN_RESPONSE=$(curl -sf --max-time 30 -X POST \
|
trigger_and_await_command "$LIDARR_URL" "$LIDARR_API_KEY" "v1" "$SCAN_PAYLOAD" "${LIDARR_IMPORT_SCAN_TIMEOUT:-600}"
|
||||||
-H "X-Api-Key: $LIDARR_API_KEY" \
|
|
||||||
-H "Content-Type: application/json" \
|
|
||||||
-d "$SCAN_PAYLOAD" \
|
|
||||||
"${LIDARR_URL}/api/v1/command" 2>/dev/null)
|
|
||||||
|
|
||||||
SCAN_CMD_ID=$(echo "$SCAN_RESPONSE" | jq -r '.id // empty' 2>/dev/null)
|
|
||||||
|
|
||||||
if [[ -z "$SCAN_CMD_ID" ]]; then
|
|
||||||
warn "Could not trigger import scan — proceeding without pre-flight"
|
|
||||||
else
|
|
||||||
info "Import scan queued (command ID: $SCAN_CMD_ID) — waiting for completion..."
|
|
||||||
POLL_TIMEOUT=${LIDARR_IMPORT_SCAN_TIMEOUT:-600}
|
|
||||||
POLLED=0
|
|
||||||
while [[ "$POLLED" -lt "$POLL_TIMEOUT" ]]; do
|
|
||||||
SCAN_STATUS=$(curl -sf --max-time 10 \
|
|
||||||
-H "X-Api-Key: $LIDARR_API_KEY" \
|
|
||||||
"${LIDARR_URL}/api/v1/command/${SCAN_CMD_ID}" 2>/dev/null | \
|
|
||||||
jq -r '.status // empty' 2>/dev/null)
|
|
||||||
case "$SCAN_STATUS" in
|
|
||||||
completed) info "Import scan complete ✅"; break ;;
|
|
||||||
failed) warn "Import scan reported failed — proceeding anyway"; break ;;
|
|
||||||
esac
|
|
||||||
sleep 10
|
|
||||||
(( POLLED += 10 ))
|
|
||||||
[[ $(( POLLED % 60 )) -eq 0 ]] && log " Still scanning... (${POLLED}s elapsed)"
|
|
||||||
done
|
|
||||||
[[ "$POLLED" -ge "$POLL_TIMEOUT" ]] && \
|
|
||||||
warn "Import scan timed out after ${POLL_TIMEOUT}s — proceeding anyway"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Fetch Lidarr Tracked Files ━━━
|
# ━━━ Fetch Lidarr Tracked Files ━━━
|
||||||
@@ -402,7 +274,7 @@ check_arr_version "$LIDARR_URL" "$LIDARR_API_KEY" "v1" "$LIDARR_VERSION_MAJOR" "
|
|||||||
info "Querying Lidarr API..."
|
info "Querying Lidarr API..."
|
||||||
|
|
||||||
# Fetch all artists
|
# Fetch all artists
|
||||||
ARTIST_RESPONSE=$(lidarr_api "artist") || {
|
ARTIST_RESPONSE=$(arr_api "$LIDARR_URL" "$LIDARR_API_KEY" "v1" "artist" "Lidarr") || {
|
||||||
error "Failed to fetch artists from Lidarr"
|
error "Failed to fetch artists from Lidarr"
|
||||||
notify "Lidarr cleanup failed on $(hostname) — could not fetch artists" \
|
notify "Lidarr cleanup failed on $(hostname) — could not fetch artists" \
|
||||||
"Lidarr Cleanup" "warning"
|
"Lidarr Cleanup" "warning"
|
||||||
@@ -427,7 +299,7 @@ TRACKED_FILE="$TMP_DIR/tracked_paths.txt"
|
|||||||
|
|
||||||
while IFS= read -r artist_id; do
|
while IFS= read -r artist_id; do
|
||||||
[[ -z "$artist_id" ]] && continue
|
[[ -z "$artist_id" ]] && continue
|
||||||
ARTIST_TRACKS=$(lidarr_api "trackFile?artistId=${artist_id}" 2>/dev/null)
|
ARTIST_TRACKS=$(arr_api "$LIDARR_URL" "$LIDARR_API_KEY" "v1" "trackFile?artistId=${artist_id}" "Lidarr" 2>/dev/null)
|
||||||
if [[ -n "$ARTIST_TRACKS" ]]; then
|
if [[ -n "$ARTIST_TRACKS" ]]; then
|
||||||
while IFS= read -r api_path; do
|
while IFS= read -r api_path; do
|
||||||
[[ -z "$api_path" ]] && continue
|
[[ -z "$api_path" ]] && continue
|
||||||
@@ -459,25 +331,7 @@ fi
|
|||||||
info "$ARTIST_COUNT artists | $TRACKED_COUNT tracked files"
|
info "$ARTIST_COUNT artists | $TRACKED_COUNT tracked files"
|
||||||
|
|
||||||
# Safety Layer 6 — percentage drop vs last known count
|
# Safety Layer 6 — percentage drop vs last known count
|
||||||
if [[ -f "$LIDARR_TRACKED_COUNT_FILE" ]]; then
|
check_tracked_count_floor "$TRACKED_COUNT" "$LIDARR_TRACKED_COUNT_FILE" "$LIDARR_MIN_TRACKED_PCT" "Lidarr Cleanup"
|
||||||
LAST_COUNT=$(cat "$LIDARR_TRACKED_COUNT_FILE" 2>/dev/null || echo 0)
|
|
||||||
if [[ "$LAST_COUNT" -gt 0 ]]; then
|
|
||||||
PCT=$(awk "BEGIN {printf \"%d\", ($TRACKED_COUNT / $LAST_COUNT) * 100}")
|
|
||||||
if [[ "$PCT" -lt "$LIDARR_MIN_TRACKED_PCT" ]]; then
|
|
||||||
error "Tracked count dropped to ${PCT}% of last run ($TRACKED_COUNT vs $LAST_COUNT)"
|
|
||||||
error "Suggests API issue — aborting to prevent mass deletion"
|
|
||||||
error "If expected (large library removal) delete: $LIDARR_TRACKED_COUNT_FILE"
|
|
||||||
notify "Lidarr cleanup aborted on $(hostname) — tracked count dropped to ${PCT}%" \
|
|
||||||
"Lidarr Cleanup" "warning"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
info "Tracked count: ${PCT}% of last run ($TRACKED_COUNT vs $LAST_COUNT) ✅"
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
info "No previous count on record — first run, saving baseline"
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "$TRACKED_COUNT" > "$LIDARR_TRACKED_COUNT_FILE"
|
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Scan Music Root ━━━
|
# ━━━ Scan Music Root ━━━
|
||||||
@@ -496,7 +350,6 @@ JUNK_BYTES=0
|
|||||||
|
|
||||||
AGE_SECONDS=$(( LIDARR_ORPHAN_AGE * 86400 ))
|
AGE_SECONDS=$(( LIDARR_ORPHAN_AGE * 86400 ))
|
||||||
NOW=$(date +%s)
|
NOW=$(date +%s)
|
||||||
MAX_DELETE_BYTES=$(awk "BEGIN {printf \"%d\", $LIDARR_MAX_DELETE_GB * 1073741824}")
|
|
||||||
|
|
||||||
while IFS= read -r filepath; do
|
while IFS= read -r filepath; do
|
||||||
[[ -z "$filepath" ]] && continue
|
[[ -z "$filepath" ]] && continue
|
||||||
@@ -508,7 +361,7 @@ while IFS= read -r filepath; do
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
# Protected — never delete
|
# Protected — never delete
|
||||||
if is_protected_file "$filepath"; then
|
if matches_pattern_list "$filepath" "${LIDARR_PROTECTED_PATTERNS[@]}"; then
|
||||||
log "$ICON_PROTECTED PROTECTED: $filepath"
|
log "$ICON_PROTECTED PROTECTED: $filepath"
|
||||||
(( PROTECTED_COUNT++ ))
|
(( PROTECTED_COUNT++ ))
|
||||||
continue
|
continue
|
||||||
@@ -516,7 +369,7 @@ while IFS= read -r filepath; do
|
|||||||
|
|
||||||
FILE_SIZE=$(stat -c%s "$filepath" 2>/dev/null || echo 0)
|
FILE_SIZE=$(stat -c%s "$filepath" 2>/dev/null || echo 0)
|
||||||
|
|
||||||
if is_music_file "$filepath"; then
|
if has_extension "$filepath" "${LIDARR_EXTENSIONS[@]}"; then
|
||||||
FILE_MTIME=$(stat -c %Y "$filepath" 2>/dev/null || echo 0)
|
FILE_MTIME=$(stat -c %Y "$filepath" 2>/dev/null || echo 0)
|
||||||
FILE_AGE=$(( NOW - FILE_MTIME ))
|
FILE_AGE=$(( NOW - FILE_MTIME ))
|
||||||
|
|
||||||
@@ -543,21 +396,7 @@ TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT ))
|
|||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Safety Layer 7 — Deletion Size Threshold ━━━
|
# ━━━ Safety Layer 7 — Deletion Size Threshold ━━━
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
if [[ "$TOTAL_DELETE_BYTES" -gt "$MAX_DELETE_BYTES" ]]; then
|
check_delete_size_threshold "$TOTAL_DELETE_BYTES" "$LIDARR_MAX_DELETE_GB" "Lidarr Cleanup"
|
||||||
TOTAL_HUMAN=$(awk "BEGIN {printf \"%.1fGB\", $TOTAL_DELETE_BYTES / 1073741824}")
|
|
||||||
if [[ "$I_KNOW" != true ]]; then
|
|
||||||
echo ""
|
|
||||||
error "Deletion would exceed ${LIDARR_MAX_DELETE_GB}GB — $TOTAL_HUMAN would be deleted"
|
|
||||||
error "Review ORPHAN lines above carefully before proceeding"
|
|
||||||
error "Rerun with: --i-know-what-im-doing"
|
|
||||||
error "To also bypass age check: add --skip-age-check"
|
|
||||||
notify "Lidarr cleanup halted on $(hostname) — ${TOTAL_HUMAN} requires --i-know-what-im-doing" \
|
|
||||||
"Lidarr Cleanup" "warning"
|
|
||||||
exit 1
|
|
||||||
else
|
|
||||||
warn "OVERRIDE — deletion is $TOTAL_HUMAN — proceeding with --i-know-what-im-doing"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
# ── Execute Deletions ─────────────────────────────────────────────────────────────────────────
|
# ── Execute Deletions ─────────────────────────────────────────────────────────────────────────
|
||||||
# All safety layers passed — delete orphans and junk
|
# All safety layers passed — delete orphans and junk
|
||||||
@@ -565,12 +404,12 @@ if [[ "$DRY_RUN" == false ]]; then
|
|||||||
while IFS= read -r filepath; do
|
while IFS= read -r filepath; do
|
||||||
[[ -z "$filepath" ]] && continue
|
[[ -z "$filepath" ]] && continue
|
||||||
[[ -n "${TRACKED_MAP[$filepath]:-}" ]] && continue
|
[[ -n "${TRACKED_MAP[$filepath]:-}" ]] && continue
|
||||||
is_protected_file "$filepath" && continue
|
matches_pattern_list "$filepath" "${LIDARR_PROTECTED_PATTERNS[@]}" && continue
|
||||||
|
|
||||||
FILE_MTIME=$(stat -c %Y "$filepath" 2>/dev/null || echo 0)
|
FILE_MTIME=$(stat -c %Y "$filepath" 2>/dev/null || echo 0)
|
||||||
FILE_AGE=$(( NOW - FILE_MTIME ))
|
FILE_AGE=$(( NOW - FILE_MTIME ))
|
||||||
|
|
||||||
if is_music_file "$filepath"; then
|
if has_extension "$filepath" "${LIDARR_EXTENSIONS[@]}"; then
|
||||||
[[ "$FILE_AGE" -lt "$AGE_SECONDS" ]] && \
|
[[ "$FILE_AGE" -lt "$AGE_SECONDS" ]] && \
|
||||||
[[ "$SKIP_AGE_CHECK" != true ]] && continue
|
[[ "$SKIP_AGE_CHECK" != true ]] && continue
|
||||||
fi
|
fi
|
||||||
@@ -586,16 +425,7 @@ fi
|
|||||||
|
|
||||||
END=$(date +%s)
|
END=$(date +%s)
|
||||||
|
|
||||||
format_bytes() {
|
# format_bytes() — provided by common.sh
|
||||||
local bytes=$1
|
|
||||||
if (( bytes > 1073741824 )); then
|
|
||||||
awk "BEGIN {printf \"%.1fGB\", $bytes / 1073741824}"
|
|
||||||
elif (( bytes > 1048576 )); then
|
|
||||||
awk "BEGIN {printf \"%.1fMB\", $bytes / 1048576}"
|
|
||||||
else
|
|
||||||
echo "${bytes}B"
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
ORPHAN_HUMAN=$(format_bytes "$ORPHAN_BYTES")
|
ORPHAN_HUMAN=$(format_bytes "$ORPHAN_BYTES")
|
||||||
JUNK_HUMAN=$(format_bytes "$JUNK_BYTES")
|
JUNK_HUMAN=$(format_bytes "$JUNK_BYTES")
|
||||||
|
|||||||
@@ -283,17 +283,6 @@ read_file_mbid() {
|
|||||||
esac
|
esac
|
||||||
}
|
}
|
||||||
|
|
||||||
translate_container_path() {
|
|
||||||
local cpath="$1"
|
|
||||||
for cp in "${!ARR_PATH_MAP[@]}"; do
|
|
||||||
if [[ "$cpath" == "${cp}"* ]]; then
|
|
||||||
echo "${ARR_PATH_MAP[$cp]}${cpath#$cp}"
|
|
||||||
return
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
echo "$cpath"
|
|
||||||
}
|
|
||||||
|
|
||||||
# ── Safety checks ─────────────────────────────────────────────────────────────────────────────
|
# ── Safety checks ─────────────────────────────────────────────────────────────────────────────
|
||||||
echo ""
|
echo ""
|
||||||
echo "━━━ $ICON_SHIELD Safety Checks ━━━"
|
echo "━━━ $ICON_SHIELD Safety Checks ━━━"
|
||||||
@@ -322,7 +311,7 @@ while IFS= read -r artist; do
|
|||||||
aid=$(echo "$artist" | jq -r '.id')
|
aid=$(echo "$artist" | jq -r '.id')
|
||||||
apath=$(echo "$artist" | jq -r '.path // empty')
|
apath=$(echo "$artist" | jq -r '.path // empty')
|
||||||
aname=$(echo "$artist" | jq -r '.artistName // empty')
|
aname=$(echo "$artist" | jq -r '.artistName // empty')
|
||||||
[[ -n "$apath" ]] && ARTIST_PATH_CACHE[$aid]=$(translate_container_path "$apath")
|
[[ -n "$apath" ]] && ARTIST_PATH_CACHE[$aid]=$(translate_path "$apath")
|
||||||
[[ -n "$aname" ]] && ARTIST_NAME_CACHE[$aid]="$aname"
|
[[ -n "$aname" ]] && ARTIST_NAME_CACHE[$aid]="$aname"
|
||||||
done < <(echo "$ALL_ARTISTS" | jq -c '.[]' 2>/dev/null)
|
done < <(echo "$ALL_ARTISTS" | jq -c '.[]' 2>/dev/null)
|
||||||
unset ALL_ARTISTS
|
unset ALL_ARTISTS
|
||||||
|
|||||||
@@ -134,7 +134,6 @@
|
|||||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
|
||||||
source "$SCRIPT_DIR/../load_config.sh"
|
source "$SCRIPT_DIR/../load_config.sh"
|
||||||
source "$SCRIPT_DIR/../Kernel/decision_engine.sh"
|
|
||||||
|
|
||||||
parse_args "$@"
|
parse_args "$@"
|
||||||
|
|
||||||
@@ -185,7 +184,7 @@ log "$ICON_GEAR Config: threshold=${THRESHOLD} lookback=${LOOKBACK_DAYS}d max-se
|
|||||||
|
|
||||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no movies will be added to Radarr"
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no movies will be added to Radarr"
|
||||||
|
|
||||||
_fmt_rating() { local v="${1:-0}"; echo "${v::-1}.${v: -1}" 2>/dev/null || echo "$v"; }
|
# _fmt_rating() — provided by common.sh
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Status ━━━
|
# ━━━ Status ━━━
|
||||||
@@ -275,35 +274,7 @@ _freq_score() {
|
|||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
# vote_avg_int = vote_average × 10 as integer (e.g. 7.8 → 78)
|
# _rating_score_s2(), _votes_score(), _breadth_score() — provided by common.sh
|
||||||
_rating_score_s2() {
|
|
||||||
local v="$1"
|
|
||||||
if (( v >= 80 )); then echo 40
|
|
||||||
elif (( v >= 75 )); then echo 32
|
|
||||||
elif (( v >= 70 )); then echo 25
|
|
||||||
elif (( v >= 65 )); then echo 18
|
|
||||||
elif (( v >= 60 )); then echo 12
|
|
||||||
else echo 5
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
_votes_score() {
|
|
||||||
local c="$1"
|
|
||||||
if (( c >= 10000 )); then echo 20
|
|
||||||
elif (( c >= 5000 )); then echo 15
|
|
||||||
elif (( c >= 1000 )); then echo 10
|
|
||||||
elif (( c >= 200 )); then echo 5
|
|
||||||
else echo 2
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
_breadth_score() {
|
|
||||||
local seeds="$1"
|
|
||||||
if (( seeds >= 3 )); then echo 40
|
|
||||||
elif (( seeds == 2 )); then echo 25
|
|
||||||
else echo 10
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Fetch Emby Libraries + Recently Watched Movies ━━━
|
# ━━━ Fetch Emby Libraries + Recently Watched Movies ━━━
|
||||||
|
|||||||
@@ -148,7 +148,6 @@
|
|||||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
|
||||||
source "$SCRIPT_DIR/../load_config.sh"
|
source "$SCRIPT_DIR/../load_config.sh"
|
||||||
source "$SCRIPT_DIR/../Kernel/decision_engine.sh"
|
|
||||||
|
|
||||||
parse_args "$@"
|
parse_args "$@"
|
||||||
|
|
||||||
@@ -205,7 +204,7 @@ log "$ICON_GEAR Config: threshold=${THRESHOLD} lookback=${LOOKBACK_DAYS}d max-se
|
|||||||
|
|
||||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no shows will be added to Sonarr"
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no shows will be added to Sonarr"
|
||||||
|
|
||||||
_fmt_rating() { local v="${1:-0}"; echo "${v::-1}.${v: -1}" 2>/dev/null || echo "$v"; }
|
# _fmt_rating() — provided by common.sh
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Status ━━━
|
# ━━━ Status ━━━
|
||||||
@@ -315,37 +314,7 @@ _volume_score() {
|
|||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
# Stage 2: TMDB vote_average × 10 (0-40)
|
# _rating_score_s2(), _votes_score(), _breadth_score() — provided by common.sh
|
||||||
_rating_score_s2() {
|
|
||||||
local v="$1"
|
|
||||||
if (( v >= 80 )); then echo 40
|
|
||||||
elif (( v >= 75 )); then echo 32
|
|
||||||
elif (( v >= 70 )); then echo 25
|
|
||||||
elif (( v >= 65 )); then echo 18
|
|
||||||
elif (( v >= 60 )); then echo 12
|
|
||||||
else echo 5
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
# Stage 2: vote count (0-20)
|
|
||||||
_votes_score() {
|
|
||||||
local c="$1"
|
|
||||||
if (( c >= 10000 )); then echo 20
|
|
||||||
elif (( c >= 5000 )); then echo 15
|
|
||||||
elif (( c >= 1000 )); then echo 10
|
|
||||||
elif (( c >= 200 )); then echo 5
|
|
||||||
else echo 2
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
# Stage 2: seed breadth (0-40)
|
|
||||||
_breadth_score() {
|
|
||||||
local seeds="$1"
|
|
||||||
if (( seeds >= 3 )); then echo 40
|
|
||||||
elif (( seeds == 2 )); then echo 25
|
|
||||||
else echo 10
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Fetch Emby Series Library ━━━
|
# ━━━ Fetch Emby Series Library ━━━
|
||||||
@@ -636,9 +605,6 @@ done < <(echo "$SONARR_SERIES_JSON" | jq -r '.[] |
|
|||||||
|
|
||||||
log "${#SONARR_TVDB[@]} series in Sonarr | ${#EMBY_TMDB_IDS[@]} series in Emby"
|
log "${#SONARR_TVDB[@]} series in Sonarr | ${#EMBY_TMDB_IDS[@]} series in Emby"
|
||||||
|
|
||||||
_in_sonarr() { [[ "${SONARR_TVDB["$1"]+x}" || "${SONARR_TMDB["$2"]+x}" ]]; }
|
|
||||||
_in_emby() { [[ "${EMBY_TMDB_IDS["$1"]+x}" || "${EMBY_TVDB_IDS["$2"]+x}" ]]; }
|
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Score Stage 2 Candidates ━━━
|
# ━━━ Score Stage 2 Candidates ━━━
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
|
|||||||
+17
-185
@@ -64,7 +64,7 @@
|
|||||||
#
|
#
|
||||||
# acquire_lock "wait" — large scans take time, wait for previous run to finish
|
# acquire_lock "wait" — large scans take time, wait for previous run to finish
|
||||||
# jq + curl validation — exits if either tool missing
|
# jq + curl validation — exits if either tool missing
|
||||||
# DOCKER_TIMEOUT — container checks protected against daemon hangs
|
# ARR_DOCKER_TIMEOUT — container checks protected against daemon hangs (script-local, not common.sh's DOCKER_TIMEOUT)
|
||||||
# notify_emby_scan() — triggers Emby clean after deletion
|
# notify_emby_scan() — triggers Emby clean after deletion
|
||||||
# platform_require_cmd — notify script validated before use
|
# platform_require_cmd — notify script validated before use
|
||||||
# Silent by default — orphans/junk warn(), clean library logs silently
|
# Silent by default — orphans/junk warn(), clean library logs silently
|
||||||
@@ -120,37 +120,12 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|||||||
source "$SCRIPT_DIR/../load_config.sh"
|
source "$SCRIPT_DIR/../load_config.sh"
|
||||||
|
|
||||||
# ── Special flag pre-processing ───────────────────────────────────────────────────────────────
|
# ── Special flag pre-processing ───────────────────────────────────────────────────────────────
|
||||||
I_KNOW=false
|
parse_destructive_flags "$@"
|
||||||
SKIP_AGE_CHECK=false
|
|
||||||
FILTERED_ARGS=()
|
|
||||||
for arg in "$@"; do
|
|
||||||
case "$arg" in
|
|
||||||
--i-know-what-im-doing) I_KNOW=true ;;
|
|
||||||
--skip-age-check) SKIP_AGE_CHECK=true ;;
|
|
||||||
*) FILTERED_ARGS+=("$arg") ;;
|
|
||||||
esac
|
|
||||||
done
|
|
||||||
|
|
||||||
parse_args "${FILTERED_ARGS[@]}"
|
parse_args "${FILTERED_ARGS[@]}"
|
||||||
|
|
||||||
# ── Nuclear mode warning ──────────────────────────────────────────────────────────────────────
|
# ── Nuclear mode warning ──────────────────────────────────────────────────────────────────────
|
||||||
if [[ "$I_KNOW" == true ]] && [[ "$SKIP_AGE_CHECK" == true ]] && [[ "$DRY_RUN" != true ]]; then
|
nuclear_mode_warning
|
||||||
echo ""
|
|
||||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
||||||
echo "⚠️ WARNING — NUCLEAR MODE ACTIVE"
|
|
||||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
||||||
echo " Flags: --i-know-what-im-doing --skip-age-check"
|
|
||||||
echo " Age check: BYPASSED — deletes on first pass"
|
|
||||||
echo " Size threshold: BYPASSED — no GB limit"
|
|
||||||
echo " Data recovery: NOT POSSIBLE after deletion"
|
|
||||||
echo ""
|
|
||||||
echo " Review --dry-run output before proceeding."
|
|
||||||
echo " You have 10 seconds to cancel (Ctrl+C)..."
|
|
||||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
||||||
sleep 10
|
|
||||||
echo " Proceeding..."
|
|
||||||
echo ""
|
|
||||||
fi
|
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Setup ━━━
|
# ━━━ Setup ━━━
|
||||||
@@ -191,15 +166,11 @@ if [[ -z "${RADARR_URL:-}" ]] || [[ -z "${RADARR_API_KEY:-}" ]]; then
|
|||||||
exit 0
|
exit 0
|
||||||
fi
|
fi
|
||||||
|
|
||||||
DOCKER_TIMEOUT=15
|
ARR_DOCKER_TIMEOUT=15
|
||||||
RADARR_CONTAINER="Radarr"
|
RADARR_CONTAINER="Radarr"
|
||||||
|
|
||||||
# Build path map from MY_ID's Radarr path map
|
# Build path map from MY_ID's Radarr path map
|
||||||
declare -A ARR_PATH_MAP
|
build_arr_path_map "RADARR"
|
||||||
local_path_map_var="${MY_ID}_RADARR_PATH_MAP"
|
|
||||||
eval "for key in \"\${!${local_path_map_var}[@]}\"; do
|
|
||||||
ARR_PATH_MAP[\"\$key\"]=\"\${${local_path_map_var}[\$key]}\"
|
|
||||||
done"
|
|
||||||
|
|
||||||
require_var RADARR_URL
|
require_var RADARR_URL
|
||||||
require_var RADARR_API_KEY
|
require_var RADARR_API_KEY
|
||||||
@@ -247,90 +218,13 @@ fi
|
|||||||
echo ""
|
echo ""
|
||||||
echo "━━━ $ICON_SHIELD Safety Checks ━━━"
|
echo "━━━ $ICON_SHIELD Safety Checks ━━━"
|
||||||
|
|
||||||
CONTAINER_RUNNING=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
|
check_container_health "$RADARR_CONTAINER" "$ARR_DOCKER_TIMEOUT" "Radarr Cleanup"
|
||||||
'{{.State.Running}}' "$RADARR_CONTAINER" 2>/dev/null)
|
|
||||||
if [[ "$CONTAINER_RUNNING" != "true" ]]; then
|
|
||||||
error "$RADARR_CONTAINER is not running — aborting"
|
|
||||||
notify "Radarr cleanup aborted on $(hostname) — container not running" \
|
|
||||||
"Radarr Cleanup" "warning"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
CONTAINER_HEALTH=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
|
|
||||||
'{{.State.Health.Status}}' "$RADARR_CONTAINER" 2>/dev/null)
|
|
||||||
case "$CONTAINER_HEALTH" in
|
|
||||||
healthy) info "$RADARR_CONTAINER is healthy" ;;
|
|
||||||
"") info "$RADARR_CONTAINER has no health check — proceeding" ;;
|
|
||||||
starting)
|
|
||||||
error "$RADARR_CONTAINER is still starting — aborting"
|
|
||||||
notify "Radarr cleanup aborted on $(hostname) — container still starting" \
|
|
||||||
"Radarr Cleanup" "warning"
|
|
||||||
exit 1 ;;
|
|
||||||
unhealthy)
|
|
||||||
error "$RADARR_CONTAINER is unhealthy — aborting"
|
|
||||||
notify "Radarr cleanup aborted on $(hostname) — container unhealthy" \
|
|
||||||
"Radarr Cleanup" "warning"
|
|
||||||
exit 1 ;;
|
|
||||||
*) warn "$RADARR_CONTAINER health: $CONTAINER_HEALTH — proceeding with caution" ;;
|
|
||||||
esac
|
|
||||||
|
|
||||||
info "Safety layer 1 passed — container healthy"
|
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ── HELPER FUNCTIONS ──────────────────────────────────────────────────────────────────────────
|
# ── HELPER FUNCTIONS ──────────────────────────────────────────────────────────────────────────
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
|
|
||||||
radarr_api() {
|
# check_container_health(), arr_api(), has_extension(), matches_pattern_list(), format_bytes() — common.sh
|
||||||
local endpoint="$1"
|
|
||||||
local response http_code body
|
|
||||||
|
|
||||||
response=$(curl -sf \
|
|
||||||
--max-time 30 \
|
|
||||||
-H "X-Api-Key: $RADARR_API_KEY" \
|
|
||||||
-w "\n%{http_code}" \
|
|
||||||
"${RADARR_URL}/api/v3/${endpoint}" 2>/dev/null)
|
|
||||||
|
|
||||||
http_code=$(echo "$response" | tail -1)
|
|
||||||
body=$(echo "$response" | head -n -1)
|
|
||||||
|
|
||||||
if [[ "$http_code" != "200" ]]; then
|
|
||||||
error "Radarr API HTTP $http_code for: $endpoint"
|
|
||||||
return 1
|
|
||||||
fi
|
|
||||||
echo "$body"
|
|
||||||
}
|
|
||||||
|
|
||||||
is_video_file() {
|
|
||||||
local ext="${1##*.}"
|
|
||||||
ext="${ext,,}"
|
|
||||||
for valid_ext in "${RADARR_EXTENSIONS[@]}"; do
|
|
||||||
[[ "$ext" == "$valid_ext" ]] && return 0
|
|
||||||
done
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|
||||||
is_protected_file() {
|
|
||||||
local filename
|
|
||||||
filename=$(basename "$1")
|
|
||||||
for pattern in "${RADARR_PROTECTED_PATTERNS[@]}"; do
|
|
||||||
# shellcheck disable=SC2254
|
|
||||||
case "$filename" in
|
|
||||||
$pattern) return 0 ;;
|
|
||||||
esac
|
|
||||||
done
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|
||||||
format_bytes() {
|
|
||||||
local bytes=$1
|
|
||||||
if (( bytes > 1073741824 )); then
|
|
||||||
awk "BEGIN {printf \"%.1fGB\", $bytes / 1073741824}"
|
|
||||||
elif (( bytes > 1048576 )); then
|
|
||||||
awk "BEGIN {printf \"%.1fMB\", $bytes / 1048576}"
|
|
||||||
else
|
|
||||||
echo "${bytes}B"
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Pre-flight: Radarr Import Scan ━━━
|
# ━━━ Pre-flight: Radarr Import Scan ━━━
|
||||||
@@ -340,7 +234,7 @@ echo "━━━ $ICON_SYNC Pre-flight: Radarr Import Scan ━━━"
|
|||||||
|
|
||||||
# Fetch root folders from Radarr API and translate container paths to host paths
|
# Fetch root folders from Radarr API and translate container paths to host paths
|
||||||
mapfile -t SCAN_ROOTS < <(
|
mapfile -t SCAN_ROOTS < <(
|
||||||
radarr_api "rootfolder" | \
|
arr_api "$RADARR_URL" "$RADARR_API_KEY" "v3" "rootfolder" "Radarr" | \
|
||||||
jq -r '.[].path' 2>/dev/null | \
|
jq -r '.[].path' 2>/dev/null | \
|
||||||
while IFS= read -r cp; do translate_path "$cp"; done
|
while IFS= read -r cp; do translate_path "$cp"; done
|
||||||
)
|
)
|
||||||
@@ -357,36 +251,7 @@ info "Scan targets (${#SCAN_ROOTS[@]}): ${SCAN_ROOTS[*]}"
|
|||||||
info "Triggering ProcessMonitoredDownloads pre-flight"
|
info "Triggering ProcessMonitoredDownloads pre-flight"
|
||||||
SCAN_PAYLOAD='{"name": "ProcessMonitoredDownloads"}'
|
SCAN_PAYLOAD='{"name": "ProcessMonitoredDownloads"}'
|
||||||
|
|
||||||
SCAN_RESPONSE=$(curl -sf --max-time 30 -X POST \
|
trigger_and_await_command "$RADARR_URL" "$RADARR_API_KEY" "v3" "$SCAN_PAYLOAD" "${RADARR_IMPORT_SCAN_TIMEOUT:-600}"
|
||||||
-H "X-Api-Key: $RADARR_API_KEY" \
|
|
||||||
-H "Content-Type: application/json" \
|
|
||||||
-d "$SCAN_PAYLOAD" \
|
|
||||||
"${RADARR_URL}/api/v3/command" 2>/dev/null)
|
|
||||||
|
|
||||||
SCAN_CMD_ID=$(echo "$SCAN_RESPONSE" | jq -r '.id // empty' 2>/dev/null)
|
|
||||||
|
|
||||||
if [[ -z "$SCAN_CMD_ID" ]]; then
|
|
||||||
warn "Could not trigger import scan — proceeding without pre-flight"
|
|
||||||
else
|
|
||||||
info "Import scan queued (command ID: $SCAN_CMD_ID) — waiting for completion..."
|
|
||||||
POLL_TIMEOUT=${RADARR_IMPORT_SCAN_TIMEOUT:-600}
|
|
||||||
POLLED=0
|
|
||||||
while [[ "$POLLED" -lt "$POLL_TIMEOUT" ]]; do
|
|
||||||
SCAN_STATUS=$(curl -sf --max-time 10 \
|
|
||||||
-H "X-Api-Key: $RADARR_API_KEY" \
|
|
||||||
"${RADARR_URL}/api/v3/command/${SCAN_CMD_ID}" 2>/dev/null | \
|
|
||||||
jq -r '.status // empty' 2>/dev/null)
|
|
||||||
case "$SCAN_STATUS" in
|
|
||||||
completed) info "Import scan complete ✅"; break ;;
|
|
||||||
failed) warn "Import scan reported failed — proceeding anyway"; break ;;
|
|
||||||
esac
|
|
||||||
sleep 10
|
|
||||||
(( POLLED += 10 ))
|
|
||||||
[[ $(( POLLED % 60 )) -eq 0 ]] && log " Still scanning... (${POLLED}s elapsed)"
|
|
||||||
done
|
|
||||||
[[ "$POLLED" -ge "$POLL_TIMEOUT" ]] && \
|
|
||||||
warn "Import scan timed out after ${POLL_TIMEOUT}s — proceeding anyway"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Fetch Radarr Tracked Files ━━━
|
# ━━━ Fetch Radarr Tracked Files ━━━
|
||||||
@@ -406,7 +271,7 @@ check_arr_version "$RADARR_URL" "$RADARR_API_KEY" "v3" "$RADARR_VERSION_MAJOR" "
|
|||||||
info "Querying Radarr API..."
|
info "Querying Radarr API..."
|
||||||
|
|
||||||
# Fetch all movies
|
# Fetch all movies
|
||||||
MOVIES_RESPONSE=$(radarr_api "movie") || {
|
MOVIES_RESPONSE=$(arr_api "$RADARR_URL" "$RADARR_API_KEY" "v3" "movie" "Radarr") || {
|
||||||
error "Failed to fetch movies from Radarr"
|
error "Failed to fetch movies from Radarr"
|
||||||
notify "Radarr cleanup failed on $(hostname) — could not fetch movies" \
|
notify "Radarr cleanup failed on $(hostname) — could not fetch movies" \
|
||||||
"Radarr Cleanup" "warning"
|
"Radarr Cleanup" "warning"
|
||||||
@@ -435,7 +300,7 @@ while IFS= read -r movie_id; do
|
|||||||
(( MOVIE_INDEX++ ))
|
(( MOVIE_INDEX++ ))
|
||||||
[[ $(( MOVIE_INDEX % 100 )) -eq 0 ]] && \
|
[[ $(( MOVIE_INDEX % 100 )) -eq 0 ]] && \
|
||||||
log "Fetching files: $MOVIE_INDEX/$MOVIE_COUNT movies..."
|
log "Fetching files: $MOVIE_INDEX/$MOVIE_COUNT movies..."
|
||||||
MOVIE_FILES=$(radarr_api "moviefile?movieId=${movie_id}" 2>/dev/null)
|
MOVIE_FILES=$(arr_api "$RADARR_URL" "$RADARR_API_KEY" "v3" "moviefile?movieId=${movie_id}" "Radarr" 2>/dev/null)
|
||||||
if [[ -n "$MOVIE_FILES" ]]; then
|
if [[ -n "$MOVIE_FILES" ]]; then
|
||||||
while IFS= read -r api_path; do
|
while IFS= read -r api_path; do
|
||||||
[[ -z "$api_path" ]] && continue
|
[[ -z "$api_path" ]] && continue
|
||||||
@@ -467,25 +332,7 @@ fi
|
|||||||
info "$MOVIE_COUNT movies | $TRACKED_COUNT tracked movie files"
|
info "$MOVIE_COUNT movies | $TRACKED_COUNT tracked movie files"
|
||||||
|
|
||||||
# Safety Layer 6 — percentage drop vs last known count
|
# Safety Layer 6 — percentage drop vs last known count
|
||||||
if [[ -f "$RADARR_TRACKED_COUNT_FILE" ]]; then
|
check_tracked_count_floor "$TRACKED_COUNT" "$RADARR_TRACKED_COUNT_FILE" "$RADARR_MIN_TRACKED_PCT" "Radarr Cleanup"
|
||||||
LAST_COUNT=$(cat "$RADARR_TRACKED_COUNT_FILE" 2>/dev/null || echo 0)
|
|
||||||
if [[ "$LAST_COUNT" -gt 0 ]]; then
|
|
||||||
PCT=$(awk "BEGIN {printf \"%d\", ($TRACKED_COUNT / $LAST_COUNT) * 100}")
|
|
||||||
if [[ "$PCT" -lt "$RADARR_MIN_TRACKED_PCT" ]]; then
|
|
||||||
error "Tracked count dropped to ${PCT}% of last run ($TRACKED_COUNT vs $LAST_COUNT)"
|
|
||||||
error "Suggests API issue — aborting to prevent mass deletion"
|
|
||||||
error "If expected (large library removal) delete: $RADARR_TRACKED_COUNT_FILE"
|
|
||||||
notify "Radarr cleanup aborted on $(hostname) — tracked count dropped to ${PCT}%" \
|
|
||||||
"Radarr Cleanup" "warning"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
info "Tracked count: ${PCT}% of last run ($TRACKED_COUNT vs $LAST_COUNT) ✅"
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
info "No previous count on record — first run, saving baseline"
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "$TRACKED_COUNT" > "$RADARR_TRACKED_COUNT_FILE"
|
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Scan Movies Root ━━━
|
# ━━━ Scan Movies Root ━━━
|
||||||
@@ -504,7 +351,6 @@ JUNK_BYTES=0
|
|||||||
|
|
||||||
AGE_SECONDS=$(( RADARR_ORPHAN_AGE * 86400 ))
|
AGE_SECONDS=$(( RADARR_ORPHAN_AGE * 86400 ))
|
||||||
NOW=$(date +%s)
|
NOW=$(date +%s)
|
||||||
MAX_DELETE_BYTES=$(awk "BEGIN {printf \"%d\", $RADARR_MAX_DELETE_GB * 1073741824}")
|
|
||||||
|
|
||||||
while IFS= read -r filepath; do
|
while IFS= read -r filepath; do
|
||||||
[[ -z "$filepath" ]] && continue
|
[[ -z "$filepath" ]] && continue
|
||||||
@@ -514,7 +360,7 @@ while IFS= read -r filepath; do
|
|||||||
continue
|
continue
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if is_protected_file "$filepath"; then
|
if matches_pattern_list "$filepath" "${RADARR_PROTECTED_PATTERNS[@]}"; then
|
||||||
log "$ICON_PROTECTED PROTECTED: $filepath"
|
log "$ICON_PROTECTED PROTECTED: $filepath"
|
||||||
(( PROTECTED_COUNT++ ))
|
(( PROTECTED_COUNT++ ))
|
||||||
continue
|
continue
|
||||||
@@ -522,7 +368,7 @@ while IFS= read -r filepath; do
|
|||||||
|
|
||||||
FILE_SIZE=$(stat -c%s "$filepath" 2>/dev/null || echo 0)
|
FILE_SIZE=$(stat -c%s "$filepath" 2>/dev/null || echo 0)
|
||||||
|
|
||||||
if is_video_file "$filepath"; then
|
if has_extension "$filepath" "${RADARR_EXTENSIONS[@]}"; then
|
||||||
FILE_MTIME=$(stat -c %Y "$filepath" 2>/dev/null || echo 0)
|
FILE_MTIME=$(stat -c %Y "$filepath" 2>/dev/null || echo 0)
|
||||||
FILE_AGE=$(( NOW - FILE_MTIME ))
|
FILE_AGE=$(( NOW - FILE_MTIME ))
|
||||||
|
|
||||||
@@ -553,33 +399,19 @@ TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT ))
|
|||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Safety Layer 7 — Deletion Size Threshold ━━━
|
# ━━━ Safety Layer 7 — Deletion Size Threshold ━━━
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
if [[ "$TOTAL_DELETE_BYTES" -gt "$MAX_DELETE_BYTES" ]]; then
|
check_delete_size_threshold "$TOTAL_DELETE_BYTES" "$RADARR_MAX_DELETE_GB" "Radarr Cleanup"
|
||||||
TOTAL_HUMAN=$(awk "BEGIN {printf \"%.1fGB\", $TOTAL_DELETE_BYTES / 1073741824}")
|
|
||||||
if [[ "$I_KNOW" != true ]]; then
|
|
||||||
echo ""
|
|
||||||
error "Deletion would exceed ${RADARR_MAX_DELETE_GB}GB — $TOTAL_HUMAN would be deleted"
|
|
||||||
error "Review ORPHAN lines above carefully before proceeding"
|
|
||||||
error "Rerun with: --i-know-what-im-doing"
|
|
||||||
error "To also bypass age check: add --skip-age-check"
|
|
||||||
notify "Radarr cleanup halted on $(hostname) — ${TOTAL_HUMAN} requires --i-know-what-im-doing" \
|
|
||||||
"Radarr Cleanup" "warning"
|
|
||||||
exit 1
|
|
||||||
else
|
|
||||||
warn "OVERRIDE — deletion is $TOTAL_HUMAN — proceeding with --i-know-what-im-doing"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
# ── Execute Deletions ─────────────────────────────────────────────────────────────────────────
|
# ── Execute Deletions ─────────────────────────────────────────────────────────────────────────
|
||||||
if [[ "$DRY_RUN" == false ]]; then
|
if [[ "$DRY_RUN" == false ]]; then
|
||||||
while IFS= read -r filepath; do
|
while IFS= read -r filepath; do
|
||||||
[[ -z "$filepath" ]] && continue
|
[[ -z "$filepath" ]] && continue
|
||||||
[[ -n "${TRACKED_MAP[$filepath]:-}" ]] && continue
|
[[ -n "${TRACKED_MAP[$filepath]:-}" ]] && continue
|
||||||
is_protected_file "$filepath" && continue
|
matches_pattern_list "$filepath" "${RADARR_PROTECTED_PATTERNS[@]}" && continue
|
||||||
|
|
||||||
FILE_MTIME=$(stat -c %Y "$filepath" 2>/dev/null || echo 0)
|
FILE_MTIME=$(stat -c %Y "$filepath" 2>/dev/null || echo 0)
|
||||||
FILE_AGE=$(( NOW - FILE_MTIME ))
|
FILE_AGE=$(( NOW - FILE_MTIME ))
|
||||||
|
|
||||||
if is_video_file "$filepath"; then
|
if has_extension "$filepath" "${RADARR_EXTENSIONS[@]}"; then
|
||||||
[[ "$FILE_AGE" -lt "$AGE_SECONDS" ]] && \
|
[[ "$FILE_AGE" -lt "$AGE_SECONDS" ]] && \
|
||||||
[[ "$SKIP_AGE_CHECK" != true ]] && continue
|
[[ "$SKIP_AGE_CHECK" != true ]] && continue
|
||||||
fi
|
fi
|
||||||
|
|||||||
+17
-185
@@ -64,7 +64,7 @@
|
|||||||
#
|
#
|
||||||
# acquire_lock "wait" — large scans take time, wait for previous run to finish
|
# acquire_lock "wait" — large scans take time, wait for previous run to finish
|
||||||
# jq + curl validation — exits if either tool missing
|
# jq + curl validation — exits if either tool missing
|
||||||
# DOCKER_TIMEOUT — container checks protected against daemon hangs
|
# ARR_DOCKER_TIMEOUT — container checks protected against daemon hangs (script-local, not common.sh's DOCKER_TIMEOUT)
|
||||||
# notify_emby_scan() — triggers Emby clean after deletion
|
# notify_emby_scan() — triggers Emby clean after deletion
|
||||||
# platform_require_cmd — notify script validated before use
|
# platform_require_cmd — notify script validated before use
|
||||||
# Silent by default — orphans/junk warn(), clean library logs silently
|
# Silent by default — orphans/junk warn(), clean library logs silently
|
||||||
@@ -120,37 +120,12 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|||||||
source "$SCRIPT_DIR/../load_config.sh"
|
source "$SCRIPT_DIR/../load_config.sh"
|
||||||
|
|
||||||
# ── Special flag pre-processing ───────────────────────────────────────────────────────────────
|
# ── Special flag pre-processing ───────────────────────────────────────────────────────────────
|
||||||
I_KNOW=false
|
parse_destructive_flags "$@"
|
||||||
SKIP_AGE_CHECK=false
|
|
||||||
FILTERED_ARGS=()
|
|
||||||
for arg in "$@"; do
|
|
||||||
case "$arg" in
|
|
||||||
--i-know-what-im-doing) I_KNOW=true ;;
|
|
||||||
--skip-age-check) SKIP_AGE_CHECK=true ;;
|
|
||||||
*) FILTERED_ARGS+=("$arg") ;;
|
|
||||||
esac
|
|
||||||
done
|
|
||||||
|
|
||||||
parse_args "${FILTERED_ARGS[@]}"
|
parse_args "${FILTERED_ARGS[@]}"
|
||||||
|
|
||||||
# ── Nuclear mode warning ──────────────────────────────────────────────────────────────────────
|
# ── Nuclear mode warning ──────────────────────────────────────────────────────────────────────
|
||||||
if [[ "$I_KNOW" == true ]] && [[ "$SKIP_AGE_CHECK" == true ]] && [[ "$DRY_RUN" != true ]]; then
|
nuclear_mode_warning
|
||||||
echo ""
|
|
||||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
||||||
echo "⚠️ WARNING — NUCLEAR MODE ACTIVE"
|
|
||||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
||||||
echo " Flags: --i-know-what-im-doing --skip-age-check"
|
|
||||||
echo " Age check: BYPASSED — deletes on first pass"
|
|
||||||
echo " Size threshold: BYPASSED — no GB limit"
|
|
||||||
echo " Data recovery: NOT POSSIBLE after deletion"
|
|
||||||
echo ""
|
|
||||||
echo " Review --dry-run output before proceeding."
|
|
||||||
echo " You have 10 seconds to cancel (Ctrl+C)..."
|
|
||||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
||||||
sleep 10
|
|
||||||
echo " Proceeding..."
|
|
||||||
echo ""
|
|
||||||
fi
|
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Setup ━━━
|
# ━━━ Setup ━━━
|
||||||
@@ -191,15 +166,11 @@ if [[ -z "${SONARR_URL:-}" ]] || [[ -z "${SONARR_API_KEY:-}" ]]; then
|
|||||||
exit 0
|
exit 0
|
||||||
fi
|
fi
|
||||||
|
|
||||||
DOCKER_TIMEOUT=15
|
ARR_DOCKER_TIMEOUT=15
|
||||||
SONARR_CONTAINER="Sonarr"
|
SONARR_CONTAINER="Sonarr"
|
||||||
|
|
||||||
# Build path map from MY_ID's Sonarr path map
|
# Build path map from MY_ID's Sonarr path map
|
||||||
declare -A ARR_PATH_MAP
|
build_arr_path_map "SONARR"
|
||||||
local_path_map_var="${MY_ID}_SONARR_PATH_MAP"
|
|
||||||
eval "for key in \"\${!${local_path_map_var}[@]}\"; do
|
|
||||||
ARR_PATH_MAP[\"\$key\"]=\"\${${local_path_map_var}[\$key]}\"
|
|
||||||
done"
|
|
||||||
|
|
||||||
require_var SONARR_URL
|
require_var SONARR_URL
|
||||||
require_var SONARR_API_KEY
|
require_var SONARR_API_KEY
|
||||||
@@ -247,90 +218,13 @@ fi
|
|||||||
echo ""
|
echo ""
|
||||||
echo "━━━ $ICON_SHIELD Safety Checks ━━━"
|
echo "━━━ $ICON_SHIELD Safety Checks ━━━"
|
||||||
|
|
||||||
CONTAINER_RUNNING=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
|
check_container_health "$SONARR_CONTAINER" "$ARR_DOCKER_TIMEOUT" "Sonarr Cleanup"
|
||||||
'{{.State.Running}}' "$SONARR_CONTAINER" 2>/dev/null)
|
|
||||||
if [[ "$CONTAINER_RUNNING" != "true" ]]; then
|
|
||||||
error "$SONARR_CONTAINER is not running — aborting"
|
|
||||||
notify "Sonarr cleanup aborted on $(hostname) — container not running" \
|
|
||||||
"Sonarr Cleanup" "warning"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
CONTAINER_HEALTH=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
|
|
||||||
'{{.State.Health.Status}}' "$SONARR_CONTAINER" 2>/dev/null)
|
|
||||||
case "$CONTAINER_HEALTH" in
|
|
||||||
healthy) info "$SONARR_CONTAINER is healthy" ;;
|
|
||||||
"") info "$SONARR_CONTAINER has no health check — proceeding" ;;
|
|
||||||
starting)
|
|
||||||
error "$SONARR_CONTAINER is still starting — aborting"
|
|
||||||
notify "Sonarr cleanup aborted on $(hostname) — container still starting" \
|
|
||||||
"Sonarr Cleanup" "warning"
|
|
||||||
exit 1 ;;
|
|
||||||
unhealthy)
|
|
||||||
error "$SONARR_CONTAINER is unhealthy — aborting"
|
|
||||||
notify "Sonarr cleanup aborted on $(hostname) — container unhealthy" \
|
|
||||||
"Sonarr Cleanup" "warning"
|
|
||||||
exit 1 ;;
|
|
||||||
*) warn "$SONARR_CONTAINER health: $CONTAINER_HEALTH — proceeding with caution" ;;
|
|
||||||
esac
|
|
||||||
|
|
||||||
info "Safety layer 1 passed — container healthy"
|
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ── HELPER FUNCTIONS ──────────────────────────────────────────────────────────────────────────
|
# ── HELPER FUNCTIONS ──────────────────────────────────────────────────────────────────────────
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
|
|
||||||
sonarr_api() {
|
# check_container_health(), arr_api(), has_extension(), matches_pattern_list(), format_bytes() — common.sh
|
||||||
local endpoint="$1"
|
|
||||||
local response http_code body
|
|
||||||
|
|
||||||
response=$(curl -sf \
|
|
||||||
--max-time 30 \
|
|
||||||
-H "X-Api-Key: $SONARR_API_KEY" \
|
|
||||||
-w "\n%{http_code}" \
|
|
||||||
"${SONARR_URL}/api/v3/${endpoint}" 2>/dev/null)
|
|
||||||
|
|
||||||
http_code=$(echo "$response" | tail -1)
|
|
||||||
body=$(echo "$response" | head -n -1)
|
|
||||||
|
|
||||||
if [[ "$http_code" != "200" ]]; then
|
|
||||||
error "Sonarr API HTTP $http_code for: $endpoint"
|
|
||||||
return 1
|
|
||||||
fi
|
|
||||||
echo "$body"
|
|
||||||
}
|
|
||||||
|
|
||||||
is_video_file() {
|
|
||||||
local ext="${1##*.}"
|
|
||||||
ext="${ext,,}"
|
|
||||||
for valid_ext in "${SONARR_EXTENSIONS[@]}"; do
|
|
||||||
[[ "$ext" == "$valid_ext" ]] && return 0
|
|
||||||
done
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|
||||||
is_protected_file() {
|
|
||||||
local filename
|
|
||||||
filename=$(basename "$1")
|
|
||||||
for pattern in "${SONARR_PROTECTED_PATTERNS[@]}"; do
|
|
||||||
# shellcheck disable=SC2254
|
|
||||||
case "$filename" in
|
|
||||||
$pattern) return 0 ;;
|
|
||||||
esac
|
|
||||||
done
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|
||||||
format_bytes() {
|
|
||||||
local bytes=$1
|
|
||||||
if (( bytes > 1073741824 )); then
|
|
||||||
awk "BEGIN {printf \"%.1fGB\", $bytes / 1073741824}"
|
|
||||||
elif (( bytes > 1048576 )); then
|
|
||||||
awk "BEGIN {printf \"%.1fMB\", $bytes / 1048576}"
|
|
||||||
else
|
|
||||||
echo "${bytes}B"
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Pre-flight: Sonarr Import Scan ━━━
|
# ━━━ Pre-flight: Sonarr Import Scan ━━━
|
||||||
@@ -340,7 +234,7 @@ echo "━━━ $ICON_SYNC Pre-flight: Sonarr Import Scan ━━━"
|
|||||||
|
|
||||||
# Fetch root folders from Sonarr API and translate container paths to host paths
|
# Fetch root folders from Sonarr API and translate container paths to host paths
|
||||||
mapfile -t SCAN_ROOTS < <(
|
mapfile -t SCAN_ROOTS < <(
|
||||||
sonarr_api "rootfolder" | \
|
arr_api "$SONARR_URL" "$SONARR_API_KEY" "v3" "rootfolder" "Sonarr" | \
|
||||||
jq -r '.[].path' 2>/dev/null | \
|
jq -r '.[].path' 2>/dev/null | \
|
||||||
while IFS= read -r cp; do translate_path "$cp"; done
|
while IFS= read -r cp; do translate_path "$cp"; done
|
||||||
)
|
)
|
||||||
@@ -357,36 +251,7 @@ info "Scan targets (${#SCAN_ROOTS[@]}): ${SCAN_ROOTS[*]}"
|
|||||||
info "Triggering ProcessMonitoredDownloads pre-flight"
|
info "Triggering ProcessMonitoredDownloads pre-flight"
|
||||||
SCAN_PAYLOAD='{"name": "ProcessMonitoredDownloads"}'
|
SCAN_PAYLOAD='{"name": "ProcessMonitoredDownloads"}'
|
||||||
|
|
||||||
SCAN_RESPONSE=$(curl -sf --max-time 30 -X POST \
|
trigger_and_await_command "$SONARR_URL" "$SONARR_API_KEY" "v3" "$SCAN_PAYLOAD" "${SONARR_IMPORT_SCAN_TIMEOUT:-600}"
|
||||||
-H "X-Api-Key: $SONARR_API_KEY" \
|
|
||||||
-H "Content-Type: application/json" \
|
|
||||||
-d "$SCAN_PAYLOAD" \
|
|
||||||
"${SONARR_URL}/api/v3/command" 2>/dev/null)
|
|
||||||
|
|
||||||
SCAN_CMD_ID=$(echo "$SCAN_RESPONSE" | jq -r '.id // empty' 2>/dev/null)
|
|
||||||
|
|
||||||
if [[ -z "$SCAN_CMD_ID" ]]; then
|
|
||||||
warn "Could not trigger import scan — proceeding without pre-flight"
|
|
||||||
else
|
|
||||||
info "Import scan queued (command ID: $SCAN_CMD_ID) — waiting for completion..."
|
|
||||||
POLL_TIMEOUT=${SONARR_IMPORT_SCAN_TIMEOUT:-600}
|
|
||||||
POLLED=0
|
|
||||||
while [[ "$POLLED" -lt "$POLL_TIMEOUT" ]]; do
|
|
||||||
SCAN_STATUS=$(curl -sf --max-time 10 \
|
|
||||||
-H "X-Api-Key: $SONARR_API_KEY" \
|
|
||||||
"${SONARR_URL}/api/v3/command/${SCAN_CMD_ID}" 2>/dev/null | \
|
|
||||||
jq -r '.status // empty' 2>/dev/null)
|
|
||||||
case "$SCAN_STATUS" in
|
|
||||||
completed) info "Import scan complete ✅"; break ;;
|
|
||||||
failed) warn "Import scan reported failed — proceeding anyway"; break ;;
|
|
||||||
esac
|
|
||||||
sleep 10
|
|
||||||
(( POLLED += 10 ))
|
|
||||||
[[ $(( POLLED % 60 )) -eq 0 ]] && log " Still scanning... (${POLLED}s elapsed)"
|
|
||||||
done
|
|
||||||
[[ "$POLLED" -ge "$POLL_TIMEOUT" ]] && \
|
|
||||||
warn "Import scan timed out after ${POLL_TIMEOUT}s — proceeding anyway"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Fetch Sonarr Tracked Files ━━━
|
# ━━━ Fetch Sonarr Tracked Files ━━━
|
||||||
@@ -406,7 +271,7 @@ check_arr_version "$SONARR_URL" "$SONARR_API_KEY" "v3" "$SONARR_VERSION_MAJOR" "
|
|||||||
info "Querying Sonarr API..."
|
info "Querying Sonarr API..."
|
||||||
|
|
||||||
# Fetch all series
|
# Fetch all series
|
||||||
SERIES_RESPONSE=$(sonarr_api "series") || {
|
SERIES_RESPONSE=$(arr_api "$SONARR_URL" "$SONARR_API_KEY" "v3" "series" "Sonarr") || {
|
||||||
error "Failed to fetch series from Sonarr"
|
error "Failed to fetch series from Sonarr"
|
||||||
notify "Sonarr cleanup failed on $(hostname) — could not fetch series" \
|
notify "Sonarr cleanup failed on $(hostname) — could not fetch series" \
|
||||||
"Sonarr Cleanup" "warning"
|
"Sonarr Cleanup" "warning"
|
||||||
@@ -435,7 +300,7 @@ while IFS= read -r series_id; do
|
|||||||
(( SERIES_INDEX++ ))
|
(( SERIES_INDEX++ ))
|
||||||
[[ $(( SERIES_INDEX % 50 )) -eq 0 ]] && \
|
[[ $(( SERIES_INDEX % 50 )) -eq 0 ]] && \
|
||||||
log "Fetching files: $SERIES_INDEX/$SERIES_COUNT series..."
|
log "Fetching files: $SERIES_INDEX/$SERIES_COUNT series..."
|
||||||
SERIES_FILES=$(sonarr_api "episodefile?seriesId=${series_id}" 2>/dev/null)
|
SERIES_FILES=$(arr_api "$SONARR_URL" "$SONARR_API_KEY" "v3" "episodefile?seriesId=${series_id}" "Sonarr" 2>/dev/null)
|
||||||
if [[ -n "$SERIES_FILES" ]]; then
|
if [[ -n "$SERIES_FILES" ]]; then
|
||||||
while IFS= read -r api_path; do
|
while IFS= read -r api_path; do
|
||||||
[[ -z "$api_path" ]] && continue
|
[[ -z "$api_path" ]] && continue
|
||||||
@@ -466,25 +331,7 @@ fi
|
|||||||
info "$SERIES_COUNT series | $TRACKED_COUNT tracked episode files"
|
info "$SERIES_COUNT series | $TRACKED_COUNT tracked episode files"
|
||||||
|
|
||||||
# Safety Layer 6 — percentage drop vs last known count
|
# Safety Layer 6 — percentage drop vs last known count
|
||||||
if [[ -f "$SONARR_TRACKED_COUNT_FILE" ]]; then
|
check_tracked_count_floor "$TRACKED_COUNT" "$SONARR_TRACKED_COUNT_FILE" "$SONARR_MIN_TRACKED_PCT" "Sonarr Cleanup"
|
||||||
LAST_COUNT=$(cat "$SONARR_TRACKED_COUNT_FILE" 2>/dev/null || echo 0)
|
|
||||||
if [[ "$LAST_COUNT" -gt 0 ]]; then
|
|
||||||
PCT=$(awk "BEGIN {printf \"%d\", ($TRACKED_COUNT / $LAST_COUNT) * 100}")
|
|
||||||
if [[ "$PCT" -lt "$SONARR_MIN_TRACKED_PCT" ]]; then
|
|
||||||
error "Tracked count dropped to ${PCT}% of last run ($TRACKED_COUNT vs $LAST_COUNT)"
|
|
||||||
error "Suggests API issue — aborting to prevent mass deletion"
|
|
||||||
error "If expected (large library removal) delete: $SONARR_TRACKED_COUNT_FILE"
|
|
||||||
notify "Sonarr cleanup aborted on $(hostname) — tracked count dropped to ${PCT}%" \
|
|
||||||
"Sonarr Cleanup" "warning"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
info "Tracked count: ${PCT}% of last run ($TRACKED_COUNT vs $LAST_COUNT) ✅"
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
info "No previous count on record — first run, saving baseline"
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "$TRACKED_COUNT" > "$SONARR_TRACKED_COUNT_FILE"
|
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Scan TV Root ━━━
|
# ━━━ Scan TV Root ━━━
|
||||||
@@ -503,7 +350,6 @@ JUNK_BYTES=0
|
|||||||
|
|
||||||
AGE_SECONDS=$(( SONARR_ORPHAN_AGE * 86400 ))
|
AGE_SECONDS=$(( SONARR_ORPHAN_AGE * 86400 ))
|
||||||
NOW=$(date +%s)
|
NOW=$(date +%s)
|
||||||
MAX_DELETE_BYTES=$(awk "BEGIN {printf \"%d\", $SONARR_MAX_DELETE_GB * 1073741824}")
|
|
||||||
|
|
||||||
while IFS= read -r filepath; do
|
while IFS= read -r filepath; do
|
||||||
[[ -z "$filepath" ]] && continue
|
[[ -z "$filepath" ]] && continue
|
||||||
@@ -513,7 +359,7 @@ while IFS= read -r filepath; do
|
|||||||
continue
|
continue
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if is_protected_file "$filepath"; then
|
if matches_pattern_list "$filepath" "${SONARR_PROTECTED_PATTERNS[@]}"; then
|
||||||
log "$ICON_PROTECTED PROTECTED: $filepath"
|
log "$ICON_PROTECTED PROTECTED: $filepath"
|
||||||
(( PROTECTED_COUNT++ ))
|
(( PROTECTED_COUNT++ ))
|
||||||
continue
|
continue
|
||||||
@@ -521,7 +367,7 @@ while IFS= read -r filepath; do
|
|||||||
|
|
||||||
FILE_SIZE=$(stat -c%s "$filepath" 2>/dev/null || echo 0)
|
FILE_SIZE=$(stat -c%s "$filepath" 2>/dev/null || echo 0)
|
||||||
|
|
||||||
if is_video_file "$filepath"; then
|
if has_extension "$filepath" "${SONARR_EXTENSIONS[@]}"; then
|
||||||
FILE_MTIME=$(stat -c %Y "$filepath" 2>/dev/null || echo 0)
|
FILE_MTIME=$(stat -c %Y "$filepath" 2>/dev/null || echo 0)
|
||||||
FILE_AGE=$(( NOW - FILE_MTIME ))
|
FILE_AGE=$(( NOW - FILE_MTIME ))
|
||||||
|
|
||||||
@@ -552,33 +398,19 @@ TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT ))
|
|||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Safety Layer 7 — Deletion Size Threshold ━━━
|
# ━━━ Safety Layer 7 — Deletion Size Threshold ━━━
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
if [[ "$TOTAL_DELETE_BYTES" -gt "$MAX_DELETE_BYTES" ]]; then
|
check_delete_size_threshold "$TOTAL_DELETE_BYTES" "$SONARR_MAX_DELETE_GB" "Sonarr Cleanup"
|
||||||
TOTAL_HUMAN=$(awk "BEGIN {printf \"%.1fGB\", $TOTAL_DELETE_BYTES / 1073741824}")
|
|
||||||
if [[ "$I_KNOW" != true ]]; then
|
|
||||||
echo ""
|
|
||||||
error "Deletion would exceed ${SONARR_MAX_DELETE_GB}GB — $TOTAL_HUMAN would be deleted"
|
|
||||||
error "Review ORPHAN lines above carefully before proceeding"
|
|
||||||
error "Rerun with: --i-know-what-im-doing"
|
|
||||||
error "To also bypass age check: add --skip-age-check"
|
|
||||||
notify "Sonarr cleanup halted on $(hostname) — ${TOTAL_HUMAN} requires --i-know-what-im-doing" \
|
|
||||||
"Sonarr Cleanup" "warning"
|
|
||||||
exit 1
|
|
||||||
else
|
|
||||||
warn "OVERRIDE — deletion is $TOTAL_HUMAN — proceeding with --i-know-what-im-doing"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
# ── Execute Deletions ─────────────────────────────────────────────────────────────────────────
|
# ── Execute Deletions ─────────────────────────────────────────────────────────────────────────
|
||||||
if [[ "$DRY_RUN" == false ]]; then
|
if [[ "$DRY_RUN" == false ]]; then
|
||||||
while IFS= read -r filepath; do
|
while IFS= read -r filepath; do
|
||||||
[[ -z "$filepath" ]] && continue
|
[[ -z "$filepath" ]] && continue
|
||||||
[[ -n "${TRACKED_MAP[$filepath]:-}" ]] && continue
|
[[ -n "${TRACKED_MAP[$filepath]:-}" ]] && continue
|
||||||
is_protected_file "$filepath" && continue
|
matches_pattern_list "$filepath" "${SONARR_PROTECTED_PATTERNS[@]}" && continue
|
||||||
|
|
||||||
FILE_MTIME=$(stat -c %Y "$filepath" 2>/dev/null || echo 0)
|
FILE_MTIME=$(stat -c %Y "$filepath" 2>/dev/null || echo 0)
|
||||||
FILE_AGE=$(( NOW - FILE_MTIME ))
|
FILE_AGE=$(( NOW - FILE_MTIME ))
|
||||||
|
|
||||||
if is_video_file "$filepath"; then
|
if has_extension "$filepath" "${SONARR_EXTENSIONS[@]}"; then
|
||||||
[[ "$FILE_AGE" -lt "$AGE_SECONDS" ]] && \
|
[[ "$FILE_AGE" -lt "$AGE_SECONDS" ]] && \
|
||||||
[[ "$SKIP_AGE_CHECK" != true ]] && continue
|
[[ "$SKIP_AGE_CHECK" != true ]] && continue
|
||||||
fi
|
fi
|
||||||
|
|||||||
@@ -103,13 +103,7 @@ case "$ARR_TYPE" in
|
|||||||
esac
|
esac
|
||||||
|
|
||||||
# ── Remote node list ──────────────────────────────────────────────────────────────────────────
|
# ── Remote node list ──────────────────────────────────────────────────────────────────────────
|
||||||
declare -a REMOTE_NODES=()
|
discover_remote_nodes
|
||||||
for _hv in HOST1 HOST2 HOST3 HOST4 HOST5 HOST6 HOST7 HOST8; do
|
|
||||||
[[ "$_hv" == "$MY_ID" ]] && continue
|
|
||||||
[[ -z "${!_hv:-}" ]] && continue
|
|
||||||
REMOTE_NODES+=("$_hv")
|
|
||||||
done
|
|
||||||
unset _hv
|
|
||||||
|
|
||||||
if [[ "${#REMOTE_NODES[@]}" -eq 0 ]]; then
|
if [[ "${#REMOTE_NODES[@]}" -eq 0 ]]; then
|
||||||
echo "No remote nodes configured — nothing to push"
|
echo "No remote nodes configured — nothing to push"
|
||||||
|
|||||||
@@ -802,6 +802,9 @@
|
|||||||
# Crash loop threshold
|
# Crash loop threshold
|
||||||
WATCHDOG_CRASH_LIMIT=5 # RestartCount above this = critical crash loop
|
WATCHDOG_CRASH_LIMIT=5 # RestartCount above this = critical crash loop
|
||||||
|
|
||||||
|
# Required-container strikes — consecutive down-checks before docker_watchdog.sh attempts a restart
|
||||||
|
WATCHDOG_REQUIRED_STRIKE_LIMIT=2
|
||||||
|
|
||||||
# Startup grace period — skip restarts while system is still booting
|
# Startup grace period — skip restarts while system is still booting
|
||||||
WATCHDOG_STARTUP_GRACE=600 # seconds after boot before watchdog acts on failures
|
WATCHDOG_STARTUP_GRACE=600 # seconds after boot before watchdog acts on failures
|
||||||
|
|
||||||
@@ -891,6 +894,9 @@
|
|||||||
# Seconds of warning broadcast to logged-in users before server_reboot.sh reboots.
|
# Seconds of warning broadcast to logged-in users before server_reboot.sh reboots.
|
||||||
# Gives users time to save work — 300s = 5 minutes.
|
# Gives users time to save work — 300s = 5 minutes.
|
||||||
REBOOT_SLEEP=300
|
REBOOT_SLEEP=300
|
||||||
|
# Seconds to wait for graceful VM shutdown (ACPI signal via virsh) before libvirt
|
||||||
|
# stops it anyway — reboot takes priority.
|
||||||
|
REBOOT_VM_WAIT=30
|
||||||
|
|
||||||
# ━━━ Mover ━━━
|
# ━━━ Mover ━━━
|
||||||
# Seconds to wait before mover_stop.sh sends SIGTERM to the mover process.
|
# Seconds to wait before mover_stop.sh sends SIGTERM to the mover process.
|
||||||
@@ -1085,6 +1091,7 @@
|
|||||||
LIDARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run
|
LIDARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run
|
||||||
# protects against API returning partial data on a bad day
|
# protects against API returning partial data on a bad day
|
||||||
LIDARR_TRACKED_COUNT_FILE="$DATA_DIR/lidarr_tracked.count"
|
LIDARR_TRACKED_COUNT_FILE="$DATA_DIR/lidarr_tracked.count"
|
||||||
|
LIDARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan
|
||||||
LIDARR_EXTENSIONS=("flac" "mp3" "m4a" "wav" "aac" "ogg" "opus" "wma")
|
LIDARR_EXTENSIONS=("flac" "mp3" "m4a" "wav" "aac" "ogg" "opus" "wma")
|
||||||
LIDARR_PROTECTED_PATTERNS=(
|
LIDARR_PROTECTED_PATTERNS=(
|
||||||
# Metadata
|
# Metadata
|
||||||
@@ -1153,6 +1160,7 @@
|
|||||||
SONARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run
|
SONARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run
|
||||||
# protects against API returning partial data on a bad day
|
# protects against API returning partial data on a bad day
|
||||||
SONARR_TRACKED_COUNT_FILE="$DATA_DIR/sonarr_tracked.count"
|
SONARR_TRACKED_COUNT_FILE="$DATA_DIR/sonarr_tracked.count"
|
||||||
|
SONARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan
|
||||||
SONARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "ts" "wmv" "mov")
|
SONARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "ts" "wmv" "mov")
|
||||||
SONARR_PROTECTED_PATTERNS=(
|
SONARR_PROTECTED_PATTERNS=(
|
||||||
# Subtitles
|
# Subtitles
|
||||||
@@ -1179,6 +1187,7 @@
|
|||||||
RADARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run
|
RADARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run
|
||||||
# protects against API returning partial data on a bad day
|
# protects against API returning partial data on a bad day
|
||||||
RADARR_TRACKED_COUNT_FILE="$DATA_DIR/radarr_tracked.count"
|
RADARR_TRACKED_COUNT_FILE="$DATA_DIR/radarr_tracked.count"
|
||||||
|
RADARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan
|
||||||
RADARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "wmv" "mov")
|
RADARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "wmv" "mov")
|
||||||
RADARR_PROTECTED_PATTERNS=(
|
RADARR_PROTECTED_PATTERNS=(
|
||||||
# Subtitles
|
# Subtitles
|
||||||
|
|||||||
@@ -797,6 +797,9 @@
|
|||||||
# Crash loop threshold
|
# Crash loop threshold
|
||||||
WATCHDOG_CRASH_LIMIT=5 # RestartCount above this = critical crash loop
|
WATCHDOG_CRASH_LIMIT=5 # RestartCount above this = critical crash loop
|
||||||
|
|
||||||
|
# Required-container strikes — consecutive down-checks before docker_watchdog.sh attempts a restart
|
||||||
|
WATCHDOG_REQUIRED_STRIKE_LIMIT=2
|
||||||
|
|
||||||
# Startup grace period — skip restarts while system is still booting
|
# Startup grace period — skip restarts while system is still booting
|
||||||
WATCHDOG_STARTUP_GRACE=600 # seconds after boot before watchdog acts on failures
|
WATCHDOG_STARTUP_GRACE=600 # seconds after boot before watchdog acts on failures
|
||||||
|
|
||||||
@@ -886,6 +889,9 @@
|
|||||||
# Seconds of warning broadcast to logged-in users before server_reboot.sh reboots.
|
# Seconds of warning broadcast to logged-in users before server_reboot.sh reboots.
|
||||||
# Gives users time to save work — 300s = 5 minutes.
|
# Gives users time to save work — 300s = 5 minutes.
|
||||||
REBOOT_SLEEP=300
|
REBOOT_SLEEP=300
|
||||||
|
# Seconds to wait for graceful VM shutdown (ACPI signal via virsh) before libvirt
|
||||||
|
# stops it anyway — reboot takes priority.
|
||||||
|
REBOOT_VM_WAIT=30
|
||||||
|
|
||||||
# ━━━ Mover ━━━
|
# ━━━ Mover ━━━
|
||||||
# Seconds to wait before mover_stop.sh sends SIGTERM to the mover process.
|
# Seconds to wait before mover_stop.sh sends SIGTERM to the mover process.
|
||||||
@@ -1080,6 +1086,7 @@
|
|||||||
LIDARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run
|
LIDARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run
|
||||||
# protects against API returning partial data on a bad day
|
# protects against API returning partial data on a bad day
|
||||||
LIDARR_TRACKED_COUNT_FILE="$DATA_DIR/lidarr_tracked.count"
|
LIDARR_TRACKED_COUNT_FILE="$DATA_DIR/lidarr_tracked.count"
|
||||||
|
LIDARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan
|
||||||
LIDARR_EXTENSIONS=("flac" "mp3" "m4a" "wav" "aac" "ogg" "opus" "wma")
|
LIDARR_EXTENSIONS=("flac" "mp3" "m4a" "wav" "aac" "ogg" "opus" "wma")
|
||||||
LIDARR_PROTECTED_PATTERNS=(
|
LIDARR_PROTECTED_PATTERNS=(
|
||||||
# Metadata
|
# Metadata
|
||||||
@@ -1144,6 +1151,7 @@
|
|||||||
SONARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run
|
SONARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run
|
||||||
# protects against API returning partial data on a bad day
|
# protects against API returning partial data on a bad day
|
||||||
SONARR_TRACKED_COUNT_FILE="$DATA_DIR/sonarr_tracked.count"
|
SONARR_TRACKED_COUNT_FILE="$DATA_DIR/sonarr_tracked.count"
|
||||||
|
SONARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan
|
||||||
SONARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "ts" "wmv" "mov")
|
SONARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "ts" "wmv" "mov")
|
||||||
SONARR_PROTECTED_PATTERNS=(
|
SONARR_PROTECTED_PATTERNS=(
|
||||||
# Subtitles
|
# Subtitles
|
||||||
@@ -1168,6 +1176,7 @@
|
|||||||
RADARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run
|
RADARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run
|
||||||
# protects against API returning partial data on a bad day
|
# protects against API returning partial data on a bad day
|
||||||
RADARR_TRACKED_COUNT_FILE="$DATA_DIR/radarr_tracked.count"
|
RADARR_TRACKED_COUNT_FILE="$DATA_DIR/radarr_tracked.count"
|
||||||
|
RADARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan
|
||||||
RADARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "wmv" "mov")
|
RADARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "wmv" "mov")
|
||||||
RADARR_PROTECTED_PATTERNS=(
|
RADARR_PROTECTED_PATTERNS=(
|
||||||
# Subtitles
|
# Subtitles
|
||||||
|
|||||||
@@ -186,9 +186,7 @@ FAILED=()
|
|||||||
|
|
||||||
for container in "${RUNNING[@]}"; do
|
for container in "${RUNNING[@]}"; do
|
||||||
[[ -z "$container" ]] && continue
|
[[ -z "$container" ]] && continue
|
||||||
local c_start
|
|
||||||
c_start=$(date +%s)
|
c_start=$(date +%s)
|
||||||
local c_image
|
|
||||||
c_image=$(docker inspect --format '{{.Config.Image}}' "$container" 2>/dev/null || echo "unknown")
|
c_image=$(docker inspect --format '{{.Config.Image}}' "$container" 2>/dev/null || echo "unknown")
|
||||||
log "━━━ $ICON_CONTAINERS $container ($c_image) ━━━"
|
log "━━━ $ICON_CONTAINERS $container ($c_image) ━━━"
|
||||||
|
|
||||||
|
|||||||
@@ -159,71 +159,7 @@ fi
|
|||||||
|
|
||||||
# docker_cmd, verify_running, retry_docker — defined in common.sh
|
# docker_cmd, verify_running, retry_docker — defined in common.sh
|
||||||
|
|
||||||
# Builds a dependency-safe restart order from DAILY_RESTART_CONTAINERS.
|
# build_restart_order() / check_dependency_delay() — provided by common.sh
|
||||||
# Containers that are dependencies of others restart first.
|
|
||||||
# Returns ordered list in ORDERED_RESTART array.
|
|
||||||
build_restart_order() {
|
|
||||||
ORDERED_RESTART=()
|
|
||||||
local remaining=("${DAILY_RESTART_CONTAINERS[@]}")
|
|
||||||
local placed=()
|
|
||||||
|
|
||||||
# First pass — add dependency containers that appear in our list
|
|
||||||
for container in "${remaining[@]}"; do
|
|
||||||
[[ -z "$container" ]] && continue
|
|
||||||
local is_dependency=false
|
|
||||||
# Check if this container is a dependency of any other in our list
|
|
||||||
for dep_string in "${WATCHDOG_DEPENDENCIES[@]}"; do
|
|
||||||
if [[ "$dep_string" == *"$container"* ]]; then
|
|
||||||
is_dependency=true
|
|
||||||
break
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
# Also check associative array format
|
|
||||||
for dependent in "${!WATCHDOG_DEPENDENCIES[@]}"; do
|
|
||||||
if [[ "${WATCHDOG_DEPENDENCIES[$dependent]}" == *"$container"* ]]; then
|
|
||||||
is_dependency=true
|
|
||||||
break
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
if [[ "$is_dependency" == true ]]; then
|
|
||||||
# Check not already placed
|
|
||||||
local already=false
|
|
||||||
for p in "${placed[@]}"; do [[ "$p" == "$container" ]] && already=true && break; done
|
|
||||||
if [[ "$already" == false ]]; then
|
|
||||||
ORDERED_RESTART+=("$container")
|
|
||||||
placed+=("$container")
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
|
|
||||||
# Second pass — add remaining containers (dependents and independents)
|
|
||||||
for container in "${remaining[@]}"; do
|
|
||||||
[[ -z "$container" ]] && continue
|
|
||||||
local already=false
|
|
||||||
for p in "${placed[@]}"; do [[ "$p" == "$container" ]] && already=true && break; done
|
|
||||||
if [[ "$already" == false ]]; then
|
|
||||||
ORDERED_RESTART+=("$container")
|
|
||||||
placed+=("$container")
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
# Checks if a container is a dependent of the previously restarted container.
|
|
||||||
# If so, waits CONTAINER_DELAY before restarting to allow dependency to settle.
|
|
||||||
# Usage: check_dependency_delay "$container" "$last_restarted"
|
|
||||||
check_dependency_delay() {
|
|
||||||
local container="$1"
|
|
||||||
local last="$2"
|
|
||||||
[[ -z "$last" ]] && return
|
|
||||||
|
|
||||||
local deps="${WATCHDOG_DEPENDENCIES[$container]:-}"
|
|
||||||
if [[ -n "$deps" ]] && [[ "$deps" == *"$last"* ]]; then
|
|
||||||
echo " Waiting ${CONTAINER_DELAY}s — $container depends on $last..."
|
|
||||||
sleep "$CONTAINER_DELAY"
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Daily Restart ━━━
|
# ━━━ Daily Restart ━━━
|
||||||
@@ -241,8 +177,7 @@ RESTARTED=()
|
|||||||
SKIPPED=()
|
SKIPPED=()
|
||||||
|
|
||||||
# Build dependency-safe restart order
|
# Build dependency-safe restart order
|
||||||
build_restart_order
|
build_restart_order DAILY_RESTART_CONTAINERS
|
||||||
log "$ICON_GEAR Restart order: ${ORDERED_RESTART[*]}"
|
|
||||||
|
|
||||||
LAST_RESTARTED=""
|
LAST_RESTARTED=""
|
||||||
|
|
||||||
|
|||||||
@@ -158,58 +158,7 @@ fi
|
|||||||
|
|
||||||
# docker_cmd, retry_docker, verify_running — defined in common.sh
|
# docker_cmd, retry_docker, verify_running — defined in common.sh
|
||||||
|
|
||||||
# Builds a dependency-safe restart order from WEEKLY_RESTART_CONTAINERS.
|
# build_restart_order() / check_dependency_delay() — provided by common.sh
|
||||||
# Containers that are dependencies of others restart first.
|
|
||||||
build_restart_order() {
|
|
||||||
ORDERED_RESTART=()
|
|
||||||
local remaining=("${WEEKLY_RESTART_CONTAINERS[@]}")
|
|
||||||
local placed=()
|
|
||||||
|
|
||||||
# First pass — add dependency containers that appear in our list
|
|
||||||
for container in "${remaining[@]}"; do
|
|
||||||
[[ -z "$container" ]] && continue
|
|
||||||
local is_dependency=false
|
|
||||||
for dependent in "${!WATCHDOG_DEPENDENCIES[@]}"; do
|
|
||||||
if [[ "${WATCHDOG_DEPENDENCIES[$dependent]}" == *"$container"* ]]; then
|
|
||||||
is_dependency=true
|
|
||||||
break
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
if [[ "$is_dependency" == true ]]; then
|
|
||||||
local already=false
|
|
||||||
for p in "${placed[@]}"; do [[ "$p" == "$container" ]] && already=true && break; done
|
|
||||||
if [[ "$already" == false ]]; then
|
|
||||||
ORDERED_RESTART+=("$container")
|
|
||||||
placed+=("$container")
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
|
|
||||||
# Second pass — add remaining containers (dependents and independents)
|
|
||||||
for container in "${remaining[@]}"; do
|
|
||||||
[[ -z "$container" ]] && continue
|
|
||||||
local already=false
|
|
||||||
for p in "${placed[@]}"; do [[ "$p" == "$container" ]] && already=true && break; done
|
|
||||||
if [[ "$already" == false ]]; then
|
|
||||||
ORDERED_RESTART+=("$container")
|
|
||||||
placed+=("$container")
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
|
|
||||||
log "Restart order: ${ORDERED_RESTART[*]}"
|
|
||||||
}
|
|
||||||
|
|
||||||
# Waits CONTAINER_DELAY if this container depends on the last restarted one.
|
|
||||||
check_dependency_delay() {
|
|
||||||
local container="$1"
|
|
||||||
local last="$2"
|
|
||||||
[[ -z "$last" ]] && return
|
|
||||||
local deps="${WATCHDOG_DEPENDENCIES[$container]:-}"
|
|
||||||
if [[ -n "$deps" ]] && [[ "$deps" == *"$last"* ]]; then
|
|
||||||
log "Waiting ${CONTAINER_DELAY}s — $container depends on $last..."
|
|
||||||
sleep "$CONTAINER_DELAY"
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Weekly Restart ━━━
|
# ━━━ Weekly Restart ━━━
|
||||||
@@ -226,15 +175,13 @@ RESTARTED=()
|
|||||||
SKIPPED=()
|
SKIPPED=()
|
||||||
|
|
||||||
# Build dependency-safe restart order
|
# Build dependency-safe restart order
|
||||||
build_restart_order
|
build_restart_order WEEKLY_RESTART_CONTAINERS
|
||||||
|
|
||||||
LAST_RESTARTED=""
|
LAST_RESTARTED=""
|
||||||
|
|
||||||
for container in "${ORDERED_RESTART[@]}"; do
|
for container in "${ORDERED_RESTART[@]}"; do
|
||||||
[[ -z "$container" ]] && continue
|
[[ -z "$container" ]] && continue
|
||||||
local c_start
|
|
||||||
c_start=$(date +%s)
|
c_start=$(date +%s)
|
||||||
local c_image
|
|
||||||
c_image=$(docker inspect --format '{{.Config.Image}}' "$container" 2>/dev/null || echo "unknown")
|
c_image=$(docker inspect --format '{{.Config.Image}}' "$container" 2>/dev/null || echo "unknown")
|
||||||
log "━━━ $ICON_CONTAINERS $container ($c_image) ━━━"
|
log "━━━ $ICON_CONTAINERS $container ($c_image) ━━━"
|
||||||
|
|
||||||
|
|||||||
@@ -602,11 +602,13 @@ if [[ -n "$QBIT_URL" ]] && [[ -n "$QBIT_USERNAME" ]]; then
|
|||||||
# Age check — must be old enough
|
# Age check — must be old enough
|
||||||
[[ "$AGE_DAYS" -lt "$QBIT_FAILSAFE_MIN_DAYS" ]] && ((SKIPPED++)) && continue
|
[[ "$AGE_DAYS" -lt "$QBIT_FAILSAFE_MIN_DAYS" ]] && ((SKIPPED++)) && continue
|
||||||
|
|
||||||
# Ratio check — if configured
|
# Ratio check — if configured. awk handles the fractional comparison;
|
||||||
|
# bash's [[ -lt ]] only does integers and would treat e.g. 1.4 and 1.5
|
||||||
|
# as equal once truncated. Missing/empty ratio defaults to 0 (protected).
|
||||||
if [[ "$QBIT_FAILSAFE_MIN_RATIO" != "0" ]]; then
|
if [[ "$QBIT_FAILSAFE_MIN_RATIO" != "0" ]]; then
|
||||||
RATIO_INT="${RATIO%.*}"
|
if (( $(awk "BEGIN {print (${RATIO:-0} < $QBIT_FAILSAFE_MIN_RATIO) ? 1 : 0}") )); then
|
||||||
MIN_RATIO_INT="${QBIT_FAILSAFE_MIN_RATIO%.*}"
|
((SKIPPED++)) && continue
|
||||||
[[ "$RATIO_INT" -lt "$MIN_RATIO_INT" ]] && ((SKIPPED++)) && continue
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [[ "$DRY_RUN" == true ]]; then
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
|
|||||||
@@ -1000,12 +1000,19 @@ while [[ "$FALLBACK_RUNNING" == true ]]; do
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
elif [[ "$INTERNET_UP" == false ]]; then
|
elif [[ "$INTERNET_UP" == false ]]; then
|
||||||
# Lost internet during fallback — enter DARK
|
# Lost internet during fallback — enter DARK — same actions as NO_INTERNET
|
||||||
echo ""
|
echo ""
|
||||||
echo "━━━ $ICON_NET Entering DARK — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
echo "━━━ $ICON_NET Entering DARK — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||||
warn "Lost internet during fallback — entering DARK state"
|
warn "Lost internet during fallback — entering DARK state"
|
||||||
state_set state "DARK"
|
state_set state "DARK"
|
||||||
local_ddns_stop
|
local_ddns_stop
|
||||||
|
|
||||||
|
# Stop containers configured to stop on internet loss
|
||||||
|
read -r -a stop_on_no_net <<< "$(get_stop_on_no_net)"
|
||||||
|
for container in "${stop_on_no_net[@]}"; do
|
||||||
|
[[ -n "$container" ]] && local_stop "$container"
|
||||||
|
done
|
||||||
|
|
||||||
notify "DARK state on $(hostname) — lost internet during fallback" \
|
notify "DARK state on $(hostname) — lost internet during fallback" \
|
||||||
"Fallback" "warning"
|
"Fallback" "warning"
|
||||||
|
|
||||||
|
|||||||
@@ -65,6 +65,7 @@ score_candidate() {
|
|||||||
local popularity_score="${2:-0}"
|
local popularity_score="${2:-0}"
|
||||||
local recency_score="${3:-0}"
|
local recency_score="${3:-0}"
|
||||||
local quality_score="${4:-0}"
|
local quality_score="${4:-0}"
|
||||||
|
local TOTAL_SCORE
|
||||||
|
|
||||||
TOTAL_SCORE=$(( \
|
TOTAL_SCORE=$(( \
|
||||||
user_score + \
|
user_score + \
|
||||||
|
|||||||
+12
-31
@@ -159,43 +159,24 @@ fi
|
|||||||
check_cert() {
|
check_cert() {
|
||||||
local domain="$1"
|
local domain="$1"
|
||||||
local port="${2:-443}"
|
local port="${2:-443}"
|
||||||
_CERT_DAYS=""
|
|
||||||
_CERT_EXPIRY=""
|
|
||||||
|
|
||||||
local expiry_str
|
if ! check_cert_expiry "$domain" "$port" "$CERT_TIMEOUT"; then
|
||||||
expiry_str=$(echo | timeout "$CERT_TIMEOUT" openssl s_client \
|
if [[ -n "$_CERT_EXPIRY_RAW" ]]; then
|
||||||
-connect "${domain}:${port}" \
|
error "$ICON_CERT $domain — could not parse expiry date: $_CERT_EXPIRY_RAW"
|
||||||
-servername "$domain" \
|
else
|
||||||
2>/dev/null | openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2)
|
error "$ICON_CERT $domain — could not retrieve certificate (unreachable or no TLS)"
|
||||||
|
fi
|
||||||
if [[ -z "$expiry_str" ]]; then
|
|
||||||
error "$ICON_CERT $domain — could not retrieve certificate (unreachable or no TLS)"
|
|
||||||
return 3
|
return 3
|
||||||
fi
|
fi
|
||||||
|
|
||||||
local expiry_epoch
|
if [[ "$_CERT_DAYS" -le "$CERT_CRIT_DAYS" ]]; then
|
||||||
expiry_epoch=$(date -d "$expiry_str" +%s 2>/dev/null)
|
error "$ICON_CERT $domain — CRITICAL: ${_CERT_DAYS} days remaining (expires $_CERT_EXPIRY)"
|
||||||
|
|
||||||
if [[ -z "$expiry_epoch" ]]; then
|
|
||||||
error "$ICON_CERT $domain — could not parse expiry date: $expiry_str"
|
|
||||||
return 3
|
|
||||||
fi
|
|
||||||
|
|
||||||
local now days_remaining expiry_display
|
|
||||||
now=$(date +%s)
|
|
||||||
days_remaining=$(( (expiry_epoch - now) / 86400 ))
|
|
||||||
expiry_display=$(date -d "$expiry_str" '+%Y-%m-%d' 2>/dev/null)
|
|
||||||
_CERT_DAYS=$days_remaining
|
|
||||||
_CERT_EXPIRY=$expiry_display
|
|
||||||
|
|
||||||
if [[ "$days_remaining" -le "$CERT_CRIT_DAYS" ]]; then
|
|
||||||
error "$ICON_CERT $domain — CRITICAL: ${days_remaining} days remaining (expires $expiry_display)"
|
|
||||||
return 2
|
return 2
|
||||||
elif [[ "$days_remaining" -le "$CERT_WARN_DAYS" ]]; then
|
elif [[ "$_CERT_DAYS" -le "$CERT_WARN_DAYS" ]]; then
|
||||||
warn "$ICON_CERT $domain — WARNING: ${days_remaining} days remaining (expires $expiry_display)"
|
warn "$ICON_CERT $domain — WARNING: ${_CERT_DAYS} days remaining (expires $_CERT_EXPIRY)"
|
||||||
return 1
|
return 1
|
||||||
else
|
else
|
||||||
log "$ICON_CERT $domain — OK: ${days_remaining} days remaining (expires $expiry_display)"
|
log "$ICON_CERT $domain — OK: ${_CERT_DAYS} days remaining (expires $_CERT_EXPIRY)"
|
||||||
return 0
|
return 0
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
@@ -285,7 +266,7 @@ fi
|
|||||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||||
|
|
||||||
# ── Write JSON status cache ───────────────────────────────────────────────────
|
# ── Write JSON status cache ───────────────────────────────────────────────────
|
||||||
_CERT_CACHE_FILE="$SCRIPTS_DIR/State_Files/cert_status.json"
|
_CERT_CACHE_FILE="$STATE_DIR/cert_status.json"
|
||||||
{
|
{
|
||||||
printf '{"checked_at":%d,"host":"%s","warn_days":%d,"crit_days":%d,"dry_run":%s,"domains":[\n' \
|
printf '{"checked_at":%d,"host":"%s","warn_days":%d,"crit_days":%d,"dry_run":%s,"domains":[\n' \
|
||||||
"$(date +%s)" "$MY_ID" "$CERT_WARN_DAYS" "$CERT_CRIT_DAYS" \
|
"$(date +%s)" "$MY_ID" "$CERT_WARN_DAYS" "$CERT_CRIT_DAYS" \
|
||||||
|
|||||||
@@ -316,7 +316,7 @@ echo ""
|
|||||||
echo "━━━ $ICON_RAM Transcode Status ━━━"
|
echo "━━━ $ICON_RAM Transcode Status ━━━"
|
||||||
if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
|
if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
|
||||||
RAMDISK_USED_KB=$(df "$RAMDISK_PATH" --output=used 2>/dev/null | tail -1 | tr -d ' ')
|
RAMDISK_USED_KB=$(df "$RAMDISK_PATH" --output=used 2>/dev/null | tail -1 | tr -d ' ')
|
||||||
RAMDISK_USED_GB=$(awk "BEGIN {printf \"%.2f\", ${RAMDISK_USED_KB:-0} / 1048576}")
|
RAMDISK_USED_GB=$(kb_to_gb "$RAMDISK_USED_KB")
|
||||||
SYMLINK=$(readlink "$TRANSCODE_LINK" 2>/dev/null || echo "unknown")
|
SYMLINK=$(readlink "$TRANSCODE_LINK" 2>/dev/null || echo "unknown")
|
||||||
echo " $ICON_RAM Ramdisk usage: ${RAMDISK_USED_GB}GB / ${RAMDISK_SIZE:-8G}"
|
echo " $ICON_RAM Ramdisk usage: ${RAMDISK_USED_GB}GB / ${RAMDISK_SIZE:-8G}"
|
||||||
echo " $ICON_LINK Symlink target: $SYMLINK"
|
echo " $ICON_LINK Symlink target: $SYMLINK"
|
||||||
|
|||||||
@@ -156,11 +156,7 @@ if [[ "$SHOW_STATUS" == true ]]; then
|
|||||||
for drive in /dev/sd? /dev/nvme?; do
|
for drive in /dev/sd? /dev/nvme?; do
|
||||||
[[ ! -e "$drive" ]] && continue
|
[[ ! -e "$drive" ]] && continue
|
||||||
drive_name=$(basename "$drive")
|
drive_name=$(basename "$drive")
|
||||||
ignored=false
|
if is_in_list "$drive_name" "${SMART_IGNORE_DRIVES[@]}"; then
|
||||||
for ignore in "${SMART_IGNORE_DRIVES[@]}"; do
|
|
||||||
[[ "$drive_name" == "$ignore" ]] && ignored=true && break
|
|
||||||
done
|
|
||||||
if [[ "$ignored" == true ]]; then
|
|
||||||
echo " $ICON_WARN $drive — ignored"
|
echo " $ICON_WARN $drive — ignored"
|
||||||
else
|
else
|
||||||
echo " $ICON_SMART $drive — would check"
|
echo " $ICON_SMART $drive — would check"
|
||||||
@@ -201,18 +197,7 @@ get_drive_temp() {
|
|||||||
echo "${temp:-}"
|
echo "${temp:-}"
|
||||||
}
|
}
|
||||||
|
|
||||||
# Detect if a drive is SSD/NVMe (rotational=0)
|
# is_ssd() — provided by common.sh
|
||||||
is_ssd() {
|
|
||||||
local drive="$1"
|
|
||||||
local dev_name
|
|
||||||
dev_name=$(basename "$drive" | sed 's/nvme[0-9]/nvme0/')
|
|
||||||
local rotational="/sys/block/$(basename "$drive")/queue/rotational"
|
|
||||||
[[ -f "$rotational" ]] && [[ "$(cat "$rotational" 2>/dev/null)" == "0" ]] && return 0
|
|
||||||
# NVMe is always SSD
|
|
||||||
[[ "$drive" == *nvme* ]] && return 0
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ SMART Health Check ━━━
|
# ━━━ SMART Health Check ━━━
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
@@ -232,12 +217,7 @@ for drive in /dev/sd? /dev/nvme?; do
|
|||||||
drive_name=$(basename "$drive")
|
drive_name=$(basename "$drive")
|
||||||
|
|
||||||
# Check ignore list
|
# Check ignore list
|
||||||
ignored=false
|
if is_in_list "$drive_name" "${SMART_IGNORE_DRIVES[@]}"; then
|
||||||
for ignore in "${SMART_IGNORE_DRIVES[@]}"; do
|
|
||||||
[[ "$drive_name" == "$ignore" ]] && ignored=true && break
|
|
||||||
done
|
|
||||||
|
|
||||||
if [[ "$ignored" == true ]]; then
|
|
||||||
log "$drive_name — ignored (SMART_IGNORE_DRIVES)"
|
log "$drive_name — ignored (SMART_IGNORE_DRIVES)"
|
||||||
DRIVES_SKIP+=("$drive_name")
|
DRIVES_SKIP+=("$drive_name")
|
||||||
continue
|
continue
|
||||||
|
|||||||
@@ -270,7 +270,7 @@ fi
|
|||||||
# ── Transcode Ramdisk ─────────────────────────────────────────────────────────────────────────
|
# ── Transcode Ramdisk ─────────────────────────────────────────────────────────────────────────
|
||||||
if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
|
if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
|
||||||
RAMDISK_USED_KB=$(df "$RAMDISK_PATH" --output=used 2>/dev/null | tail -1 | tr -d ' ')
|
RAMDISK_USED_KB=$(df "$RAMDISK_PATH" --output=used 2>/dev/null | tail -1 | tr -d ' ')
|
||||||
RAMDISK_USED_GB=$(awk "BEGIN {printf \"%.2f\", ${RAMDISK_USED_KB:-0} / 1048576}")
|
RAMDISK_USED_GB=$(kb_to_gb "$RAMDISK_USED_KB")
|
||||||
SYMLINK_TARGET=$(readlink "$TRANSCODE_LINK" 2>/dev/null || echo "unknown")
|
SYMLINK_TARGET=$(readlink "$TRANSCODE_LINK" 2>/dev/null || echo "unknown")
|
||||||
DIGEST_LINES+=("$ICON_RAM Transcodes: ${RAMDISK_USED_GB}GB used → $SYMLINK_TARGET")
|
DIGEST_LINES+=("$ICON_RAM Transcodes: ${RAMDISK_USED_GB}GB used → $SYMLINK_TARGET")
|
||||||
|
|
||||||
@@ -309,7 +309,7 @@ if [[ -f "${BANDWIDTH_LOG:-}" ]] && [[ -s "$BANDWIDTH_LOG" ]]; then
|
|||||||
YESTERDAY=$(date -d "yesterday" '+%Y-%m-%d')
|
YESTERDAY=$(date -d "yesterday" '+%Y-%m-%d')
|
||||||
YESTERDAY_BYTES=$(awk -F'|' -v d="$YESTERDAY" '$1==d{sum+=$6} END{print sum+0}' \
|
YESTERDAY_BYTES=$(awk -F'|' -v d="$YESTERDAY" '$1==d{sum+=$6} END{print sum+0}' \
|
||||||
"$BANDWIDTH_LOG")
|
"$BANDWIDTH_LOG")
|
||||||
YESTERDAY_GB=$(awk "BEGIN {printf \"%.2f\", ${YESTERDAY_BYTES:-0} / 1073741824}")
|
YESTERDAY_GB=$(bytes_to_gb "$YESTERDAY_BYTES")
|
||||||
YESTERDAY_LARGE=$(awk -F'|' -v d="$YESTERDAY" '$1==d && $7=="LARGE"' \
|
YESTERDAY_LARGE=$(awk -F'|' -v d="$YESTERDAY" '$1==d && $7=="LARGE"' \
|
||||||
"$BANDWIDTH_LOG" | wc -l)
|
"$BANDWIDTH_LOG" | wc -l)
|
||||||
|
|
||||||
@@ -329,12 +329,8 @@ if [[ ${#CERT_MONITOR_DOMAINS[@]} -gt 0 ]] && command -v openssl >/dev/null 2>&1
|
|||||||
CERT_ISSUES=()
|
CERT_ISSUES=()
|
||||||
for domain in "${CERT_MONITOR_DOMAINS[@]}"; do
|
for domain in "${CERT_MONITOR_DOMAINS[@]}"; do
|
||||||
[[ -z "$domain" ]] && continue
|
[[ -z "$domain" ]] && continue
|
||||||
expiry_str=$(echo | timeout "${CERT_TIMEOUT:-10}" openssl s_client \
|
if check_cert_expiry "$domain" 443 "${CERT_TIMEOUT:-10}"; then
|
||||||
-connect "${domain}:443" -servername "$domain" \
|
days_remaining="$_CERT_DAYS"
|
||||||
2>/dev/null | openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2)
|
|
||||||
if [[ -n "$expiry_str" ]]; then
|
|
||||||
expiry_epoch=$(date -d "$expiry_str" +%s 2>/dev/null)
|
|
||||||
days_remaining=$(( (expiry_epoch - $(date +%s)) / 86400 ))
|
|
||||||
if [[ "$days_remaining" -le "${CERT_CRIT_DAYS:-7}" ]]; then
|
if [[ "$days_remaining" -le "${CERT_CRIT_DAYS:-7}" ]]; then
|
||||||
CERT_ISSUES+=("$domain: ${days_remaining}d CRITICAL")
|
CERT_ISSUES+=("$domain: ${days_remaining}d CRITICAL")
|
||||||
SHOULD_SEND=true
|
SHOULD_SEND=true
|
||||||
|
|||||||
@@ -261,9 +261,9 @@ else
|
|||||||
ARC_SIZE=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats)
|
ARC_SIZE=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats)
|
||||||
ARC_META_USED=$(awk '/^arc_meta_used / {print $3}' /proc/spl/kstat/zfs/arcstats)
|
ARC_META_USED=$(awk '/^arc_meta_used / {print $3}' /proc/spl/kstat/zfs/arcstats)
|
||||||
|
|
||||||
ARC_MAX_GB=$(awk "BEGIN {printf \"%.1f\", $ARC_MAX / 1073741824}")
|
ARC_MAX_GB=$(bytes_to_gb "$ARC_MAX" 1)
|
||||||
ARC_CUR_GB=$(awk "BEGIN {printf \"%.1f\", $ARC_SIZE / 1073741824}")
|
ARC_CUR_GB=$(bytes_to_gb "$ARC_SIZE" 1)
|
||||||
ARC_META_GB=$(awk "BEGIN {printf \"%.1f\", $ARC_META_USED / 1073741824}")
|
ARC_META_GB=$(bytes_to_gb "$ARC_META_USED" 1)
|
||||||
ARC_PCT=$(awk "BEGIN {printf \"%.1f\", $ARC_SIZE * 100 / $ARC_MAX}")
|
ARC_PCT=$(awk "BEGIN {printf \"%.1f\", $ARC_SIZE * 100 / $ARC_MAX}")
|
||||||
ARC_PCT_INT=$(printf "%.0f" "$ARC_PCT")
|
ARC_PCT_INT=$(printf "%.0f" "$ARC_PCT")
|
||||||
|
|
||||||
@@ -287,8 +287,8 @@ else
|
|||||||
META_MISSES=$(awk '/^demand_metadata_misses / {print $3}' \
|
META_MISSES=$(awk '/^demand_metadata_misses / {print $3}' \
|
||||||
/proc/spl/kstat/zfs/arcstats 2>/dev/null || echo 0)
|
/proc/spl/kstat/zfs/arcstats 2>/dev/null || echo 0)
|
||||||
|
|
||||||
MRU_GB=$(awk "BEGIN {printf \"%.2f\", $META_MRU_GHOST / 1073741824}")
|
MRU_GB=$(bytes_to_gb "$META_MRU_GHOST")
|
||||||
MFU_GB=$(awk "BEGIN {printf \"%.2f\", $META_MFU_GHOST / 1073741824}")
|
MFU_GB=$(bytes_to_gb "$META_MFU_GHOST")
|
||||||
|
|
||||||
echo " $ICON_ZFS MRU Ghost: ${MRU_GB}GB"
|
echo " $ICON_ZFS MRU Ghost: ${MRU_GB}GB"
|
||||||
echo " $ICON_ZFS MFU Ghost: ${MFU_GB}GB"
|
echo " $ICON_ZFS MFU Ghost: ${MFU_GB}GB"
|
||||||
|
|||||||
@@ -88,8 +88,9 @@
|
|||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||||
|
|
||||||
source "$SCRIPT_DIR/../load_config.sh"
|
source "$ECOSYSTEM_ROOT/load_config.sh"
|
||||||
|
|
||||||
parse_args "$@"
|
parse_args "$@"
|
||||||
|
|
||||||
@@ -158,17 +159,6 @@ if [[ "$SHOW_STATUS" == true ]]; then
|
|||||||
exit 0
|
exit 0
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# ==============================================================================================
|
|
||||||
# ━━━ Validate Child Scripts ━━━
|
|
||||||
# ==============================================================================================
|
|
||||||
for _entry in "${TRANSCODE_MANAGEMENT_SCRIPTS[@]}"; do
|
|
||||||
_script_path="$SCRIPT_DIR/../$_entry"
|
|
||||||
if [[ ! -f "$_script_path" ]]; then
|
|
||||||
error "$(basename "$_entry") not found: $_script_path"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Pre-run State Snapshot ━━━
|
# ━━━ Pre-run State Snapshot ━━━
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
@@ -188,36 +178,22 @@ fi
|
|||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# transcode_manager.sh writes to TRANSCODE_DAILY_LOG after each run — no flag here
|
# transcode_manager.sh writes to TRANSCODE_DAILY_LOG after each run — no flag here
|
||||||
# suppresses that; it owns the log write for this cycle regardless of position ✅
|
# suppresses that; it owns the log write for this cycle regardless of position ✅
|
||||||
DRY_FLAG=""
|
JOB_PASS=()
|
||||||
[[ "$DRY_RUN" == true ]] && DRY_FLAG="--dry-run"
|
JOB_FAIL=()
|
||||||
|
|
||||||
WORST_EXIT=0
|
|
||||||
PASS_COUNT=0
|
|
||||||
FAIL_NAMES=()
|
|
||||||
for _entry in "${TRANSCODE_MANAGEMENT_SCRIPTS[@]}"; do
|
for _entry in "${TRANSCODE_MANAGEMENT_SCRIPTS[@]}"; do
|
||||||
_script_path="$SCRIPT_DIR/../$_entry"
|
[[ -z "$_entry" ]] && continue
|
||||||
_script_name=$(basename "$_entry" .sh)
|
run_orch_child "$_entry"
|
||||||
_start=$(date +%s)
|
|
||||||
bash "$_script_path" $DRY_FLAG
|
|
||||||
_exit=$?
|
|
||||||
log "$_script_name: $(format_duration $(( $(date +%s) - _start ))) (exit $_exit)"
|
|
||||||
if [[ "$_exit" -ne 0 ]]; then
|
|
||||||
WORST_EXIT=1
|
|
||||||
FAIL_NAMES+=("$_script_name")
|
|
||||||
else
|
|
||||||
PASS_COUNT=$(( PASS_COUNT + 1 ))
|
|
||||||
fi
|
|
||||||
done
|
done
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Summary — minimal one-liner by default (7-min cadence — keep it quiet when healthy) ━━━
|
# ━━━ Summary — minimal one-liner by default (7-min cadence — keep it quiet when healthy) ━━━
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
if [[ "$WORST_EXIT" -eq 0 ]]; then
|
if [[ "${#JOB_FAIL[@]}" -eq 0 ]]; then
|
||||||
echo "$ICON_SUCCESS Transcode cycle — $PASS_COUNT/${#TRANSCODE_MANAGEMENT_SCRIPTS[@]} passed"
|
echo "$ICON_SUCCESS Transcode cycle — ${#JOB_PASS[@]}/${#TRANSCODE_MANAGEMENT_SCRIPTS[@]} passed"
|
||||||
else
|
else
|
||||||
error "Transcode cycle — failed: ${FAIL_NAMES[*]}"
|
error "Transcode cycle — failed: ${JOB_FAIL[*]}"
|
||||||
if [[ "$DRY_RUN" != true ]]; then
|
if [[ "$DRY_RUN" != true ]]; then
|
||||||
notify "Transcode management failure on $(hostname) ($MY_ID) — ${FAIL_NAMES[*]}" \
|
notify "Transcode management failure on $(hostname) ($MY_ID) — ${JOB_FAIL[*]}" \
|
||||||
"Transcode Management" "warning"
|
"Transcode Management" "warning"
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
@@ -225,5 +201,5 @@ fi
|
|||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Exit ━━━
|
# ━━━ Exit ━━━
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# Return worst exit code — caller knows if any script failed
|
[[ "${#JOB_FAIL[@]}" -gt 0 ]] && exit 1
|
||||||
exit "$WORST_EXIT"
|
exit 0
|
||||||
@@ -177,40 +177,12 @@ log "Startup grace: past — uptime $(format_duration $UPTIME_SECONDS)"
|
|||||||
# ━━━ Run Watchdog Cycle ━━━
|
# ━━━ Run Watchdog Cycle ━━━
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
CYCLE_START=$(date +%s)
|
CYCLE_START=$(date +%s)
|
||||||
PASS=()
|
JOB_PASS=()
|
||||||
FAIL=()
|
JOB_FAIL=()
|
||||||
|
|
||||||
run_watchdog() {
|
|
||||||
local name="$1" script="$2"
|
|
||||||
|
|
||||||
if [[ ! -f "$script" ]]; then
|
|
||||||
error "$name — not found: $script"
|
|
||||||
FAIL+=("$name:missing")
|
|
||||||
return 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
[[ ! -x "$script" ]] && chmod +x "$script"
|
|
||||||
|
|
||||||
local extra_args=()
|
|
||||||
[[ "$DRY_RUN" == true ]] && extra_args+=("--dry-run")
|
|
||||||
[[ "$ENABLE_LOGGING" == true ]] && extra_args+=("--log")
|
|
||||||
|
|
||||||
local _ws
|
|
||||||
_ws=$(date +%s)
|
|
||||||
log "$ICON_START $name"
|
|
||||||
if bash "$script" "${extra_args[@]}"; then
|
|
||||||
log "$ICON_DONE $name — done in $(format_duration $(( $(date +%s) - _ws )))"
|
|
||||||
PASS+=("$name")
|
|
||||||
return 0
|
|
||||||
else
|
|
||||||
error "$name — non-zero exit ($(format_duration $(( $(date +%s) - _ws ))))"
|
|
||||||
FAIL+=("$name")
|
|
||||||
return 1
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
for _entry in "${WATCHDOG_ORCHESTRATOR_SCRIPTS[@]}"; do
|
for _entry in "${WATCHDOG_ORCHESTRATOR_SCRIPTS[@]}"; do
|
||||||
run_watchdog "$(_watchdog_display_name "$_entry")" "$ECOSYSTEM_ROOT/$_entry"
|
[[ -z "$_entry" ]] && continue
|
||||||
|
run_orch_child "$_entry"
|
||||||
done
|
done
|
||||||
|
|
||||||
CYCLE_END=$(date +%s)
|
CYCLE_END=$(date +%s)
|
||||||
@@ -236,19 +208,19 @@ fi
|
|||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Summary — minimal one-liner by default, full breakdown on failure or --log ━━━
|
# ━━━ Summary — minimal one-liner by default, full breakdown on failure or --log ━━━
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
if [[ "${#FAIL[@]}" -gt 0 || "$ENABLE_LOGGING" == true ]]; then
|
if [[ "${#JOB_FAIL[@]}" -gt 0 || "$ENABLE_LOGGING" == true ]]; then
|
||||||
echo ""
|
echo ""
|
||||||
echo "━━━━━ $ICON_SUMMARY WATCHDOG CYCLE — $MY_ID — $(date '+%H:%M:%S') ━━━━━"
|
echo "━━━━━ $ICON_SUMMARY WATCHDOG CYCLE — $MY_ID — $(date '+%H:%M:%S') ━━━━━"
|
||||||
for p in "${PASS[@]}"; do log " $ICON_DONE $p"; done
|
for p in "${JOB_PASS[@]}"; do log " $ICON_DONE $p"; done
|
||||||
for f in "${FAIL[@]}"; do error " $ICON_ERROR $f"; done
|
for f in "${JOB_FAIL[@]}"; do error " $ICON_ERROR $f"; done
|
||||||
echo "$ICON_TIME Duration: $(format_duration $DURATION)"
|
echo "$ICON_TIME Duration: $(format_duration $DURATION)"
|
||||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||||
else
|
else
|
||||||
echo "$ICON_DONE Watchdog cycle — ${#PASS[@]}/${#WATCHDOG_ORCHESTRATOR_SCRIPTS[@]} passed ($(format_duration $DURATION))"
|
echo "$ICON_DONE Watchdog cycle — ${#JOB_PASS[@]}/${#WATCHDOG_ORCHESTRATOR_SCRIPTS[@]} passed ($(format_duration $DURATION))"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [[ "${#FAIL[@]}" -gt 0 ]]; then
|
if [[ "${#JOB_FAIL[@]}" -gt 0 ]]; then
|
||||||
notify "Watchdog cycle failure on $(hostname) ($MY_ID) — ${FAIL[*]}" \
|
notify "Watchdog cycle failure on $(hostname) ($MY_ID) — ${JOB_FAIL[*]}" \
|
||||||
"Watchdog Orchestrator" "warning"
|
"Watchdog Orchestrator" "warning"
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|||||||
@@ -249,22 +249,7 @@ fi
|
|||||||
detect_hosts
|
detect_hosts
|
||||||
|
|
||||||
# ── Derive owner and mirror from PARTNERSHIP_OWNER_HOST ───────────────────────────────────────
|
# ── Derive owner and mirror from PARTNERSHIP_OWNER_HOST ───────────────────────────────────────
|
||||||
OWNER_ID="${PARTNERSHIP_OWNER_HOST:-HOST1}" # e.g. "HOST1"
|
partnership_resolve_roles
|
||||||
MIRROR_ID=$( [[ "$OWNER_ID" == "HOST1" ]] && echo "HOST2" || echo "HOST1" )
|
|
||||||
|
|
||||||
OWNER="${!OWNER_ID}" # hostname string
|
|
||||||
MIRROR="${!MIRROR_ID}"
|
|
||||||
# SSH_KEY (set by detect_hosts) is this server's own private key.
|
|
||||||
# The remote accepts it because this server's PUBLIC key was installed there via ssh_setup.sh.
|
|
||||||
# With sparse checkout, each server only has its own host{N}.conf — the other server's
|
|
||||||
# key path is never available here. Use SSH_KEY for all outbound SSH regardless of mode.
|
|
||||||
MIRROR_SSH_KEY="$SSH_KEY"
|
|
||||||
OWNER_SSH_KEY="$SSH_KEY"
|
|
||||||
|
|
||||||
AM_OWNER=false
|
|
||||||
AM_MIRROR=false
|
|
||||||
[[ "$MY_ID" == "$OWNER_ID" ]] && AM_OWNER=true
|
|
||||||
[[ "$MY_ID" == "$MIRROR_ID" ]] && AM_MIRROR=true
|
|
||||||
|
|
||||||
# State files
|
# State files
|
||||||
LOCAL_STATE_FILE="${STATE_DIR}/partnership_${LOCAL_SERVER_NAME}.db"
|
LOCAL_STATE_FILE="${STATE_DIR}/partnership_${LOCAL_SERVER_NAME}.db"
|
||||||
@@ -355,7 +340,7 @@ push_state_to_remote() {
|
|||||||
warn "DRY RUN — would push state file to remote"
|
warn "DRY RUN — would push state file to remote"
|
||||||
return 0
|
return 0
|
||||||
fi
|
fi
|
||||||
timeout "$SSH_TIMEOUT" scp -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" \
|
timeout "$SSH_TIMEOUT" scp -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes \
|
||||||
"$local_file" "root@${remote_ip}:${local_file}" 2>/dev/null && \
|
"$local_file" "root@${remote_ip}:${local_file}" 2>/dev/null && \
|
||||||
echo "State file pushed to remote ✅" || \
|
echo "State file pushed to remote ✅" || \
|
||||||
warn "Could not push state file to remote — will propagate on next sync"
|
warn "Could not push state file to remote — will propagate on next sync"
|
||||||
@@ -364,7 +349,7 @@ push_state_to_remote() {
|
|||||||
read_remote_state() {
|
read_remote_state() {
|
||||||
local remote_ip="$1" ssh_key="$2" remote_file="$3"
|
local remote_ip="$1" ssh_key="$2" remote_file="$3"
|
||||||
timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
|
timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
|
||||||
-o ConnectTimeout="$SSH_TIMEOUT" root@"$remote_ip" \
|
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$remote_ip" \
|
||||||
"cat '$remote_file' 2>/dev/null" 2>/dev/null
|
"cat '$remote_file' 2>/dev/null" 2>/dev/null
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -524,34 +509,9 @@ gather_partner_fallback_containers() {
|
|||||||
done
|
done
|
||||||
}
|
}
|
||||||
|
|
||||||
# Read a scalar var from the mirror's own config via SSH.
|
# read_remote_conf_var() / read_remote_conf_array() — provided by common.sh
|
||||||
# Sources load_config.sh + detect_hosts() on the remote so HOST* aliasing works.
|
|
||||||
read_remote_conf_var() {
|
|
||||||
local mirror_ip="$1" var_name="$2"
|
|
||||||
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
|
||||||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \
|
|
||||||
"source '$SCRIPT_DIR/../load_config.sh' 2>/dev/null
|
|
||||||
detect_hosts 2>/dev/null
|
|
||||||
printf '%s' \"\${${var_name}:-}\"" 2>/dev/null
|
|
||||||
}
|
|
||||||
|
|
||||||
# Read an array var from the mirror's own config via SSH — one element per line.
|
# derive_short_name() — provided by common.sh
|
||||||
read_remote_conf_array() {
|
|
||||||
local mirror_ip="$1" var_name="$2"
|
|
||||||
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
|
||||||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \
|
|
||||||
"source '$SCRIPT_DIR/../load_config.sh' 2>/dev/null
|
|
||||||
detect_hosts 2>/dev/null
|
|
||||||
printf '%s\n' \"\${${var_name}[@]:-}\"" 2>/dev/null
|
|
||||||
}
|
|
||||||
|
|
||||||
# Returns just the short name portion: "unRAID-Gmer4Lfe" → "Gmer4Lfe"
|
|
||||||
derive_short_name() {
|
|
||||||
local hostname="$1"
|
|
||||||
local short="${hostname,,}"
|
|
||||||
[[ "$short" == unraid-* ]] && short="${short:7}"
|
|
||||||
echo "${short^}"
|
|
||||||
}
|
|
||||||
|
|
||||||
# Start this server's own parked containers after partnership ends.
|
# Start this server's own parked containers after partnership ends.
|
||||||
start_own_stack() {
|
start_own_stack() {
|
||||||
@@ -644,7 +604,7 @@ cleanup_owner_containers_on_mirror() {
|
|||||||
|
|
||||||
local container_list
|
local container_list
|
||||||
container_list=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
container_list=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||||
-o ConnectTimeout="$SSH_TIMEOUT" root@"$mirror_ip" \
|
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \
|
||||||
"docker ps -a --format '{{.Names}}' 2>/dev/null | grep -i -- '-${owner_short}$'" 2>/dev/null)
|
"docker ps -a --format '{{.Names}}' 2>/dev/null | grep -i -- '-${owner_short}$'" 2>/dev/null)
|
||||||
|
|
||||||
if [[ -z "$container_list" ]]; then
|
if [[ -z "$container_list" ]]; then
|
||||||
@@ -658,12 +618,12 @@ cleanup_owner_containers_on_mirror() {
|
|||||||
# Collect appdata paths before removal
|
# Collect appdata paths before removal
|
||||||
local appdata_paths
|
local appdata_paths
|
||||||
appdata_paths=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
appdata_paths=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||||
-o ConnectTimeout="$SSH_TIMEOUT" root@"$mirror_ip" \
|
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \
|
||||||
"docker inspect --format '{{range .HostConfig.Binds}}{{println .}}{{end}}' '$container' 2>/dev/null \
|
"docker inspect --format '{{range .HostConfig.Binds}}{{println .}}{{end}}' '$container' 2>/dev/null \
|
||||||
| awk -F: '{print \$1}' | grep '^/mnt/.*/appdata'" 2>/dev/null)
|
| awk -F: '{print \$1}' | grep '^/mnt/.*/appdata'" 2>/dev/null)
|
||||||
|
|
||||||
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||||
-o ConnectTimeout="$SSH_TIMEOUT" root@"$mirror_ip" \
|
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \
|
||||||
"docker stop '$container' >/dev/null 2>&1
|
"docker stop '$container' >/dev/null 2>&1
|
||||||
docker rm '$container' >/dev/null 2>&1 && echo removed" 2>/dev/null | \
|
docker rm '$container' >/dev/null 2>&1 && echo removed" 2>/dev/null | \
|
||||||
grep -q removed && \
|
grep -q removed && \
|
||||||
@@ -674,7 +634,7 @@ cleanup_owner_containers_on_mirror() {
|
|||||||
while IFS= read -r path; do
|
while IFS= read -r path; do
|
||||||
[[ -z "$path" ]] && continue
|
[[ -z "$path" ]] && continue
|
||||||
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||||
-o ConnectTimeout="$SSH_TIMEOUT" root@"$mirror_ip" \
|
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \
|
||||||
"rm -rf '$path' && echo removed" 2>/dev/null | grep -q removed && \
|
"rm -rf '$path' && echo removed" 2>/dev/null | grep -q removed && \
|
||||||
echo " Appdata removed on $MIRROR: $path ✅" || \
|
echo " Appdata removed on $MIRROR: $path ✅" || \
|
||||||
warn " Failed to remove appdata on $MIRROR: $path"
|
warn " Failed to remove appdata on $MIRROR: $path"
|
||||||
@@ -707,7 +667,7 @@ start_mirror_own_stack() {
|
|||||||
continue
|
continue
|
||||||
fi
|
fi
|
||||||
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||||
-o ConnectTimeout="$SSH_TIMEOUT" root@"$mirror_ip" \
|
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \
|
||||||
"docker start '$container' >/dev/null 2>&1 && echo started" 2>/dev/null | \
|
"docker start '$container' >/dev/null 2>&1 && echo started" 2>/dev/null | \
|
||||||
grep -q started && \
|
grep -q started && \
|
||||||
echo "$container started on $MIRROR ✅" || \
|
echo "$container started on $MIRROR ✅" || \
|
||||||
@@ -881,7 +841,7 @@ check_both_healthy() {
|
|||||||
[[ -z "$mirror_ip" ]] && { error "Cannot resolve $MIRROR Tailscale IP"; return 1; }
|
[[ -z "$mirror_ip" ]] && { error "Cannot resolve $MIRROR Tailscale IP"; return 1; }
|
||||||
|
|
||||||
timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" \
|
timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" \
|
||||||
-o ConnectTimeout="$SSH_TIMEOUT" root@"$mirror_ip" \
|
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \
|
||||||
"mountpoint -q '$REMOTE_STORAGE_PATH' && timeout 10 docker ps" >/dev/null 2>&1 || {
|
"mountpoint -q '$REMOTE_STORAGE_PATH' && timeout 10 docker ps" >/dev/null 2>&1 || {
|
||||||
error "Mirror $MIRROR not healthy"
|
error "Mirror $MIRROR not healthy"
|
||||||
return 1
|
return 1
|
||||||
@@ -913,7 +873,8 @@ do_final_sync() {
|
|||||||
warn "Final sync complete — mirror has current state ✅"
|
warn "Final sync complete — mirror has current state ✅"
|
||||||
}
|
}
|
||||||
|
|
||||||
# Safe master.conf modification with error handling
|
# Safe master.conf modification with error handling — appends the key if not already present,
|
||||||
|
# since sed -i returns 0 whether or not it matched anything.
|
||||||
update_master_conf() {
|
update_master_conf() {
|
||||||
local key="$1" value="$2"
|
local key="$1" value="$2"
|
||||||
local conf="$CONF_DIR/master.conf"
|
local conf="$CONF_DIR/master.conf"
|
||||||
@@ -921,12 +882,22 @@ update_master_conf() {
|
|||||||
error "master.conf not found at $conf"
|
error "master.conf not found at $conf"
|
||||||
return 1
|
return 1
|
||||||
fi
|
fi
|
||||||
if sed -i "s|^[[:space:]]*${key}=.*| ${key}=${value}|" "$conf" 2>/dev/null; then
|
if grep -q "^[[:space:]]*${key}=" "$conf" 2>/dev/null; then
|
||||||
echo "master.conf updated: ${key}=${value}"
|
if sed -i "s|^[[:space:]]*${key}=.*| ${key}=${value}|" "$conf" 2>/dev/null; then
|
||||||
return 0
|
echo "master.conf updated: ${key}=${value}"
|
||||||
|
return 0
|
||||||
|
else
|
||||||
|
error "Failed to update master.conf: ${key}=${value}"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
else
|
else
|
||||||
error "Failed to update master.conf: ${key}=${value}"
|
if echo " ${key}=${value}" >> "$conf" 2>/dev/null; then
|
||||||
return 1
|
echo "master.conf updated: ${key}=${value} (appended)"
|
||||||
|
return 0
|
||||||
|
else
|
||||||
|
error "Failed to append to master.conf: ${key}=${value}"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1162,13 +1133,7 @@ if [[ "$MODE" == "onboard" ]]; then
|
|||||||
echo ""
|
echo ""
|
||||||
echo "Enabling partnership in master.conf..."
|
echo "Enabling partnership in master.conf..."
|
||||||
if [[ "$DRY_RUN" == false ]]; then
|
if [[ "$DRY_RUN" == false ]]; then
|
||||||
_master_conf="$CONF_DIR/master.conf"
|
update_master_conf "PARTNERSHIP_ENABLED" "true"
|
||||||
if grep -q "^[[:space:]]*PARTNERSHIP_ENABLED=" "$_master_conf" 2>/dev/null; then
|
|
||||||
sed -i "s|^[[:space:]]*PARTNERSHIP_ENABLED=.*|PARTNERSHIP_ENABLED=true|" "$_master_conf"
|
|
||||||
else
|
|
||||||
echo "PARTNERSHIP_ENABLED=true" >> "$_master_conf"
|
|
||||||
fi
|
|
||||||
echo "PARTNERSHIP_ENABLED=true in master.conf ✅"
|
|
||||||
platform_push_conf | while IFS= read -r line; do log "$line"; done
|
platform_push_conf | while IFS= read -r line; do log "$line"; done
|
||||||
else
|
else
|
||||||
warn "DRY RUN — would set PARTNERSHIP_ENABLED=true in master.conf and push"
|
warn "DRY RUN — would set PARTNERSHIP_ENABLED=true in master.conf and push"
|
||||||
@@ -1435,9 +1400,9 @@ if false; then
|
|||||||
|
|
||||||
# Update remote master.conf
|
# Update remote master.conf
|
||||||
timeout "$SSH_TIMEOUT" ssh -i "$NEW_MIRROR_SSH_KEY" \
|
timeout "$SSH_TIMEOUT" ssh -i "$NEW_MIRROR_SSH_KEY" \
|
||||||
-o ConnectTimeout="$SSH_TIMEOUT" root@"$NEW_MIRROR_IP" \
|
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$NEW_MIRROR_IP" \
|
||||||
"sed -i 's|^[[:space:]]*PARTNERSHIP_OWNER_HOST=.*| PARTNERSHIP_OWNER_HOST=\"$NEW_OWNER_ID\"|' \
|
"sed -i 's|^[[:space:]]*PARTNERSHIP_OWNER_HOST=.*| PARTNERSHIP_OWNER_HOST=\"$NEW_OWNER_ID\"|' \
|
||||||
'$SCRIPT_DIR/../master.conf'" 2>/dev/null && \
|
'$SCRIPT_DIR/../Configurations/master.conf'" 2>/dev/null && \
|
||||||
echo "Remote master.conf updated ✅" || \
|
echo "Remote master.conf updated ✅" || \
|
||||||
error "Failed to update remote master.conf — update manually"
|
error "Failed to update remote master.conf — update manually"
|
||||||
else
|
else
|
||||||
|
|||||||
@@ -108,17 +108,7 @@ fi
|
|||||||
|
|
||||||
detect_hosts
|
detect_hosts
|
||||||
|
|
||||||
OWNER_ID="${PARTNERSHIP_OWNER_HOST:-HOST1}"
|
partnership_resolve_roles
|
||||||
MIRROR_ID=$( [[ "$OWNER_ID" == "HOST1" ]] && echo "HOST2" || echo "HOST1" )
|
|
||||||
OWNER="${!OWNER_ID}"
|
|
||||||
MIRROR="${!MIRROR_ID}"
|
|
||||||
MIRROR_SSH_KEY="$SSH_KEY"
|
|
||||||
OWNER_SSH_KEY="$SSH_KEY"
|
|
||||||
|
|
||||||
AM_OWNER=false
|
|
||||||
AM_MIRROR=false
|
|
||||||
[[ "$MY_ID" == "$OWNER_ID" ]] && AM_OWNER=true
|
|
||||||
[[ "$MY_ID" == "$MIRROR_ID" ]] && AM_MIRROR=true
|
|
||||||
|
|
||||||
LOCAL_STATE_FILE="${STATE_DIR}/partnership_${LOCAL_SERVER_NAME}.db"
|
LOCAL_STATE_FILE="${STATE_DIR}/partnership_${LOCAL_SERVER_NAME}.db"
|
||||||
REMOTE_STATE_FILE="${STATE_DIR}/partnership_${REMOTE_SERVER_NAME}.db"
|
REMOTE_STATE_FILE="${STATE_DIR}/partnership_${REMOTE_SERVER_NAME}.db"
|
||||||
|
|||||||
@@ -223,20 +223,7 @@ fi
|
|||||||
|
|
||||||
detect_hosts
|
detect_hosts
|
||||||
|
|
||||||
OWNER_ID="${PARTNERSHIP_OWNER_HOST:-HOST1}"
|
partnership_resolve_roles
|
||||||
MIRROR_ID=$( [[ "$OWNER_ID" == "HOST1" ]] && echo "HOST2" || echo "HOST1" )
|
|
||||||
OWNER="${!OWNER_ID}"
|
|
||||||
MIRROR="${!MIRROR_ID}"
|
|
||||||
# SSH_KEY (set by detect_hosts) is this server's own private key.
|
|
||||||
# The remote accepts it because this server's PUBLIC key was installed there via ssh_setup.sh.
|
|
||||||
# HOST{N}_SSH_KEY lives in host{N}.conf — with sparse checkout, the other server's
|
|
||||||
# conf is never present here. Always use SSH_KEY (local private key) for outbound SSH.
|
|
||||||
MIRROR_SSH_KEY="$SSH_KEY"
|
|
||||||
|
|
||||||
AM_OWNER=false
|
|
||||||
AM_MIRROR=false
|
|
||||||
[[ "$MY_ID" == "$OWNER_ID" ]] && AM_OWNER=true
|
|
||||||
[[ "$MY_ID" == "$MIRROR_ID" ]] && AM_MIRROR=true
|
|
||||||
|
|
||||||
EXTRA_FLAGS=()
|
EXTRA_FLAGS=()
|
||||||
[[ "$DRY_RUN" == true ]] && EXTRA_FLAGS+=("--dry-run")
|
[[ "$DRY_RUN" == true ]] && EXTRA_FLAGS+=("--dry-run")
|
||||||
@@ -250,11 +237,7 @@ write_onboard_phase() {
|
|||||||
local key="${target_id}_PHASE${phase}_DONE"
|
local key="${target_id}_PHASE${phase}_DONE"
|
||||||
local state_file="$(platform_setup_db_path)"
|
local state_file="$(platform_setup_db_path)"
|
||||||
[[ "$DRY_RUN" == true ]] && { warn "DRY RUN — would write ${key}=true"; return 0; }
|
[[ "$DRY_RUN" == true ]] && { warn "DRY RUN — would write ${key}=true"; return 0; }
|
||||||
if grep -q "^${key}=" "$state_file" 2>/dev/null; then
|
set_state_var "$state_file" "$key" "true"
|
||||||
sed -i "s|^${key}=.*|${key}=true|" "$state_file"
|
|
||||||
else
|
|
||||||
echo "${key}=true" >> "$state_file"
|
|
||||||
fi
|
|
||||||
platform_push_setup_state
|
platform_push_setup_state
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -282,13 +265,7 @@ stop_mirror_stack() {
|
|||||||
local config_var="$1" label="$2"
|
local config_var="$1" label="$2"
|
||||||
local -a to_stop=()
|
local -a to_stop=()
|
||||||
|
|
||||||
mapfile -t to_stop < <(
|
mapfile -t to_stop < <(read_remote_conf_array "$MIRROR_IP" "$config_var" | grep -v '^$')
|
||||||
timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" \
|
|
||||||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$MIRROR_IP" \
|
|
||||||
"source '$SCRIPTS_ROOT/load_config.sh' 2>/dev/null
|
|
||||||
detect_hosts 2>/dev/null
|
|
||||||
printf '%s\n' \"\${${config_var}[@]:-}\"" 2>/dev/null | grep -v '^$'
|
|
||||||
)
|
|
||||||
|
|
||||||
if [[ ${#to_stop[@]} -eq 0 ]]; then
|
if [[ ${#to_stop[@]} -eq 0 ]]; then
|
||||||
log "No $label containers to stop on $MIRROR — skipping"
|
log "No $label containers to stop on $MIRROR — skipping"
|
||||||
@@ -302,7 +279,7 @@ stop_mirror_stack() {
|
|||||||
continue
|
continue
|
||||||
fi
|
fi
|
||||||
timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" \
|
timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" \
|
||||||
-o ConnectTimeout="$SSH_TIMEOUT" root@"$MIRROR_IP" \
|
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$MIRROR_IP" \
|
||||||
"docker stop '$container' 2>/dev/null
|
"docker stop '$container' 2>/dev/null
|
||||||
docker rm '$container' 2>/dev/null && echo removed" 2>/dev/null | \
|
docker rm '$container' 2>/dev/null && echo removed" 2>/dev/null | \
|
||||||
grep -q removed && \
|
grep -q removed && \
|
||||||
@@ -338,11 +315,7 @@ if [[ "$AM_MIRROR" == true ]]; then
|
|||||||
|
|
||||||
if [[ -n "$OWNER_IP" ]]; then
|
if [[ -n "$OWNER_IP" ]]; then
|
||||||
# Read OWNER's SCRIPTS_DIR via platform probe command — don't assume same path as mirror
|
# Read OWNER's SCRIPTS_DIR via platform probe command — don't assume same path as mirror
|
||||||
_probe_cmd=$(platform_scripts_dir_probe_cmd)
|
OWNER_SCRIPTS_DIR=$(resolve_remote_scripts_dir "$OWNER_IP")
|
||||||
OWNER_SCRIPTS_DIR=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
|
||||||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$OWNER_IP" \
|
|
||||||
"$_probe_cmd" 2>/dev/null | tr -d '[:space:]')
|
|
||||||
OWNER_SCRIPTS_DIR="${OWNER_SCRIPTS_DIR:-$SCRIPTS_DIR}"
|
|
||||||
|
|
||||||
if [[ "$DRY_RUN" == true ]]; then
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
warn "DRY RUN — would SSH to $OWNER ($OWNER_IP) and trigger Phase 2"
|
warn "DRY RUN — would SSH to $OWNER ($OWNER_IP) and trigger Phase 2"
|
||||||
@@ -444,11 +417,9 @@ elif [[ "$PHASE1_ONLY" == true ]]; then
|
|||||||
echo " Then click 'Push Conf' in the Partnership tab."
|
echo " Then click 'Push Conf' in the Partnership tab."
|
||||||
# Write key-ready flag so UI can show the manual-install state
|
# Write key-ready flag so UI can show the manual-install state
|
||||||
[[ "$DRY_RUN" == false ]] && {
|
[[ "$DRY_RUN" == false ]] && {
|
||||||
local kflag="${MIRROR_ID}_KEY_READY"
|
kflag="${MIRROR_ID}_KEY_READY"
|
||||||
local _setup_f="$(platform_setup_db_path)"
|
_setup_f="$(platform_setup_db_path)"
|
||||||
grep -q "^${kflag}=" "$_setup_f" 2>/dev/null \
|
set_state_var "$_setup_f" "$kflag" "true"
|
||||||
&& sed -i "s|^${kflag}=.*|${kflag}=true|" "$_setup_f" \
|
|
||||||
|| echo "${kflag}=true" >> "$_setup_f"
|
|
||||||
}
|
}
|
||||||
fi
|
fi
|
||||||
STEP_SSH_OK=false
|
STEP_SSH_OK=false
|
||||||
|
|||||||
@@ -128,17 +128,7 @@ fi
|
|||||||
|
|
||||||
detect_hosts
|
detect_hosts
|
||||||
|
|
||||||
OWNER_ID="${PARTNERSHIP_OWNER_HOST:-HOST1}"
|
partnership_resolve_roles
|
||||||
MIRROR_ID=$( [[ "$OWNER_ID" == "HOST1" ]] && echo "HOST2" || echo "HOST1" )
|
|
||||||
OWNER="${!OWNER_ID}"
|
|
||||||
MIRROR="${!MIRROR_ID}"
|
|
||||||
MIRROR_SSH_KEY="$SSH_KEY"
|
|
||||||
OWNER_SSH_KEY="$SSH_KEY"
|
|
||||||
|
|
||||||
AM_OWNER=false
|
|
||||||
AM_MIRROR=false
|
|
||||||
[[ "$MY_ID" == "$OWNER_ID" ]] && AM_OWNER=true
|
|
||||||
[[ "$MY_ID" == "$MIRROR_ID" ]] && AM_MIRROR=true
|
|
||||||
|
|
||||||
LOCAL_STATE_FILE="${STATE_DIR}/partnership_${LOCAL_SERVER_NAME}.db"
|
LOCAL_STATE_FILE="${STATE_DIR}/partnership_${LOCAL_SERVER_NAME}.db"
|
||||||
REMOTE_STATE_FILE="${STATE_DIR}/partnership_${REMOTE_SERVER_NAME}.db"
|
REMOTE_STATE_FILE="${STATE_DIR}/partnership_${REMOTE_SERVER_NAME}.db"
|
||||||
@@ -286,14 +276,11 @@ if [[ "$DRY_RUN" == false ]]; then
|
|||||||
|
|
||||||
# Push updated master.conf to new owner so both servers agree immediately.
|
# Push updated master.conf to new owner so both servers agree immediately.
|
||||||
# master.conf is shared — host-specific credentials live in host*.conf.
|
# master.conf is shared — host-specific credentials live in host*.conf.
|
||||||
_probe_cmd=$(platform_scripts_dir_probe_cmd)
|
_REMOTE_SD=$(resolve_remote_scripts_dir "$MIRROR_IP" "$SSH_KEY" "no")
|
||||||
_REMOTE_SD=$(ssh -i "$SSH_KEY" -o ConnectTimeout=5 -o StrictHostKeyChecking=no \
|
|
||||||
"root@${MIRROR_IP}" "$_probe_cmd" \
|
|
||||||
2>/dev/null | tr -d '[:space:]')
|
|
||||||
_REMOTE_SD="${_REMOTE_SD:-$SCRIPTS_DIR}"
|
|
||||||
scp -i "$SSH_KEY" \
|
scp -i "$SSH_KEY" \
|
||||||
-o ConnectTimeout="$SSH_TIMEOUT" \
|
-o ConnectTimeout="$SSH_TIMEOUT" \
|
||||||
-o StrictHostKeyChecking=no \
|
-o StrictHostKeyChecking=no \
|
||||||
|
-o BatchMode=yes \
|
||||||
"$SCRIPTS_ROOT/Configurations/master.conf" \
|
"$SCRIPTS_ROOT/Configurations/master.conf" \
|
||||||
"root@${MIRROR_IP}:${_REMOTE_SD}/Configurations/master.conf" 2>/dev/null && \
|
"root@${MIRROR_IP}:${_REMOTE_SD}/Configurations/master.conf" 2>/dev/null && \
|
||||||
echo "master.conf pushed to $NEW_OWNER ✅" || \
|
echo "master.conf pushed to $NEW_OWNER ✅" || \
|
||||||
|
|||||||
@@ -52,7 +52,7 @@ MIRROR_IP=$(resolve_tailscale_ip "$MIRROR")
|
|||||||
[[ -z "$MIRROR_IP" ]] && { error "Cannot resolve $MIRROR Tailscale IP — is Tailscale running?"; exit 1; }
|
[[ -z "$MIRROR_IP" ]] && { error "Cannot resolve $MIRROR Tailscale IP — is Tailscale running?"; exit 1; }
|
||||||
|
|
||||||
_ssh() { ssh -i "$SSH_KEY" -o ConnectTimeout=10 -o BatchMode=yes -o StrictHostKeyChecking=no root@"$MIRROR_IP" "$@"; }
|
_ssh() { ssh -i "$SSH_KEY" -o ConnectTimeout=10 -o BatchMode=yes -o StrictHostKeyChecking=no root@"$MIRROR_IP" "$@"; }
|
||||||
_scp() { scp -i "$SSH_KEY" -o ConnectTimeout=10 -o StrictHostKeyChecking=no "$@"; }
|
_scp() { scp -i "$SSH_KEY" -o ConnectTimeout=10 -o BatchMode=yes -o StrictHostKeyChecking=no "$@"; }
|
||||||
|
|
||||||
# ── Detect remote appdata pool ────────────────────────────────────────────────────────────────
|
# ── Detect remote appdata pool ────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|||||||
@@ -74,9 +74,10 @@ acquire_lock
|
|||||||
detect_hosts
|
detect_hosts
|
||||||
|
|
||||||
# ── Derive key name from hostname ─────────────────────────────────────────────────────────────
|
# ── Derive key name from hostname ─────────────────────────────────────────────────────────────
|
||||||
# Strip unraid- prefix (case-insensitive) if present → gmer4lfe_rsync_automation
|
# derive_short_name() title-cases the result (for display elsewhere) — lowercase it here,
|
||||||
SHORT_NAME="${LOCAL_SERVER_NAME,,}"
|
# same as before, since this feeds a filename → gmer4lfe_rsync_automation
|
||||||
[[ "${SHORT_NAME}" == unraid-* ]] && SHORT_NAME="${SHORT_NAME:7}"
|
SHORT_NAME="$(derive_short_name "$LOCAL_SERVER_NAME")"
|
||||||
|
SHORT_NAME="${SHORT_NAME,,}"
|
||||||
SSH_KEY_NAME="${SHORT_NAME}_rsync_automation"
|
SSH_KEY_NAME="${SHORT_NAME}_rsync_automation"
|
||||||
SSH_KEY_PATH="/root/.ssh/${SSH_KEY_NAME}"
|
SSH_KEY_PATH="/root/.ssh/${SSH_KEY_NAME}"
|
||||||
SSH_PUB_PATH="${SSH_KEY_PATH}.pub"
|
SSH_PUB_PATH="${SSH_KEY_PATH}.pub"
|
||||||
|
|||||||
@@ -220,7 +220,7 @@ deploy_container_from_xml() {
|
|||||||
log "Deploying $name..."
|
log "Deploying $name..."
|
||||||
|
|
||||||
if [[ "$DRY_RUN" == false ]]; then
|
if [[ "$DRY_RUN" == false ]]; then
|
||||||
timeout "$SSH_TIMEOUT" scp -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" \
|
timeout "$SSH_TIMEOUT" scp -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes \
|
||||||
"$xml_file" "root@${remote_ip}:${TEMPLATES_DIR}/${xml_name}" 2>/dev/null || {
|
"$xml_file" "root@${remote_ip}:${TEMPLATES_DIR}/${xml_name}" 2>/dev/null || {
|
||||||
rm -f "$_gpu_tmp"
|
rm -f "$_gpu_tmp"
|
||||||
warn " SCP failed for $xml_name — skipping $name"
|
warn " SCP failed for $xml_name — skipping $name"
|
||||||
@@ -291,9 +291,9 @@ deploy_container_from_xml() {
|
|||||||
|
|
||||||
local remote_script="/tmp/deploy_${name//[^a-zA-Z0-9_]/_}.sh"
|
local remote_script="/tmp/deploy_${name//[^a-zA-Z0-9_]/_}.sh"
|
||||||
|
|
||||||
if timeout "$SSH_TIMEOUT" scp -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" \
|
if timeout "$SSH_TIMEOUT" scp -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes \
|
||||||
"$tmp_script" "root@${remote_ip}:${remote_script}" 2>/dev/null && \
|
"$tmp_script" "root@${remote_ip}:${remote_script}" 2>/dev/null && \
|
||||||
timeout 120 ssh -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" root@"$remote_ip" \
|
timeout 120 ssh -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$remote_ip" \
|
||||||
"bash '$remote_script' 2>&1; rc=\$?; rm -f '$remote_script'; exit \$rc" 2>/dev/null | \
|
"bash '$remote_script' 2>&1; rc=\$?; rm -f '$remote_script'; exit \$rc" 2>/dev/null | \
|
||||||
grep -q "deployed:${name}"; then
|
grep -q "deployed:${name}"; then
|
||||||
echo " $name deployed ✅"
|
echo " $name deployed ✅"
|
||||||
@@ -386,7 +386,7 @@ cleanup_deployed_stack_on_remote() {
|
|||||||
| awk -F: '{print \$1}' | grep '^/mnt/.*/appdata'" 2>/dev/null)
|
| awk -F: '{print \$1}' | grep '^/mnt/.*/appdata'" 2>/dev/null)
|
||||||
|
|
||||||
timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
|
timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
|
||||||
-o ConnectTimeout="$SSH_TIMEOUT" root@"$remote_ip" \
|
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$remote_ip" \
|
||||||
"docker stop '$cname' >/dev/null 2>&1
|
"docker stop '$cname' >/dev/null 2>&1
|
||||||
docker rm '$cname' >/dev/null 2>&1 && echo removed" 2>/dev/null | \
|
docker rm '$cname' >/dev/null 2>&1 && echo removed" 2>/dev/null | \
|
||||||
grep -q removed && \
|
grep -q removed && \
|
||||||
@@ -396,7 +396,7 @@ cleanup_deployed_stack_on_remote() {
|
|||||||
while IFS= read -r path; do
|
while IFS= read -r path; do
|
||||||
[[ -z "$path" ]] && continue
|
[[ -z "$path" ]] && continue
|
||||||
timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
|
timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
|
||||||
-o ConnectTimeout="$SSH_TIMEOUT" root@"$remote_ip" \
|
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$remote_ip" \
|
||||||
"rm -rf '$path' && echo removed" 2>/dev/null | grep -q removed && \
|
"rm -rf '$path' && echo removed" 2>/dev/null | grep -q removed && \
|
||||||
echo " Appdata removed on $MIRROR: $path ✅" || \
|
echo " Appdata removed on $MIRROR: $path ✅" || \
|
||||||
warn " Failed to remove appdata on $MIRROR: $path"
|
warn " Failed to remove appdata on $MIRROR: $path"
|
||||||
@@ -504,7 +504,7 @@ reconfigure_webui() {
|
|||||||
|
|
||||||
local template
|
local template
|
||||||
template=$(timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
|
template=$(timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
|
||||||
-o ConnectTimeout="$SSH_TIMEOUT" root@"$remote_ip" \
|
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$remote_ip" \
|
||||||
"grep -rl '<WebUI>' '$TEMPLATES_DIR/' 2>/dev/null | \
|
"grep -rl '<WebUI>' '$TEMPLATES_DIR/' 2>/dev/null | \
|
||||||
xargs grep -l '\"$container\"' 2>/dev/null | head -1" 2>/dev/null)
|
xargs grep -l '\"$container\"' 2>/dev/null | head -1" 2>/dev/null)
|
||||||
|
|
||||||
@@ -514,7 +514,7 @@ reconfigure_webui() {
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
|
timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
|
||||||
-o ConnectTimeout="$SSH_TIMEOUT" root@"$remote_ip" \
|
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$remote_ip" \
|
||||||
"sed -i 's|<WebUI>.*</WebUI>|<WebUI>http://${target_ip}:${port}/</WebUI>|g' '$template'" \
|
"sed -i 's|<WebUI>.*</WebUI>|<WebUI>http://${target_ip}:${port}/</WebUI>|g' '$template'" \
|
||||||
2>/dev/null && \
|
2>/dev/null && \
|
||||||
echo "$container → http://${target_ip}:${port}/ ✅" || {
|
echo "$container → http://${target_ip}:${port}/ ✅" || {
|
||||||
|
|||||||
+7
-2
@@ -56,7 +56,7 @@
|
|||||||
#
|
#
|
||||||
# Remote Health Pre-flights
|
# Remote Health Pre-flights
|
||||||
# check_connectivity() — Tailscale IP reachable before any SSH
|
# check_connectivity() — Tailscale IP reachable before any SSH
|
||||||
# check_remote_rootfs() — aborts if remote rootfs exceeds ROOTFS_WARN_PCT
|
# check_remote_rootfs() — aborts if remote rootfs exceeds ROOTFS_WARN
|
||||||
# check_remote_share() — aborts if target directory missing or empty on remote
|
# check_remote_share() — aborts if target directory missing or empty on remote
|
||||||
# check_remote_disks() — verifies all backing disks online on remote
|
# check_remote_disks() — verifies all backing disks online on remote
|
||||||
#
|
#
|
||||||
@@ -98,7 +98,7 @@
|
|||||||
# SLEEP
|
# SLEEP
|
||||||
# Default seconds between retry attempts. (default: 60)
|
# Default seconds between retry attempts. (default: 60)
|
||||||
#
|
#
|
||||||
# ROOTFS_WARN_PCT
|
# ROOTFS_WARN
|
||||||
# Abort threshold for remote rootfs percentage full. (default: 75)
|
# Abort threshold for remote rootfs percentage full. (default: 75)
|
||||||
#
|
#
|
||||||
# PROFILES["profile_KEY"]
|
# PROFILES["profile_KEY"]
|
||||||
@@ -223,6 +223,11 @@ VV_LAST_LOG="/tmp/unraid_locks/rsync_${PROFILE_NAME}.last.log"
|
|||||||
exec 1> >(tee -a "$VV_LIVE_LOG") 2>&1
|
exec 1> >(tee -a "$VV_LIVE_LOG") 2>&1
|
||||||
|
|
||||||
# ── Load profile settings ─────────────────────────────────────────────────────────────────────
|
# ── Load profile settings ─────────────────────────────────────────────────────────────────────
|
||||||
|
# NOTE: DEFAULT_RSYNC_OPTS and PROFILE_RSYNC_OPTS are bash arrays defined in master.conf —
|
||||||
|
# any "$BW_LIMIT" inside them is expanded once, when master.conf is sourced, before this
|
||||||
|
# override runs. So this recomputed $BW_LIMIT only affects the log line below and --status;
|
||||||
|
# it does NOT change the actual rsync --bwlimit unless a profile also bakes its own
|
||||||
|
# --bwlimit="${PROFILE_BW_LIMIT[name]}" directly into that profile's PROFILE_RSYNC_OPTS entry.
|
||||||
BW_LIMIT=${PROFILE_BW_LIMIT[$PROFILE_NAME]:-$BW_LIMIT}
|
BW_LIMIT=${PROFILE_BW_LIMIT[$PROFILE_NAME]:-$BW_LIMIT}
|
||||||
RETRY_COUNT=${PROFILE_RETRY_COUNT[$PROFILE_NAME]:-$RETRY_COUNT}
|
RETRY_COUNT=${PROFILE_RETRY_COUNT[$PROFILE_NAME]:-$RETRY_COUNT}
|
||||||
SLEEP=${PROFILE_SLEEP[$PROFILE_NAME]:-$SLEEP}
|
SLEEP=${PROFILE_SLEEP[$PROFILE_NAME]:-$SLEEP}
|
||||||
|
|||||||
@@ -53,7 +53,7 @@ parse_args "$@"
|
|||||||
detect_hosts
|
detect_hosts
|
||||||
require_partnership
|
require_partnership
|
||||||
|
|
||||||
RAM_CACHE="/tmp/.cache/vv/d"
|
RAM_CACHE="$CONF_RAM_CACHE_DIR"
|
||||||
SAVE_DIR="$PERSISTENT_CONF_CACHE"
|
SAVE_DIR="$PERSISTENT_CONF_CACHE"
|
||||||
|
|
||||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||||
@@ -67,7 +67,7 @@ restored=0
|
|||||||
for conf in "$SAVE_DIR"/host*.conf; do
|
for conf in "$SAVE_DIR"/host*.conf; do
|
||||||
[[ -f "$conf" ]] || continue
|
[[ -f "$conf" ]] || continue
|
||||||
base="$(basename "$conf")"
|
base="$(basename "$conf")"
|
||||||
[[ "${base,,}" == "${MY_ID,,}.conf" ]] && continue
|
is_own_conf_file "$base" && continue
|
||||||
|
|
||||||
if [[ -f "$RAM_CACHE/$base" ]]; then
|
if [[ -f "$RAM_CACHE/$base" ]]; then
|
||||||
log "$base already in RAM cache (conf_sync succeeded) — skipping"
|
log "$base already in RAM cache (conf_sync succeeded) — skipping"
|
||||||
|
|||||||
@@ -47,7 +47,7 @@ parse_args "$@"
|
|||||||
detect_hosts
|
detect_hosts
|
||||||
require_partnership
|
require_partnership
|
||||||
|
|
||||||
RAM_CACHE="/tmp/.cache/vv/d"
|
RAM_CACHE="$CONF_RAM_CACHE_DIR"
|
||||||
SAVE_DIR="$PERSISTENT_CONF_CACHE"
|
SAVE_DIR="$PERSISTENT_CONF_CACHE"
|
||||||
|
|
||||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be written"
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be written"
|
||||||
@@ -61,7 +61,7 @@ saved=0
|
|||||||
for conf in "$RAM_CACHE"/host*.conf; do
|
for conf in "$RAM_CACHE"/host*.conf; do
|
||||||
[[ -f "$conf" ]] || continue
|
[[ -f "$conf" ]] || continue
|
||||||
base="$(basename "$conf")"
|
base="$(basename "$conf")"
|
||||||
[[ "${base,,}" == "${MY_ID,,}.conf" ]] && continue
|
is_own_conf_file "$base" && continue
|
||||||
|
|
||||||
if [[ "$DRY_RUN" == true ]]; then
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
warn "DRY RUN — would save $base → $SAVE_DIR/"
|
warn "DRY RUN — would save $base → $SAVE_DIR/"
|
||||||
|
|||||||
@@ -81,7 +81,7 @@ if [[ "${CONF_SYNC_ENABLED:-true}" == false ]]; then
|
|||||||
exit 0
|
exit 0
|
||||||
fi
|
fi
|
||||||
|
|
||||||
CACHE_DIR="/tmp/.cache/vv/d"
|
CACHE_DIR="$CONF_RAM_CACHE_DIR"
|
||||||
MY_CONF="$SCRIPTS_DIR/Configurations/${MY_ID,,}.conf"
|
MY_CONF="$SCRIPTS_DIR/Configurations/${MY_ID,,}.conf"
|
||||||
SSH_TIMEOUT=10
|
SSH_TIMEOUT=10
|
||||||
|
|
||||||
|
|||||||
@@ -146,17 +146,7 @@ cleanup_on_exit() {
|
|||||||
warn "Removing partial archive: $ARCHIVE_PATH"
|
warn "Removing partial archive: $ARCHIVE_PATH"
|
||||||
rm -f "$ARCHIVE_PATH" 2>/dev/null
|
rm -f "$ARCHIVE_PATH" 2>/dev/null
|
||||||
fi
|
fi
|
||||||
# Always restart container if it was running
|
container_force_restart_if_needed "$CONTAINER_NAME" "$CONTAINER_WAS_RUNNING"
|
||||||
if [[ "$CONTAINER_WAS_RUNNING" == true && "$DRY_RUN" == false ]]; then
|
|
||||||
local status
|
|
||||||
status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \
|
|
||||||
"$CONTAINER_NAME" 2>/dev/null)
|
|
||||||
if [[ "$status" != "true" ]]; then
|
|
||||||
warn "Restarting $CONTAINER_NAME (cleanup)..."
|
|
||||||
timeout "$DOCKER_TIMEOUT" docker start "$CONTAINER_NAME" >/dev/null 2>&1 || \
|
|
||||||
error "Failed to restart $CONTAINER_NAME — start it manually"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
}
|
}
|
||||||
trap cleanup_on_exit EXIT
|
trap cleanup_on_exit EXIT
|
||||||
|
|
||||||
@@ -166,35 +156,10 @@ trap cleanup_on_exit EXIT
|
|||||||
echo ""
|
echo ""
|
||||||
echo "━━━ $ICON_STOP Stop Container ━━━"
|
echo "━━━ $ICON_STOP Stop Container ━━━"
|
||||||
|
|
||||||
STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \
|
container_stop_for_maintenance "$CONTAINER_NAME" CONTAINER_WAS_RUNNING \
|
||||||
"$CONTAINER_NAME" 2>/dev/null)
|
"Stopping $CONTAINER_NAME for clean export..." log || exit 1
|
||||||
|
[[ "$CONTAINER_WAS_RUNNING" == false ]] && \
|
||||||
case "$STATUS" in
|
echo "$CONTAINER_NAME is not running — archiving as-is (was stopped state respected)"
|
||||||
true)
|
|
||||||
CONTAINER_WAS_RUNNING=true
|
|
||||||
log "Stopping $CONTAINER_NAME for clean export..."
|
|
||||||
if [[ "$DRY_RUN" == false ]]; then
|
|
||||||
if timeout "$DOCKER_TIMEOUT" docker stop "$CONTAINER_NAME" >/dev/null 2>&1; then
|
|
||||||
echo "$CONTAINER_NAME stopped ✅"
|
|
||||||
else
|
|
||||||
error "Failed to stop $CONTAINER_NAME — aborting export"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
warn "DRY RUN — would stop $CONTAINER_NAME"
|
|
||||||
fi
|
|
||||||
;;
|
|
||||||
false)
|
|
||||||
echo "$CONTAINER_NAME is not running — archiving as-is (was stopped state respected)"
|
|
||||||
;;
|
|
||||||
"")
|
|
||||||
error "$CONTAINER_NAME not found — check container name"
|
|
||||||
exit 1
|
|
||||||
;;
|
|
||||||
*)
|
|
||||||
warn "$CONTAINER_NAME status: $STATUS — proceeding with caution"
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Archive ━━━
|
# ━━━ Archive ━━━
|
||||||
@@ -249,37 +214,7 @@ END=$(date +%s)
|
|||||||
echo ""
|
echo ""
|
||||||
echo "━━━ $ICON_START Restart Container ━━━"
|
echo "━━━ $ICON_START Restart Container ━━━"
|
||||||
|
|
||||||
RESTART_OK=false
|
container_restart_after_maintenance "$CONTAINER_NAME" "$CONTAINER_WAS_RUNNING" 3 "Container Export"
|
||||||
|
|
||||||
if [[ "$CONTAINER_WAS_RUNNING" == true ]]; then
|
|
||||||
log "Restarting $CONTAINER_NAME..."
|
|
||||||
if [[ "$DRY_RUN" == false ]]; then
|
|
||||||
if timeout "$DOCKER_TIMEOUT" docker start "$CONTAINER_NAME" >/dev/null 2>&1; then
|
|
||||||
# Brief settle then verify
|
|
||||||
sleep 3
|
|
||||||
POST_STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
|
|
||||||
'{{.State.Running}}' "$CONTAINER_NAME" 2>/dev/null)
|
|
||||||
if [[ "$POST_STATUS" == "true" ]]; then
|
|
||||||
echo "$CONTAINER_NAME restarted and running ✅"
|
|
||||||
RESTART_OK=true
|
|
||||||
else
|
|
||||||
error "$CONTAINER_NAME started but crashed immediately — check container logs"
|
|
||||||
notify "$CONTAINER_NAME failed to stay running after export on $(hostname)" \
|
|
||||||
"Container Export" "warning"
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
error "Failed to restart $CONTAINER_NAME — start it manually"
|
|
||||||
notify "$CONTAINER_NAME failed to restart after export on $(hostname)" \
|
|
||||||
"Container Export" "warning"
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
warn "DRY RUN — would restart $CONTAINER_NAME"
|
|
||||||
RESTART_OK=true
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
echo "$CONTAINER_NAME was not running — leaving stopped (state respected) ✅"
|
|
||||||
RESTART_OK=true
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Clear trap — clean exit, cleanup_on_exit no longer needed
|
# Clear trap — clean exit, cleanup_on_exit no longer needed
|
||||||
trap - EXIT
|
trap - EXIT
|
||||||
|
|||||||
@@ -171,16 +171,7 @@ EMBY_WAS_RUNNING=false
|
|||||||
|
|
||||||
cleanup_on_exit() {
|
cleanup_on_exit() {
|
||||||
local exit_code=$?
|
local exit_code=$?
|
||||||
if [[ "$EMBY_WAS_RUNNING" == true && "$DRY_RUN" == false ]]; then
|
container_force_restart_if_needed "$EMBY_CONTAINER" "$EMBY_WAS_RUNNING"
|
||||||
local status
|
|
||||||
status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \
|
|
||||||
"$EMBY_CONTAINER" 2>/dev/null)
|
|
||||||
if [[ "$status" != "true" ]]; then
|
|
||||||
warn "Restarting $EMBY_CONTAINER (cleanup)..."
|
|
||||||
timeout "$DOCKER_TIMEOUT" docker start "$EMBY_CONTAINER" >/dev/null 2>&1 || \
|
|
||||||
error "Failed to restart $EMBY_CONTAINER — start it manually"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
}
|
}
|
||||||
|
|
||||||
trap cleanup_on_exit EXIT
|
trap cleanup_on_exit EXIT
|
||||||
@@ -191,36 +182,10 @@ trap cleanup_on_exit EXIT
|
|||||||
echo ""
|
echo ""
|
||||||
echo "━━━ $ICON_STOP Stop Emby ━━━"
|
echo "━━━ $ICON_STOP Stop Emby ━━━"
|
||||||
|
|
||||||
STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \
|
container_stop_for_maintenance "$EMBY_CONTAINER" EMBY_WAS_RUNNING \
|
||||||
"$EMBY_CONTAINER" 2>/dev/null)
|
"Stopping $EMBY_CONTAINER — active sessions will be interrupted" warn 3 || exit 1
|
||||||
|
[[ "$EMBY_WAS_RUNNING" == false ]] && \
|
||||||
case "$STATUS" in
|
log "$EMBY_CONTAINER is not running — proceeding with checks"
|
||||||
true)
|
|
||||||
EMBY_WAS_RUNNING=true
|
|
||||||
warn "Stopping $EMBY_CONTAINER — active sessions will be interrupted"
|
|
||||||
if [[ "$DRY_RUN" == false ]]; then
|
|
||||||
if timeout "$DOCKER_TIMEOUT" docker stop "$EMBY_CONTAINER" >/dev/null 2>&1; then
|
|
||||||
echo "$EMBY_CONTAINER stopped ✅"
|
|
||||||
sleep 3 # let file handles release
|
|
||||||
else
|
|
||||||
error "Failed to stop $EMBY_CONTAINER — aborting"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
warn "DRY RUN — would stop $EMBY_CONTAINER"
|
|
||||||
fi
|
|
||||||
;;
|
|
||||||
false)
|
|
||||||
log "$EMBY_CONTAINER is not running — proceeding with checks"
|
|
||||||
;;
|
|
||||||
"")
|
|
||||||
error "$EMBY_CONTAINER not found — check container name"
|
|
||||||
exit 1
|
|
||||||
;;
|
|
||||||
*)
|
|
||||||
warn "$EMBY_CONTAINER status: $STATUS — proceeding with caution"
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Database Integrity Check ━━━
|
# ━━━ Database Integrity Check ━━━
|
||||||
@@ -301,37 +266,8 @@ END=$(date +%s)
|
|||||||
echo ""
|
echo ""
|
||||||
echo "━━━ $ICON_START Restart Emby ━━━"
|
echo "━━━ $ICON_START Restart Emby ━━━"
|
||||||
|
|
||||||
RESTART_OK=false
|
container_restart_after_maintenance "$EMBY_CONTAINER" "$EMBY_WAS_RUNNING" 5 "Emby DB Repair"
|
||||||
|
[[ "$RESTART_OK" == false ]] && error "Check Docker logs: docker logs $EMBY_CONTAINER"
|
||||||
if [[ "$EMBY_WAS_RUNNING" == true ]]; then
|
|
||||||
if [[ "$DRY_RUN" == false ]]; then
|
|
||||||
log "Restarting $EMBY_CONTAINER..."
|
|
||||||
if timeout "$DOCKER_TIMEOUT" docker start "$EMBY_CONTAINER" >/dev/null 2>&1; then
|
|
||||||
sleep 5 # Emby takes longer to initialise than most containers
|
|
||||||
POST_STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
|
|
||||||
'{{.State.Running}}' "$EMBY_CONTAINER" 2>/dev/null)
|
|
||||||
if [[ "$POST_STATUS" == "true" ]]; then
|
|
||||||
echo "$EMBY_CONTAINER restarted and running ✅"
|
|
||||||
RESTART_OK=true
|
|
||||||
else
|
|
||||||
error "$EMBY_CONTAINER started but crashed — database may be corrupt"
|
|
||||||
error "Check Docker logs: docker logs $EMBY_CONTAINER"
|
|
||||||
notify "$EMBY_CONTAINER crashed on restart — possible database corruption on $(hostname)" \
|
|
||||||
"Emby DB Repair" "warning"
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
error "Failed to restart $EMBY_CONTAINER — start it manually"
|
|
||||||
notify "$EMBY_CONTAINER failed to restart after integrity check on $(hostname)" \
|
|
||||||
"Emby DB Repair" "warning"
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
warn "DRY RUN — would restart $EMBY_CONTAINER"
|
|
||||||
RESTART_OK=true
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
echo "$EMBY_CONTAINER was not running — leaving stopped (state respected) ✅"
|
|
||||||
RESTART_OK=true
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Clear EXIT trap — clean exit
|
# Clear EXIT trap — clean exit
|
||||||
trap - EXIT
|
trap - EXIT
|
||||||
|
|||||||
@@ -138,11 +138,8 @@ _build_drive_list() {
|
|||||||
[[ ! -e "$drive" ]] && continue
|
[[ ! -e "$drive" ]] && continue
|
||||||
local drive_name
|
local drive_name
|
||||||
drive_name=$(basename "$drive")
|
drive_name=$(basename "$drive")
|
||||||
local ignored=false
|
is_in_list "$drive_name" "${SMART_IGNORE_DRIVES[@]:-}" && \
|
||||||
for ignore in "${SMART_IGNORE_DRIVES[@]:-}"; do
|
{ log "Skipping $drive_name (SMART_IGNORE_DRIVES)"; continue; }
|
||||||
[[ "$drive_name" == "$ignore" ]] && ignored=true && break
|
|
||||||
done
|
|
||||||
[[ "$ignored" == true ]] && { log "Skipping $drive_name (SMART_IGNORE_DRIVES)"; continue; }
|
|
||||||
if ! smartctl -i "$drive" 2>/dev/null | grep -q "SMART support is: Enabled"; then
|
if ! smartctl -i "$drive" 2>/dev/null | grep -q "SMART support is: Enabled"; then
|
||||||
log "Skipping $drive_name — SMART not enabled"
|
log "Skipping $drive_name — SMART not enabled"
|
||||||
continue
|
continue
|
||||||
@@ -166,11 +163,7 @@ if [[ "$SHOW_STATUS" == true ]]; then
|
|||||||
drive_name=$(basename "$drive")
|
drive_name=$(basename "$drive")
|
||||||
local_result=$(smartctl -l selftest "$drive" 2>/dev/null | \
|
local_result=$(smartctl -l selftest "$drive" 2>/dev/null | \
|
||||||
grep -m1 "Extended" | awk '{print $NF}')
|
grep -m1 "Extended" | awk '{print $NF}')
|
||||||
ignored=false
|
if is_in_list "$drive_name" "${SMART_IGNORE_DRIVES[@]:-}"; then
|
||||||
for ignore in "${SMART_IGNORE_DRIVES[@]:-}"; do
|
|
||||||
[[ "$drive_name" == "$ignore" ]] && ignored=true && break
|
|
||||||
done
|
|
||||||
if [[ "$ignored" == true ]]; then
|
|
||||||
echo " $ICON_WARN $drive_name — ignored"
|
echo " $ICON_WARN $drive_name — ignored"
|
||||||
else
|
else
|
||||||
echo " $ICON_SMART $drive_name — last extended: ${local_result:-no result}"
|
echo " $ICON_SMART $drive_name — last extended: ${local_result:-no result}"
|
||||||
|
|||||||
@@ -240,15 +240,7 @@ cleanup_location() {
|
|||||||
|
|
||||||
# Format bytes freed
|
# Format bytes freed
|
||||||
local freed_human
|
local freed_human
|
||||||
if (( bytes_freed > 1073741824 )); then
|
freed_human=$(format_bytes "$bytes_freed")
|
||||||
freed_human=$(awk "BEGIN {printf \"%.1fGB\", $bytes_freed / 1073741824}")
|
|
||||||
elif (( bytes_freed > 1048576 )); then
|
|
||||||
freed_human=$(awk "BEGIN {printf \"%.1fMB\", $bytes_freed / 1048576}")
|
|
||||||
elif (( bytes_freed > 0 )); then
|
|
||||||
freed_human="${bytes_freed}B"
|
|
||||||
else
|
|
||||||
freed_human="0B"
|
|
||||||
fi
|
|
||||||
|
|
||||||
log "$label — removed $files_removed ($freed_human) | active(fresh): $files_too_young | streaming(lsof): $files_streaming | protected(old+open): $files_active | skipped: $files_skipped | failed: $files_failed"
|
log "$label — removed $files_removed ($freed_human) | active(fresh): $files_too_young | streaming(lsof): $files_streaming | protected(old+open): $files_active | skipped: $files_skipped | failed: $files_failed"
|
||||||
|
|
||||||
@@ -299,7 +291,7 @@ fi
|
|||||||
# Post-cleanup — check if ramdisk recovered enough to flip symlink back to ramdisk
|
# Post-cleanup — check if ramdisk recovered enough to flip symlink back to ramdisk
|
||||||
if [[ "$DRY_RUN" == false ]] && mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
|
if [[ "$DRY_RUN" == false ]] && mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
|
||||||
RAMDISK_USED_KB=$(df "$RAMDISK_PATH" --output=used 2>/dev/null | tail -1 | tr -d ' ')
|
RAMDISK_USED_KB=$(df "$RAMDISK_PATH" --output=used 2>/dev/null | tail -1 | tr -d ' ')
|
||||||
RAMDISK_USED_GB=$(awk "BEGIN {printf \"%.2f\", ${RAMDISK_USED_KB:-0} / 1048576}")
|
RAMDISK_USED_GB=$(kb_to_gb "$RAMDISK_USED_KB")
|
||||||
LOW_RECOVERED=$(awk "BEGIN {print ($RAMDISK_USED_GB < $RAMDISK_LOW_GB) ? 1 : 0}")
|
LOW_RECOVERED=$(awk "BEGIN {print ($RAMDISK_USED_GB < $RAMDISK_LOW_GB) ? 1 : 0}")
|
||||||
CURRENT_TARGET=$(grep "^current_target=" "$STATE_FILE" 2>/dev/null | cut -d= -f2)
|
CURRENT_TARGET=$(grep "^current_target=" "$STATE_FILE" 2>/dev/null | cut -d= -f2)
|
||||||
|
|
||||||
|
|||||||
@@ -234,18 +234,15 @@ fix_permissions() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
get_ramdisk_used_gb() {
|
get_ramdisk_used_gb() {
|
||||||
df "$RAMDISK_PATH" --output=used 2>/dev/null | \
|
kb_to_gb "$(df "$RAMDISK_PATH" --output=used 2>/dev/null | tail -1 | tr -d ' ')"
|
||||||
tail -1 | tr -d ' ' | awk '{printf "%.2f", $1/1048576}'
|
|
||||||
}
|
}
|
||||||
|
|
||||||
get_ramdisk_avail_gb() {
|
get_ramdisk_avail_gb() {
|
||||||
df "$RAMDISK_PATH" --output=avail 2>/dev/null | \
|
kb_to_gb "$(df "$RAMDISK_PATH" --output=avail 2>/dev/null | tail -1 | tr -d ' ')"
|
||||||
tail -1 | tr -d ' ' | awk '{printf "%.2f", $1/1048576}'
|
|
||||||
}
|
}
|
||||||
|
|
||||||
get_ssd_free_gb() {
|
get_ssd_free_gb() {
|
||||||
df "$TRANSCODE_SSD" --output=avail 2>/dev/null | \
|
kb_to_gb "$(df "$TRANSCODE_SSD" --output=avail 2>/dev/null | tail -1 | tr -d ' ')"
|
||||||
tail -1 | tr -d ' ' | awk '{printf "%.2f", $1/1048576}'
|
|
||||||
}
|
}
|
||||||
|
|
||||||
get_flip_count() {
|
get_flip_count() {
|
||||||
|
|||||||
@@ -125,16 +125,9 @@ log "$ICON_NET DDNS: ${DDNS_DOMAIN:-not configured} → ${DDNS_CONTAINER:-no c
|
|||||||
|
|
||||||
touch "${NETWORK_WATCHDOG_NPM_STATE_FILE}" 2>/dev/null
|
touch "${NETWORK_WATCHDOG_NPM_STATE_FILE}" 2>/dev/null
|
||||||
|
|
||||||
# ━━━ Strike helpers ━━━
|
# ━━━ Strike helpers — wrap common.sh's wd_state_get()/wd_state_set() ━━━
|
||||||
get_strikes() { grep "^${1}:" "${2}" 2>/dev/null | cut -d: -f2 || echo "0"; }
|
get_strikes() { wd_state_get "$1" "$2"; }
|
||||||
set_strikes() {
|
set_strikes() { wd_state_set "$1" "$2" "$3"; }
|
||||||
local key="$1" count="$2" file="$3"
|
|
||||||
if grep -q "^${key}:" "$file" 2>/dev/null; then
|
|
||||||
sed -i "s|^${key}:.*|${key}:${count}|" "$file"
|
|
||||||
else
|
|
||||||
echo "${key}:${count}" >> "$file"
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Status ━━━
|
# ━━━ Status ━━━
|
||||||
|
|||||||
@@ -164,16 +164,9 @@ log "$ICON_DISK Paths: ${WATCHDOG_APPDATA_PATHS[*]:-none configured}"
|
|||||||
touch "$STORAGE_WATCHDOG_STATE_FILE" 2>/dev/null
|
touch "$STORAGE_WATCHDOG_STATE_FILE" 2>/dev/null
|
||||||
touch "$WATCHDOG_APPDATA_GROWTH_FILE" 2>/dev/null
|
touch "$WATCHDOG_APPDATA_GROWTH_FILE" 2>/dev/null
|
||||||
|
|
||||||
# ━━━ Strike helpers ━━━
|
# ━━━ Strike helpers — wrap common.sh's wd_state_get()/wd_state_set() ━━━
|
||||||
get_strikes() { grep "^${1}:" "${2}" 2>/dev/null | cut -d: -f2 || echo "0"; }
|
get_strikes() { wd_state_get "$1" "$2"; }
|
||||||
set_strikes() {
|
set_strikes() { wd_state_set "$1" "$2" "$3"; }
|
||||||
local key="$1" count="$2" file="$3"
|
|
||||||
if grep -q "^${key}:" "$file" 2>/dev/null; then
|
|
||||||
sed -i "s|^${key}:.*|${key}:${count}|" "$file"
|
|
||||||
else
|
|
||||||
echo "${key}:${count}" >> "$file"
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Status ━━━
|
# ━━━ Status ━━━
|
||||||
|
|||||||
@@ -194,6 +194,9 @@
|
|||||||
# WATCHDOG_CONTAINER_RESTART_LIMIT / WATCHDOG_CONTAINER_RESTART_WINDOW
|
# WATCHDOG_CONTAINER_RESTART_LIMIT / WATCHDOG_CONTAINER_RESTART_WINDOW
|
||||||
# Restart loop protection: attempt limit and rolling window in hours
|
# Restart loop protection: attempt limit and rolling window in hours
|
||||||
#
|
#
|
||||||
|
# WATCHDOG_REQUIRED_STRIKE_LIMIT
|
||||||
|
# Consecutive down-checks on a required container before a restart is attempted (default: 2)
|
||||||
|
#
|
||||||
# WATCHDOG_BATCH_NOTIFY
|
# WATCHDOG_BATCH_NOTIFY
|
||||||
# Collect cycle events and send as one notification (default: true)
|
# Collect cycle events and send as one notification (default: true)
|
||||||
#
|
#
|
||||||
@@ -337,19 +340,15 @@ fi
|
|||||||
# ── HELPER FUNCTIONS ──────────────────────────────────────────────────────────────────────────
|
# ── HELPER FUNCTIONS ──────────────────────────────────────────────────────────────────────────
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
|
|
||||||
# Get strike count for a container from state file
|
# Get strike count for a container from state file — wraps common.sh's wd_state_get()
|
||||||
get_strikes() {
|
get_strikes() {
|
||||||
grep "^${1}:" "${2}" 2>/dev/null | cut -d: -f2 || echo "0"
|
wd_state_get "$1" "$2"
|
||||||
}
|
}
|
||||||
|
|
||||||
# Set strike count for a container in state file
|
# Set strike count for a container in state file — wraps common.sh's wd_state_set()
|
||||||
set_strikes() {
|
set_strikes() {
|
||||||
local container="$1" count="$2" file="$3"
|
local container="$1" count="$2" file="$3"
|
||||||
if grep -q "^${container}:" "$file" 2>/dev/null; then
|
wd_state_set "$container" "$count" "$file"
|
||||||
sed -i "s/^${container}:.*/${container}:${count}/" "$file"
|
|
||||||
else
|
|
||||||
echo "${container}:${count}" >> "$file"
|
|
||||||
fi
|
|
||||||
}
|
}
|
||||||
|
|
||||||
# Check if container is on the persistent skip list
|
# Check if container is on the persistent skip list
|
||||||
@@ -763,10 +762,10 @@ CYCLE_START=$(date +%s)
|
|||||||
STRIKES=$(get_strikes "$container" "$WATCHDOG_STATE_FILE")
|
STRIKES=$(get_strikes "$container" "$WATCHDOG_STATE_FILE")
|
||||||
STRIKES=$(( STRIKES + 1 ))
|
STRIKES=$(( STRIKES + 1 ))
|
||||||
set_strikes "$container" "$STRIKES" "$WATCHDOG_STATE_FILE"
|
set_strikes "$container" "$STRIKES" "$WATCHDOG_STATE_FILE"
|
||||||
warn "$container — not running, exit ${LAST_EXIT} (strike $STRIKES/$SYS_WATCHDOG_STRIKE_LIMIT)"
|
warn "$container — not running, exit ${LAST_EXIT} (strike $STRIKES/${WATCHDOG_REQUIRED_STRIKE_LIMIT:-2})"
|
||||||
((T1_WARNINGS++))
|
((T1_WARNINGS++))
|
||||||
|
|
||||||
if [[ "$STRIKES" -ge "$SYS_WATCHDOG_STRIKE_LIMIT" ]]; then
|
if [[ "$STRIKES" -ge "${WATCHDOG_REQUIRED_STRIKE_LIMIT:-2}" ]]; then
|
||||||
result=0
|
result=0
|
||||||
safe_restart "$container" "required container down (exit ${LAST_EXIT})" || result=$?
|
safe_restart "$container" "required container down (exit ${LAST_EXIT})" || result=$?
|
||||||
case $result in
|
case $result in
|
||||||
|
|||||||
@@ -165,27 +165,25 @@ trap "_release_all_locks; _rw_trap_restart_stopped" EXIT
|
|||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# rm_state_get/set use : separator for RM-internal state
|
# rm_state_get/set use : separator for RM-internal state
|
||||||
# rm_state_get_eq/set_eq use = separator for docker_watchdog coordination flags
|
# rm_state_get_eq/set_eq use = separator for docker_watchdog coordination flags
|
||||||
|
# Both wrap common.sh's wd_state_get()/wd_state_set() — file is implicit here (this
|
||||||
|
# script's own state file), unlike the generic helper which always takes it as an argument.
|
||||||
|
|
||||||
rm_state_get() {
|
rm_state_get() {
|
||||||
grep -E "^${1}:" "$RW_STATE_FILE" 2>/dev/null | cut -d: -f2-
|
wd_state_get "$1" "$RW_STATE_FILE"
|
||||||
}
|
}
|
||||||
|
|
||||||
rm_state_set() {
|
rm_state_set() {
|
||||||
local key="$1" val="$2"
|
local key="$1" val="$2"
|
||||||
grep -vE "^${key}:" "$RW_STATE_FILE" 2>/dev/null > "${RW_STATE_FILE}.tmp"
|
wd_state_set "$key" "$val" "$RW_STATE_FILE"
|
||||||
echo "${key}:${val}" >> "${RW_STATE_FILE}.tmp"
|
|
||||||
mv "${RW_STATE_FILE}.tmp" "$RW_STATE_FILE"
|
|
||||||
}
|
}
|
||||||
|
|
||||||
rm_state_get_eq() {
|
rm_state_get_eq() {
|
||||||
grep -E "^${1}=" "$RW_STATE_FILE" 2>/dev/null | cut -d= -f2-
|
wd_state_get "$1" "$RW_STATE_FILE" "="
|
||||||
}
|
}
|
||||||
|
|
||||||
rm_state_set_eq() {
|
rm_state_set_eq() {
|
||||||
local key="$1" val="$2"
|
local key="$1" val="$2"
|
||||||
grep -vE "^${key}=" "$RW_STATE_FILE" 2>/dev/null > "${RW_STATE_FILE}.tmp"
|
wd_state_set "$key" "$val" "$RW_STATE_FILE" "="
|
||||||
echo "${key}=${val}" >> "${RW_STATE_FILE}.tmp"
|
|
||||||
mv "${RW_STATE_FILE}.tmp" "$RW_STATE_FILE"
|
|
||||||
}
|
}
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
@@ -376,7 +374,8 @@ unpause_containers() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
# Stop a list of containers — returns comma-separated list of actually-stopped containers
|
# Stop a list of containers — returns comma-separated list of actually-stopped containers
|
||||||
stop_containers() {
|
# Named rw_ to avoid colliding with common.sh's stop_containers() (different signature/semantics)
|
||||||
|
rw_stop_containers() {
|
||||||
local actually_stopped=()
|
local actually_stopped=()
|
||||||
for container in "$@"; do
|
for container in "$@"; do
|
||||||
[[ -z "$container" ]] && continue
|
[[ -z "$container" ]] && continue
|
||||||
@@ -407,7 +406,8 @@ stop_containers() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
# Start a comma-separated list of containers (only those RM stopped)
|
# Start a comma-separated list of containers (only those RM stopped)
|
||||||
start_containers() {
|
# Named rw_ to avoid colliding with common.sh's start_containers() (different signature/semantics)
|
||||||
|
rw_start_containers() {
|
||||||
local IFS=','
|
local IFS=','
|
||||||
for container in $1; do
|
for container in $1; do
|
||||||
[[ -z "$container" ]] && continue
|
[[ -z "$container" ]] && continue
|
||||||
@@ -468,7 +468,7 @@ apply_level_3() {
|
|||||||
|
|
||||||
if [[ ${#RW_STOP_CONTAINERS[@]} -gt 0 ]]; then
|
if [[ ${#RW_STOP_CONTAINERS[@]} -gt 0 ]]; then
|
||||||
local newly_stopped
|
local newly_stopped
|
||||||
newly_stopped=$(stop_containers "${RW_STOP_CONTAINERS[@]}")
|
newly_stopped=$(rw_stop_containers "${RW_STOP_CONTAINERS[@]}")
|
||||||
if [[ -n "$newly_stopped" ]]; then
|
if [[ -n "$newly_stopped" ]]; then
|
||||||
if [[ -n "$STOPPED_LIST" ]]; then
|
if [[ -n "$STOPPED_LIST" ]]; then
|
||||||
STOPPED_LIST="${STOPPED_LIST},${newly_stopped}"
|
STOPPED_LIST="${STOPPED_LIST},${newly_stopped}"
|
||||||
@@ -490,7 +490,7 @@ apply_level_3() {
|
|||||||
restore_level_3() {
|
restore_level_3() {
|
||||||
echo "Restoring from level 3 — starting stopped containers"
|
echo "Restoring from level 3 — starting stopped containers"
|
||||||
if [[ -n "$STOPPED_LIST" ]]; then
|
if [[ -n "$STOPPED_LIST" ]]; then
|
||||||
start_containers "$STOPPED_LIST"
|
rw_start_containers "$STOPPED_LIST"
|
||||||
STOPPED_LIST=""
|
STOPPED_LIST=""
|
||||||
fi
|
fi
|
||||||
rm_state_set_eq "mem_shutdown_active" "false"
|
rm_state_set_eq "mem_shutdown_active" "false"
|
||||||
|
|||||||
@@ -192,15 +192,15 @@ fi
|
|||||||
# ── STATE HELPERS ─────────────────────────────────────────────────────────────────────────────
|
# ── STATE HELPERS ─────────────────────────────────────────────────────────────────────────────
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
|
|
||||||
|
# Wrap common.sh's wd_state_get()/wd_state_set() — file is implicit here (this script's
|
||||||
|
# own state file), unlike the generic helper which always takes it as an argument.
|
||||||
get_strikes() {
|
get_strikes() {
|
||||||
grep -E "^${1}:" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | cut -d':' -f2
|
wd_state_get "$1" "$SYS_WATCHDOG_STATE_FILE"
|
||||||
}
|
}
|
||||||
|
|
||||||
set_strikes() {
|
set_strikes() {
|
||||||
local key="$1" count="$2"
|
local key="$1" count="$2"
|
||||||
grep -vE "^${key}:" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null > "${SYS_WATCHDOG_STATE_FILE}.tmp"
|
wd_state_set "$key" "$count" "$SYS_WATCHDOG_STATE_FILE"
|
||||||
echo "${key}:${count}" >> "${SYS_WATCHDOG_STATE_FILE}.tmp"
|
|
||||||
mv "${SYS_WATCHDOG_STATE_FILE}.tmp" "$SYS_WATCHDOG_STATE_FILE"
|
|
||||||
}
|
}
|
||||||
|
|
||||||
increment_strikes() {
|
increment_strikes() {
|
||||||
@@ -218,14 +218,12 @@ reset_strikes() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
get_state_val() {
|
get_state_val() {
|
||||||
grep -E "^${1}=" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | cut -d'=' -f2
|
wd_state_get "$1" "$SYS_WATCHDOG_STATE_FILE" "="
|
||||||
}
|
}
|
||||||
|
|
||||||
set_state_val() {
|
set_state_val() {
|
||||||
local key="$1" val="$2"
|
local key="$1" val="$2"
|
||||||
grep -vE "^${key}=" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null > "${SYS_WATCHDOG_STATE_FILE}.tmp"
|
wd_state_set "$key" "$val" "$SYS_WATCHDOG_STATE_FILE" "="
|
||||||
echo "${key}=${val}" >> "${SYS_WATCHDOG_STATE_FILE}.tmp"
|
|
||||||
mv "${SYS_WATCHDOG_STATE_FILE}.tmp" "$SYS_WATCHDOG_STATE_FILE"
|
|
||||||
}
|
}
|
||||||
|
|
||||||
purge_old_reboots() {
|
purge_old_reboots() {
|
||||||
|
|||||||
@@ -49,8 +49,7 @@
|
|||||||
# check_arr_version() added — API version safety gate before arr operations
|
# check_arr_version() added — API version safety gate before arr operations
|
||||||
# check_api() added — pre-flight API reachability check
|
# check_api() added — pre-flight API reachability check
|
||||||
# ping_remote(), ping_internet() — non-fatal ping for fallback use
|
# ping_remote(), ping_internet() — non-fatal ping for fallback use
|
||||||
# check_local_array(), check_remote_array() — array health checks
|
# check_remote_array() — array health check
|
||||||
# check_remote_docker() — Docker daemon health check
|
|
||||||
# get_unraid_temp_thresholds() — reads thresholds from dynamix.cfg
|
# get_unraid_temp_thresholds() — reads thresholds from dynamix.cfg
|
||||||
# check_local_disk_temps() — pre-rsync temp check with exit codes 0/1/2
|
# check_local_disk_temps() — pre-rsync temp check with exit codes 0/1/2
|
||||||
# stop/start local containers added alongside existing remote variants
|
# stop/start local containers added alongside existing remote variants
|
||||||
@@ -79,15 +78,12 @@
|
|||||||
# Called by lidarr/sonarr/radarr_cleanup.sh when files are deleted
|
# Called by lidarr/sonarr/radarr_cleanup.sh when files are deleted
|
||||||
#
|
#
|
||||||
#
|
#
|
||||||
# Three new safety functions added:
|
# Two new safety functions added:
|
||||||
# check_os_version_parity() — refuses remote ops on version mismatch
|
# check_os_version_parity() — refuses remote ops on version mismatch
|
||||||
# reads OS version via platform_get_os_version() / platform_os_version_probe_cmd()
|
# reads OS version via platform_get_os_version() / platform_os_version_probe_cmd()
|
||||||
# major mismatch → abort | minor mismatch → configurable warn/abort
|
# major mismatch → abort | minor mismatch → configurable warn/abort
|
||||||
# check_remote_docker_daemon() — verifies remote Docker daemon before
|
# check_remote_docker_daemon() — verifies remote Docker daemon before
|
||||||
# issuing any remote container commands — strike system → skip/retry/exit
|
# issuing any remote container commands — strike system → skip/retry/exit
|
||||||
# validate_unraid_cmd() — verifies unRAID-specific commands exist and
|
|
||||||
# produce expected output before use — notifies and exits calling script
|
|
||||||
# if command changed or disappeared after upgrade — other scripts unaffected
|
|
||||||
#
|
#
|
||||||
# ── VERSION ───────────────────────────────────────────────────────────────────────────────────
|
# ── VERSION ───────────────────────────────────────────────────────────────────────────────────
|
||||||
# Current: v3.5
|
# Current: v3.5
|
||||||
@@ -130,6 +126,7 @@ ICON_SYNC="🔄" # transfer section header
|
|||||||
ICON_RUN="🚀" # sync starting / rsync attempt
|
ICON_RUN="🚀" # sync starting / rsync attempt
|
||||||
ICON_RETRY="🔁" # retry attempt
|
ICON_RETRY="🔁" # retry attempt
|
||||||
ICON_DONE="🏁" # transfer complete
|
ICON_DONE="🏁" # transfer complete
|
||||||
|
ICON_GIT="🌱" # git pull/push section header
|
||||||
|
|
||||||
# Summary
|
# Summary
|
||||||
ICON_SUMMARY="📋" # summary section header
|
ICON_SUMMARY="📋" # summary section header
|
||||||
@@ -269,6 +266,33 @@ format_duration() {
|
|||||||
else echo "${rem}s"; fi
|
else echo "${rem}s"; fi
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# Tiered human-readable size — picks B/MB/GB based on magnitude.
|
||||||
|
# Usage: format_bytes 5368709120 → 5.0GB
|
||||||
|
format_bytes() {
|
||||||
|
local bytes=${1:-0}
|
||||||
|
if (( bytes > 1073741824 )); then
|
||||||
|
awk "BEGIN {printf \"%.1fGB\", $bytes / 1073741824}"
|
||||||
|
elif (( bytes > 1048576 )); then
|
||||||
|
awk "BEGIN {printf \"%.1fMB\", $bytes / 1048576}"
|
||||||
|
else
|
||||||
|
echo "${bytes}B"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# Fixed-precision bytes → GB (no tiering) — for values always compared against a GB threshold.
|
||||||
|
# Usage: bytes_to_gb 5368709120 [decimals=2] → 5.00
|
||||||
|
bytes_to_gb() {
|
||||||
|
local bytes=${1:-0} decimals=${2:-2}
|
||||||
|
awk "BEGIN {printf \"%.${decimals}f\", $bytes / 1073741824}"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Fixed-precision KB → GB (no tiering) — df --output=used/avail reports in KB.
|
||||||
|
# Usage: kb_to_gb 5242880 [decimals=2] → 5.00
|
||||||
|
kb_to_gb() {
|
||||||
|
local kb=${1:-0} decimals=${2:-2}
|
||||||
|
awk "BEGIN {printf \"%.${decimals}f\", $kb / 1048576}"
|
||||||
|
}
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ── ARG PARSER ────────────────────────────────────────────────────────────────────────────────
|
# ── ARG PARSER ────────────────────────────────────────────────────────────────────────────────
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
@@ -355,6 +379,17 @@ validate_int() {
|
|||||||
log "$name validated: $value"
|
log "$name validated: $value"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# Membership check against a list of values (e.g. an ignore list).
|
||||||
|
# Usage: is_in_list "$needle" "${HAYSTACK_ARRAY[@]}"
|
||||||
|
is_in_list() {
|
||||||
|
local needle="$1"; shift
|
||||||
|
local item
|
||||||
|
for item in "$@"; do
|
||||||
|
[[ "$needle" == "$item" ]] && return 0
|
||||||
|
done
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ── HOST DETECTION ────────────────────────────────────────────────────────────────────────────
|
# ── HOST DETECTION ────────────────────────────────────────────────────────────────────────────
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
@@ -506,6 +541,7 @@ detect_hosts() {
|
|||||||
SYS_WATCHDOG_CHECK_MDSTAT SYS_WATCHDOG_CHECK_NETWORK
|
SYS_WATCHDOG_CHECK_MDSTAT SYS_WATCHDOG_CHECK_NETWORK
|
||||||
SYS_WATCHDOG_CHECK_SSHD SYS_WATCHDOG_CHECK_RUNAWAY
|
SYS_WATCHDOG_CHECK_SSHD SYS_WATCHDOG_CHECK_RUNAWAY
|
||||||
)
|
)
|
||||||
|
local _wd_var
|
||||||
for _wd_var in "${_wd_checks[@]}"; do
|
for _wd_var in "${_wd_checks[@]}"; do
|
||||||
local _wd_src="${MY_ID}_${_wd_var}"
|
local _wd_src="${MY_ID}_${_wd_var}"
|
||||||
# Only alias if the host-specific var is set — preserves master.conf defaults
|
# Only alias if the host-specific var is set — preserves master.conf defaults
|
||||||
@@ -625,6 +661,86 @@ detect_hosts() {
|
|||||||
log "$ICON_HOST Host: $LOCAL_SERVER_NAME → $REMOTE_SERVER_NAME"
|
log "$ICON_HOST Host: $LOCAL_SERVER_NAME → $REMOTE_SERVER_NAME"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# Derives owner/mirror roles from PARTNERSHIP_OWNER_HOST — used by every Partnership/*.sh
|
||||||
|
# script. Requires detect_hosts() to have already run (needs MY_ID, SSH_KEY).
|
||||||
|
# Sets: OWNER_ID MIRROR_ID OWNER MIRROR MIRROR_SSH_KEY OWNER_SSH_KEY AM_OWNER AM_MIRROR
|
||||||
|
# Does NOT set any *_STATE_FILE vars — callers that need those set them after calling this,
|
||||||
|
# since not every Partnership script needs the same set (transfer/onboard need fewer than
|
||||||
|
# manager/offboard).
|
||||||
|
partnership_resolve_roles() {
|
||||||
|
OWNER_ID="${PARTNERSHIP_OWNER_HOST:-HOST1}"
|
||||||
|
MIRROR_ID=$( [[ "$OWNER_ID" == "HOST1" ]] && echo "HOST2" || echo "HOST1" )
|
||||||
|
OWNER="${!OWNER_ID}"
|
||||||
|
MIRROR="${!MIRROR_ID}"
|
||||||
|
# SSH_KEY (set by detect_hosts) is this server's own private key. The remote accepts it
|
||||||
|
# because this server's PUBLIC key was installed there via ssh_setup.sh. With sparse
|
||||||
|
# checkout, each server only has its own host{N}.conf — the other server's key path is
|
||||||
|
# never available here. Use SSH_KEY for all outbound SSH regardless of mode.
|
||||||
|
MIRROR_SSH_KEY="$SSH_KEY"
|
||||||
|
OWNER_SSH_KEY="$SSH_KEY"
|
||||||
|
|
||||||
|
AM_OWNER=false
|
||||||
|
AM_MIRROR=false
|
||||||
|
[[ "$MY_ID" == "$OWNER_ID" ]] && AM_OWNER=true
|
||||||
|
[[ "$MY_ID" == "$MIRROR_ID" ]] && AM_MIRROR=true
|
||||||
|
}
|
||||||
|
|
||||||
|
# Read a scalar/array var from a partner's own config via SSH — the partner sources its
|
||||||
|
# own load_config.sh + detect_hosts() remotely so the value reflects ITS host*.conf, not
|
||||||
|
# ours. Requires $SCRIPT_DIR to be set by the caller (one level above the repo root, same
|
||||||
|
# convention every Partnership/*.sh script already uses for its own SCRIPT_DIR).
|
||||||
|
# Usage: read_remote_conf_var "$mirror_ip" "VAR_NAME"
|
||||||
|
# read_remote_conf_array "$mirror_ip" "ARRAY_NAME" # one element per line
|
||||||
|
read_remote_conf_var() {
|
||||||
|
local mirror_ip="$1" var_name="$2"
|
||||||
|
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||||
|
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \
|
||||||
|
"source '$SCRIPT_DIR/../load_config.sh' 2>/dev/null
|
||||||
|
detect_hosts 2>/dev/null
|
||||||
|
printf '%s' \"\${${var_name}:-}\"" 2>/dev/null
|
||||||
|
}
|
||||||
|
|
||||||
|
read_remote_conf_array() {
|
||||||
|
local mirror_ip="$1" var_name="$2"
|
||||||
|
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||||
|
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \
|
||||||
|
"source '$SCRIPT_DIR/../load_config.sh' 2>/dev/null
|
||||||
|
detect_hosts 2>/dev/null
|
||||||
|
printf '%s\n' \"\${${var_name}[@]:-}\"" 2>/dev/null
|
||||||
|
}
|
||||||
|
|
||||||
|
# Probes a remote host's SCRIPTS_DIR (in case it differs from ours — e.g. one host runs
|
||||||
|
# internal storage mode, the other flash), falling back to our own $SCRIPTS_DIR if the
|
||||||
|
# probe fails or the remote isn't reachable. strict_host_key defaults to "yes" (matches
|
||||||
|
# existing behavior everywhere except partnership_transfer.sh, which passes "no" since
|
||||||
|
# it may be contacting a mirror for the first time during an ownership transfer).
|
||||||
|
# Usage: resolve_remote_scripts_dir "$ip" ["$ssh_key"] ["no"|"yes"]
|
||||||
|
resolve_remote_scripts_dir() {
|
||||||
|
local ip="$1" ssh_key="${2:-$SSH_KEY}" strict_host_key="${3:-yes}"
|
||||||
|
local probe_cmd result
|
||||||
|
local -a opts=(-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes)
|
||||||
|
[[ "$strict_host_key" == "no" ]] && opts+=(-o StrictHostKeyChecking=no)
|
||||||
|
probe_cmd=$(platform_scripts_dir_probe_cmd)
|
||||||
|
result=$(timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" "${opts[@]}" root@"$ip" "$probe_cmd" \
|
||||||
|
2>/dev/null | tr -d '[:space:]')
|
||||||
|
echo "${result:-$SCRIPTS_DIR}"
|
||||||
|
}
|
||||||
|
|
||||||
|
# ==============================================================================================
|
||||||
|
# ── MULTI-NODE DISCOVERY ──────────────────────────────────────────────────────────────────────
|
||||||
|
# ==============================================================================================
|
||||||
|
# Scans HOST1..HOST8 for every configured node other than this one. Requires detect_hosts()
|
||||||
|
# to have already run (needs MY_ID). Sets the global REMOTE_NODES array — empty if none found.
|
||||||
|
discover_remote_nodes() {
|
||||||
|
REMOTE_NODES=()
|
||||||
|
local _hv
|
||||||
|
for _hv in HOST1 HOST2 HOST3 HOST4 HOST5 HOST6 HOST7 HOST8; do
|
||||||
|
[[ "$_hv" == "$MY_ID" ]] && continue
|
||||||
|
[[ -z "${!_hv:-}" ]] && continue
|
||||||
|
REMOTE_NODES+=("$_hv")
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ── REMOTE IP RESOLUTION ──────────────────────────────────────────────────────────────────────
|
# ── REMOTE IP RESOLUTION ──────────────────────────────────────────────────────────────────────
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
@@ -669,6 +785,34 @@ resolve_tailscale_ip() {
|
|||||||
echo "$ip"
|
echo "$ip"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# Strips the "unraid-" prefix (case-insensitive) and title-cases what remains.
|
||||||
|
# Usage: derive_short_name "unRAID-Gmer4Lfe" → "Gmer4lfe"
|
||||||
|
derive_short_name() {
|
||||||
|
local hostname="$1"
|
||||||
|
local short="${hostname,,}"
|
||||||
|
[[ "$short" == unraid-* ]] && short="${short:7}"
|
||||||
|
echo "${short^}"
|
||||||
|
}
|
||||||
|
|
||||||
|
# True if the given host*.conf basename belongs to this host (case-insensitive).
|
||||||
|
# Usage: is_own_conf_file "$(basename "$conf")"
|
||||||
|
is_own_conf_file() {
|
||||||
|
[[ "${1,,}" == "${MY_ID,,}.conf" ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
# Sets key=value in a flat state file (setup.db style) — updates in place if the
|
||||||
|
# key exists, appends if not. No indentation handling — for master.conf use
|
||||||
|
# partnership_manager.sh's update_master_conf() instead.
|
||||||
|
# Usage: set_state_var "$state_file" "$key" "$value"
|
||||||
|
set_state_var() {
|
||||||
|
local file="$1" key="$2" value="$3"
|
||||||
|
if grep -q "^${key}=" "$file" 2>/dev/null; then
|
||||||
|
sed -i "s|^${key}=.*|${key}=${value}|" "$file"
|
||||||
|
else
|
||||||
|
echo "${key}=${value}" >> "$file"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ── CONNECTIVITY CHECKS ───────────────────────────────────────────────────────────────────────
|
# ── CONNECTIVITY CHECKS ───────────────────────────────────────────────────────────────────────
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
@@ -755,26 +899,6 @@ is_vm_manager_enabled() {
|
|||||||
# ── LOCAL HEALTH CHECKS ───────────────────────────────────────────────────────────────────────
|
# ── LOCAL HEALTH CHECKS ───────────────────────────────────────────────────────────────────────
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
|
|
||||||
# Verifies local storage is mounted and has shares.
|
|
||||||
# Non-fatal — returns status for caller to decide.
|
|
||||||
# Used by fallback before starting remote containers locally.
|
|
||||||
check_local_array() {
|
|
||||||
log "Checking local array..."
|
|
||||||
if ! platform_storage_healthy; then
|
|
||||||
error "$ICON_DISK Local array is not started — storage not mounted"
|
|
||||||
return 1
|
|
||||||
fi
|
|
||||||
local _storage_path file_count
|
|
||||||
_storage_path=$(platform_storage_path)
|
|
||||||
file_count=$(ls "$_storage_path" 2>/dev/null | wc -l)
|
|
||||||
if [[ "$file_count" -eq 0 ]]; then
|
|
||||||
error "$ICON_DISK Local array appears empty — shares may not be available"
|
|
||||||
return 1
|
|
||||||
fi
|
|
||||||
log "Local array is healthy"
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ── REMOTE HEALTH CHECKS ─────────────────────────────────────────────────────────────────────
|
# ── REMOTE HEALTH CHECKS ─────────────────────────────────────────────────────────────────────
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
@@ -796,19 +920,6 @@ check_remote_array() {
|
|||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
|
|
||||||
# Verifies remote Docker daemon is responding.
|
|
||||||
# Non-fatal — returns status. A hung daemon means container commands silently fail.
|
|
||||||
check_remote_docker() {
|
|
||||||
log "Checking remote Docker daemon on $REMOTE_SERVER_NAME..."
|
|
||||||
if ! ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \
|
|
||||||
"timeout 10 docker ps" >/dev/null 2>&1; then
|
|
||||||
error "$ICON_CONTAINERS Remote Docker daemon not responding on $REMOTE_SERVER_NAME"
|
|
||||||
return 1
|
|
||||||
fi
|
|
||||||
log "Remote Docker daemon is healthy"
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
# Aborts if remote rootfs (/) usage is at or above ROOTFS_WARN threshold.
|
# Aborts if remote rootfs (/) usage is at or above ROOTFS_WARN threshold.
|
||||||
# Fatal — exits the calling script.
|
# Fatal — exits the calling script.
|
||||||
# When remote array is down rsync writes land on rootfs and fill it rapidly.
|
# When remote array is down rsync writes land on rootfs and fill it rapidly.
|
||||||
@@ -861,9 +972,14 @@ check_remote_share() {
|
|||||||
# Returns 0 (true) if device is non-rotational (SSD/NVMe), 1 (false) if HDD.
|
# Returns 0 (true) if device is non-rotational (SSD/NVMe), 1 (false) if HDD.
|
||||||
# Checks /sys/block/<device>/queue/rotational — 0=SSD, 1=HDD.
|
# Checks /sys/block/<device>/queue/rotational — 0=SSD, 1=HDD.
|
||||||
is_ssd() {
|
is_ssd() {
|
||||||
local base
|
local drive="$1"
|
||||||
base=$(basename "$1")
|
local base rotational
|
||||||
[[ "$(cat "/sys/block/$base/queue/rotational" 2>/dev/null)" == "0" ]]
|
base=$(basename "$drive")
|
||||||
|
rotational="/sys/block/$base/queue/rotational"
|
||||||
|
[[ -f "$rotational" ]] && [[ "$(cat "$rotational" 2>/dev/null)" == "0" ]] && return 0
|
||||||
|
# NVMe is always SSD — some NVMe controllers don't expose rotational correctly
|
||||||
|
[[ "$drive" == *nvme* ]] && return 0
|
||||||
|
return 1
|
||||||
}
|
}
|
||||||
|
|
||||||
# Sets globals: UNRAID_DISK_HOT UNRAID_DISK_MAX UNRAID_SSD_HOT UNRAID_SSD_MAX
|
# Sets globals: UNRAID_DISK_HOT UNRAID_DISK_MAX UNRAID_SSD_HOT UNRAID_SSD_MAX
|
||||||
@@ -878,6 +994,37 @@ get_unraid_temp_thresholds() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
# Checks local disk temperatures before rsync.
|
# Checks local disk temperatures before rsync.
|
||||||
|
# Fetches a domain's TLS cert expiry via openssl s_client + x509 and computes days
|
||||||
|
# remaining. Pure data — no logging, no thresholds — callers classify warn/crit
|
||||||
|
# themselves against their own CERT_WARN_DAYS/CERT_CRIT_DAYS and log however they like.
|
||||||
|
#
|
||||||
|
# Sets _CERT_DAYS, _CERT_EXPIRY (YYYY-MM-DD) on success. Sets _CERT_EXPIRY_RAW to the
|
||||||
|
# unparsed openssl date string only when parsing failed (empty when unreachable) so a
|
||||||
|
# caller that wants a more specific error message can distinguish the two failure modes.
|
||||||
|
#
|
||||||
|
# Usage: check_cert_expiry "$domain" [port=443] [timeout=$CERT_TIMEOUT or 10]
|
||||||
|
# Returns: 0 = fetched and parsed | 1 = unreachable or unparseable
|
||||||
|
check_cert_expiry() {
|
||||||
|
local domain="$1" port="${2:-443}" cert_timeout="${3:-${CERT_TIMEOUT:-10}}"
|
||||||
|
_CERT_DAYS=""
|
||||||
|
_CERT_EXPIRY=""
|
||||||
|
_CERT_EXPIRY_RAW=""
|
||||||
|
local expiry_str
|
||||||
|
expiry_str=$(echo | timeout "$cert_timeout" openssl s_client \
|
||||||
|
-connect "${domain}:${port}" -servername "$domain" \
|
||||||
|
2>/dev/null | openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2)
|
||||||
|
[[ -z "$expiry_str" ]] && return 1
|
||||||
|
local expiry_epoch
|
||||||
|
expiry_epoch=$(date -d "$expiry_str" +%s 2>/dev/null)
|
||||||
|
if [[ -z "$expiry_epoch" ]]; then
|
||||||
|
_CERT_EXPIRY_RAW="$expiry_str"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
_CERT_DAYS=$(( (expiry_epoch - $(date +%s)) / 86400 ))
|
||||||
|
_CERT_EXPIRY=$(date -d "$expiry_str" '+%Y-%m-%d' 2>/dev/null)
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
# Reads temps and rotational flag from /var/local/emhttp/disks.ini.
|
# Reads temps and rotational flag from /var/local/emhttp/disks.ini.
|
||||||
# Uses unRAID's own thresholds from dynamix.cfg via get_unraid_temp_thresholds().
|
# Uses unRAID's own thresholds from dynamix.cfg via get_unraid_temp_thresholds().
|
||||||
#
|
#
|
||||||
@@ -1305,6 +1452,62 @@ start_local_containers() {
|
|||||||
done
|
done
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# Builds a dependency-safe restart order — dependency containers (anything named as a
|
||||||
|
# value in WATCHDOG_DEPENDENCIES) restart first, then everything else in original order.
|
||||||
|
# Sets ORDERED_RESTART. Usage: build_restart_order CONTAINERS_ARRAY_NAME (bare name, e.g.
|
||||||
|
# build_restart_order DAILY_RESTART_CONTAINERS — not "${DAILY_RESTART_CONTAINERS[@]}").
|
||||||
|
build_restart_order() {
|
||||||
|
local -n _bro_containers="$1"
|
||||||
|
ORDERED_RESTART=()
|
||||||
|
local remaining=("${_bro_containers[@]}")
|
||||||
|
local placed=()
|
||||||
|
|
||||||
|
# First pass — add dependency containers that appear in our list
|
||||||
|
for container in "${remaining[@]}"; do
|
||||||
|
[[ -z "$container" ]] && continue
|
||||||
|
local is_dependency=false
|
||||||
|
for dependent in "${!WATCHDOG_DEPENDENCIES[@]}"; do
|
||||||
|
if [[ "${WATCHDOG_DEPENDENCIES[$dependent]}" == *"$container"* ]]; then
|
||||||
|
is_dependency=true
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
if [[ "$is_dependency" == true ]]; then
|
||||||
|
local already=false
|
||||||
|
for p in "${placed[@]}"; do [[ "$p" == "$container" ]] && already=true && break; done
|
||||||
|
if [[ "$already" == false ]]; then
|
||||||
|
ORDERED_RESTART+=("$container")
|
||||||
|
placed+=("$container")
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
# Second pass — add remaining containers (dependents and independents)
|
||||||
|
for container in "${remaining[@]}"; do
|
||||||
|
[[ -z "$container" ]] && continue
|
||||||
|
local already=false
|
||||||
|
for p in "${placed[@]}"; do [[ "$p" == "$container" ]] && already=true && break; done
|
||||||
|
if [[ "$already" == false ]]; then
|
||||||
|
ORDERED_RESTART+=("$container")
|
||||||
|
placed+=("$container")
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
log "Restart order: ${ORDERED_RESTART[*]}"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Waits CONTAINER_DELAY if this container depends on the last restarted one.
|
||||||
|
# Usage: check_dependency_delay "$container" "$last_restarted"
|
||||||
|
check_dependency_delay() {
|
||||||
|
local container="$1" last="$2"
|
||||||
|
[[ -z "$last" ]] && return
|
||||||
|
local deps="${WATCHDOG_DEPENDENCIES[$container]:-}"
|
||||||
|
if [[ -n "$deps" ]] && [[ "$deps" == *"$last"* ]]; then
|
||||||
|
log "Waiting ${CONTAINER_DELAY}s — $container depends on $last..."
|
||||||
|
sleep "$CONTAINER_DELAY"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ── RSYNC OPTIONS ─────────────────────────────────────────────────────────────────────────────
|
# ── RSYNC OPTIONS ─────────────────────────────────────────────────────────────────────────────
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
@@ -1380,12 +1583,6 @@ _lock_file() {
|
|||||||
echo "$LOCK_DIR/${1:-$(_lock_name)}.lock"
|
echo "$LOCK_DIR/${1:-$(_lock_name)}.lock"
|
||||||
}
|
}
|
||||||
|
|
||||||
# Internal — release lock on exit (registered via EXIT trap)
|
|
||||||
_release_on_exit() {
|
|
||||||
local lockfile="$1"
|
|
||||||
[[ -f "$lockfile" ]] && rm -f "$lockfile"
|
|
||||||
}
|
|
||||||
|
|
||||||
# Acquire exclusive lock for this script.
|
# Acquire exclusive lock for this script.
|
||||||
# Usage: acquire_lock [strict|wait|continuous]
|
# Usage: acquire_lock [strict|wait|continuous]
|
||||||
acquire_lock() {
|
acquire_lock() {
|
||||||
@@ -1524,17 +1721,30 @@ acquire_rsync_lock() {
|
|||||||
log "$ICON_LOCK rsync lock acquired: profile '$profile' (PID $$, active: $(( current_count + 1 ))/$RSYNC_MAX_CONCURRENT)"
|
log "$ICON_LOCK rsync lock acquired: profile '$profile' (PID $$, active: $(( current_count + 1 ))/$RSYNC_MAX_CONCURRENT)"
|
||||||
}
|
}
|
||||||
|
|
||||||
# Internal — release rsync lock and decrement global counter on exit
|
# ==============================================================================================
|
||||||
_release_rsync_on_exit() {
|
# ── FLAT STATE-FILE KEY/VALUE HELPERS ─────────────────────────────────────────────────────────
|
||||||
local profile_lock="$1"
|
# ==============================================================================================
|
||||||
[[ -f "$profile_lock" ]] && rm -f "$profile_lock"
|
# Generic get/set for flat "key<sep>value" state files (one entry per line) — the pattern
|
||||||
if [[ -f "$RSYNC_COUNT_FILE" ]]; then
|
# every watchdog's strike-tracking and state-file logic was independently reimplementing.
|
||||||
local count
|
# grep -v + tempfile-swap on write, not sed -i in place — avoids sed treating a key containing
|
||||||
count=$(cat "$RSYNC_COUNT_FILE" 2>/dev/null || echo 1)
|
# regex metacharacters (container names, etc.) as part of the substitution pattern.
|
||||||
count=$(( count - 1 ))
|
#
|
||||||
[[ "$count" -lt 0 ]] && count=0
|
# Usage: wd_state_get "$key" "$file" [sep=:]
|
||||||
echo "$count" > "$RSYNC_COUNT_FILE"
|
# wd_state_set "$key" "$value" "$file" [sep=:]
|
||||||
fi
|
#
|
||||||
|
# Callers with a fixed state file and/or separator (e.g. stability_watchdog.sh's
|
||||||
|
# get_strikes/get_state_val, resource_watchdog.sh's rm_state_get/rm_state_get_eq) should
|
||||||
|
# keep their own thin same-named wrapper around these rather than changing call sites.
|
||||||
|
wd_state_get() {
|
||||||
|
local key="$1" file="$2" sep="${3:-:}"
|
||||||
|
grep -E "^${key}${sep}" "$file" 2>/dev/null | cut -d"$sep" -f2-
|
||||||
|
}
|
||||||
|
|
||||||
|
wd_state_set() {
|
||||||
|
local key="$1" value="$2" file="$3" sep="${4:-:}"
|
||||||
|
grep -vE "^${key}${sep}" "$file" 2>/dev/null > "${file}.tmp"
|
||||||
|
echo "${key}${sep}${value}" >> "${file}.tmp"
|
||||||
|
mv "${file}.tmp" "$file"
|
||||||
}
|
}
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
@@ -1632,6 +1842,348 @@ translate_path() {
|
|||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# ==============================================================================================
|
||||||
|
# ── CONTAINER STOP/RESTART FOR MAINTENANCE ────────────────────────────────────────────────────
|
||||||
|
# ==============================================================================================
|
||||||
|
# The "stop a container for a maintenance window, guarantee it comes back" shape shared by
|
||||||
|
# container_data_export.sh and emby_database_repair.sh. Callers still register their own EXIT
|
||||||
|
# trap (they may need extra cleanup, e.g. removing a partial archive) — the trap should call
|
||||||
|
# container_force_restart_if_needed() for the actual "still down? bring it back" part.
|
||||||
|
|
||||||
|
# Usage: container_stop_for_maintenance "$container" was_running_var_name \
|
||||||
|
# ["pre-stop reason message"] [log|warn] [post_stop_sleep=0]
|
||||||
|
# The reason message (if given) prints via the chosen level (default: log) right before the
|
||||||
|
# stop attempt, only when the container was actually running — callers use this for their
|
||||||
|
# own context ("stopping for clean export" vs "active sessions will be interrupted").
|
||||||
|
# Sets the named var true/false. Returns 1 (caller should exit) if the container doesn't
|
||||||
|
# exist or fails to stop; 0 otherwise (including the "wasn't running" case — caller logs
|
||||||
|
# its own context-specific message for that, since wording/visibility differs per caller).
|
||||||
|
container_stop_for_maintenance() {
|
||||||
|
local container="$1"
|
||||||
|
local -n _csm_was_running="$2"
|
||||||
|
local pre_stop_msg="${3:-}" pre_stop_level="${4:-log}" post_stop_sleep="${5:-0}"
|
||||||
|
_csm_was_running=false
|
||||||
|
|
||||||
|
local status
|
||||||
|
status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
|
||||||
|
case "$status" in
|
||||||
|
true)
|
||||||
|
_csm_was_running=true
|
||||||
|
if [[ -n "$pre_stop_msg" ]]; then
|
||||||
|
if [[ "$pre_stop_level" == "warn" ]]; then warn "$pre_stop_msg"; else log "$pre_stop_msg"; fi
|
||||||
|
fi
|
||||||
|
if [[ "$DRY_RUN" == false ]]; then
|
||||||
|
if timeout "$DOCKER_TIMEOUT" docker stop "$container" >/dev/null 2>&1; then
|
||||||
|
echo "$container stopped ✅"
|
||||||
|
[[ "$post_stop_sleep" -gt 0 ]] && sleep "$post_stop_sleep"
|
||||||
|
else
|
||||||
|
error "Failed to stop $container — aborting"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
warn "DRY RUN — would stop $container"
|
||||||
|
fi
|
||||||
|
;;
|
||||||
|
false) : ;; # not running — caller logs its own context-specific message
|
||||||
|
"")
|
||||||
|
error "$container not found — check container name"
|
||||||
|
return 1
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
warn "$container status: $status — proceeding with caution"
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
# Force-restart from inside a script's own EXIT trap if the container is still down after
|
||||||
|
# a crash mid-maintenance. No-op if it wasn't running before, in dry-run, or already up.
|
||||||
|
# Usage: container_force_restart_if_needed "$container" "$was_running"
|
||||||
|
container_force_restart_if_needed() {
|
||||||
|
local container="$1" was_running="$2"
|
||||||
|
[[ "$was_running" == true && "$DRY_RUN" == false ]] || return 0
|
||||||
|
local status
|
||||||
|
status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
|
||||||
|
if [[ "$status" != "true" ]]; then
|
||||||
|
warn "Restarting $container (cleanup)..."
|
||||||
|
timeout "$DOCKER_TIMEOUT" docker start "$container" >/dev/null 2>&1 || \
|
||||||
|
error "Failed to restart $container — start it manually"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# Normal-path restart after maintenance completes — verifies it stayed up, notifies on failure.
|
||||||
|
# Sets RESTART_OK true/false.
|
||||||
|
# Usage: container_restart_after_maintenance "$container" "$was_running" [settle_sleep=3] [notify_label]
|
||||||
|
container_restart_after_maintenance() {
|
||||||
|
local container="$1" was_running="$2" settle_sleep="${3:-3}" notify_label="${4:-Container Maintenance}"
|
||||||
|
RESTART_OK=false
|
||||||
|
|
||||||
|
if [[ "$was_running" != true ]]; then
|
||||||
|
echo "$container was not running — leaving stopped (state respected) ✅"
|
||||||
|
RESTART_OK=true
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
|
warn "DRY RUN — would restart $container"
|
||||||
|
RESTART_OK=true
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
log "Restarting $container..."
|
||||||
|
if timeout "$DOCKER_TIMEOUT" docker start "$container" >/dev/null 2>&1; then
|
||||||
|
sleep "$settle_sleep"
|
||||||
|
local post_status
|
||||||
|
post_status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
|
||||||
|
if [[ "$post_status" == "true" ]]; then
|
||||||
|
echo "$container restarted and running ✅"
|
||||||
|
RESTART_OK=true
|
||||||
|
else
|
||||||
|
error "$container started but crashed — check container logs"
|
||||||
|
notify "$container failed to stay running after maintenance on $(hostname)" \
|
||||||
|
"$notify_label" "warning"
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
error "Failed to restart $container — start it manually"
|
||||||
|
notify "$container failed to restart after maintenance on $(hostname)" \
|
||||||
|
"$notify_label" "warning"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# ==============================================================================================
|
||||||
|
# ── ARR CLEANUP SAFETY GATES ──────────────────────────────────────────────────────────────────
|
||||||
|
# ==============================================================================================
|
||||||
|
# Shared by lidarr/radarr/sonarr_cleanup.sh — these scripts DELETE files, so every function
|
||||||
|
# here is a direct byte-for-byte port of what was independently duplicated three times, not
|
||||||
|
# a rewrite. Callers should behave identically to before this consolidation.
|
||||||
|
|
||||||
|
# Safety Layer 1 — container running, not starting/unhealthy. Exits 1 (with notify) on failure.
|
||||||
|
# Usage: check_container_health "$LIDARR_CONTAINER" "$ARR_DOCKER_TIMEOUT" "Lidarr Cleanup"
|
||||||
|
check_container_health() {
|
||||||
|
local container="$1" docker_timeout="$2" notify_label="$3"
|
||||||
|
|
||||||
|
local running
|
||||||
|
running=$(timeout "$docker_timeout" docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
|
||||||
|
if [[ "$running" != "true" ]]; then
|
||||||
|
error "$container is not running — aborting"
|
||||||
|
notify "$notify_label aborted on $(hostname) — container not running" "$notify_label" "warning"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
local health
|
||||||
|
health=$(timeout "$docker_timeout" docker inspect -f '{{.State.Health.Status}}' "$container" 2>/dev/null)
|
||||||
|
case "$health" in
|
||||||
|
healthy) info "$container is healthy" ;;
|
||||||
|
"") info "$container has no health check — proceeding" ;;
|
||||||
|
starting)
|
||||||
|
error "$container is still starting — aborting"
|
||||||
|
notify "$notify_label aborted on $(hostname) — container still starting" "$notify_label" "warning"
|
||||||
|
exit 1 ;;
|
||||||
|
unhealthy)
|
||||||
|
error "$container is unhealthy — aborting"
|
||||||
|
notify "$notify_label aborted on $(hostname) — container unhealthy" "$notify_label" "warning"
|
||||||
|
exit 1 ;;
|
||||||
|
*) warn "$container health: $health — proceeding with caution" ;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
info "Safety layer 1 passed — container healthy"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Safety Layer 6 — abort if tracked count dropped below min_pct of the last known count
|
||||||
|
# (protects against a misconfigured path / partial API response silently wiping the library).
|
||||||
|
# Writes the new baseline only if the check passes. Exits 1 (with notify) on failure.
|
||||||
|
# Usage: check_tracked_count_floor "$TRACKED_COUNT" "$LIDARR_TRACKED_COUNT_FILE" "$LIDARR_MIN_TRACKED_PCT" "Lidarr Cleanup"
|
||||||
|
check_tracked_count_floor() {
|
||||||
|
local tracked_count="$1" baseline_file="$2" min_pct="$3" notify_label="$4"
|
||||||
|
|
||||||
|
if [[ -f "$baseline_file" ]]; then
|
||||||
|
local last_count pct
|
||||||
|
last_count=$(cat "$baseline_file" 2>/dev/null || echo 0)
|
||||||
|
if [[ "$last_count" -gt 0 ]]; then
|
||||||
|
pct=$(awk "BEGIN {printf \"%d\", ($tracked_count / $last_count) * 100}")
|
||||||
|
if [[ "$pct" -lt "$min_pct" ]]; then
|
||||||
|
error "Tracked count dropped to ${pct}% of last run ($tracked_count vs $last_count)"
|
||||||
|
error "Suggests API issue — aborting to prevent mass deletion"
|
||||||
|
error "If expected (large library removal) delete: $baseline_file"
|
||||||
|
notify "$notify_label aborted on $(hostname) — tracked count dropped to ${pct}%" \
|
||||||
|
"$notify_label" "warning"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
info "Tracked count: ${pct}% of last run ($tracked_count vs $last_count) ✅"
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
info "No previous count on record — first run, saving baseline"
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "$tracked_count" > "$baseline_file"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Filters --i-know-what-im-doing / --skip-age-check out of "$@" before parse_args sees them
|
||||||
|
# (both are cleanup-script-specific, not part of the shared arg parser).
|
||||||
|
# Sets I_KNOW, SKIP_AGE_CHECK, FILTERED_ARGS — call parse_args "${FILTERED_ARGS[@]}" after.
|
||||||
|
parse_destructive_flags() {
|
||||||
|
I_KNOW=false
|
||||||
|
SKIP_AGE_CHECK=false
|
||||||
|
FILTERED_ARGS=()
|
||||||
|
local arg
|
||||||
|
for arg in "$@"; do
|
||||||
|
case "$arg" in
|
||||||
|
--i-know-what-im-doing) I_KNOW=true ;;
|
||||||
|
--skip-age-check) SKIP_AGE_CHECK=true ;;
|
||||||
|
*) FILTERED_ARGS+=("$arg") ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
# Prints the nuclear-mode banner and sleeps 10s (Ctrl+C window) when both destructive flags
|
||||||
|
# are set and this isn't a dry run. Call after parse_args. No-op otherwise.
|
||||||
|
nuclear_mode_warning() {
|
||||||
|
if [[ "$I_KNOW" == true ]] && [[ "$SKIP_AGE_CHECK" == true ]] && [[ "$DRY_RUN" != true ]]; then
|
||||||
|
echo ""
|
||||||
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||||
|
echo "⚠️ WARNING — NUCLEAR MODE ACTIVE"
|
||||||
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||||
|
echo " Flags: --i-know-what-im-doing --skip-age-check"
|
||||||
|
echo " Age check: BYPASSED — deletes on first pass"
|
||||||
|
echo " Size threshold: BYPASSED — no GB limit"
|
||||||
|
echo " Data recovery: NOT POSSIBLE after deletion"
|
||||||
|
echo ""
|
||||||
|
echo " Review --dry-run output before proceeding."
|
||||||
|
echo " You have 10 seconds to cancel (Ctrl+C)..."
|
||||||
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||||
|
sleep 10
|
||||||
|
echo " Proceeding..."
|
||||||
|
echo ""
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# Safety Layer 7 — abort if total deletion size exceeds max_gb and --i-know-what-im-doing
|
||||||
|
# wasn't passed. Sets TOTAL_HUMAN. Exits 1 (with notify) on failure; warns and continues
|
||||||
|
# if I_KNOW is true (set by parse_destructive_flags).
|
||||||
|
# Usage: check_delete_size_threshold "$TOTAL_DELETE_BYTES" "$LIDARR_MAX_DELETE_GB" "Lidarr Cleanup"
|
||||||
|
check_delete_size_threshold() {
|
||||||
|
local total_bytes="$1" max_gb="$2" notify_label="$3"
|
||||||
|
local max_bytes
|
||||||
|
max_bytes=$(awk "BEGIN {printf \"%d\", $max_gb * 1073741824}")
|
||||||
|
if [[ "$total_bytes" -gt "$max_bytes" ]]; then
|
||||||
|
TOTAL_HUMAN=$(awk "BEGIN {printf \"%.1fGB\", $total_bytes / 1073741824}")
|
||||||
|
if [[ "$I_KNOW" != true ]]; then
|
||||||
|
echo ""
|
||||||
|
error "Deletion would exceed ${max_gb}GB — $TOTAL_HUMAN would be deleted"
|
||||||
|
error "Review ORPHAN lines above carefully before proceeding"
|
||||||
|
error "Rerun with: --i-know-what-im-doing"
|
||||||
|
error "To also bypass age check: add --skip-age-check"
|
||||||
|
notify "$notify_label halted on $(hostname) — ${TOTAL_HUMAN} requires --i-know-what-im-doing" \
|
||||||
|
"$notify_label" "warning"
|
||||||
|
exit 1
|
||||||
|
else
|
||||||
|
warn "OVERRIDE — deletion is $TOTAL_HUMAN — proceeding with --i-know-what-im-doing"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# True if filepath's extension (case-insensitive) matches one of the given extensions.
|
||||||
|
# Usage: has_extension "$filepath" "${LIDARR_EXTENSIONS[@]}"
|
||||||
|
has_extension() {
|
||||||
|
local filepath="$1"; shift
|
||||||
|
local ext="${filepath##*.}"
|
||||||
|
ext="${ext,,}"
|
||||||
|
local valid_ext
|
||||||
|
for valid_ext in "$@"; do
|
||||||
|
[[ "$ext" == "$valid_ext" ]] && return 0
|
||||||
|
done
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
# True if filepath's basename matches one of the given glob patterns.
|
||||||
|
# Usage: matches_pattern_list "$filepath" "${LIDARR_PROTECTED_PATTERNS[@]}"
|
||||||
|
matches_pattern_list() {
|
||||||
|
local filepath="$1"; shift
|
||||||
|
local filename
|
||||||
|
filename=$(basename "$filepath")
|
||||||
|
local pattern
|
||||||
|
for pattern in "$@"; do
|
||||||
|
# shellcheck disable=SC2254
|
||||||
|
case "$filename" in
|
||||||
|
$pattern) return 0 ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
# Builds the global ARR_PATH_MAP associative array (container path → host path) from
|
||||||
|
# ${MY_ID}_<ARR_TYPE>_PATH_MAP, for translate_path() to use. Usage: build_arr_path_map "LIDARR"
|
||||||
|
build_arr_path_map() {
|
||||||
|
local arr_type="$1"
|
||||||
|
declare -gA ARR_PATH_MAP=()
|
||||||
|
local _bapm_var="${MY_ID}_${arr_type}_PATH_MAP"
|
||||||
|
eval "for key in \"\${!${_bapm_var}[@]}\"; do
|
||||||
|
ARR_PATH_MAP[\"\$key\"]=\"\${${_bapm_var}[\$key]}\"
|
||||||
|
done"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Generic arr API GET wrapper with HTTP status check.
|
||||||
|
# Usage: arr_api "$RADARR_URL" "$RADARR_API_KEY" "v3" "movie" "Radarr"
|
||||||
|
arr_api() {
|
||||||
|
local base_url="$1" api_key="$2" api_version="$3" endpoint="$4" label="${5:-Arr}"
|
||||||
|
local response http_code body
|
||||||
|
|
||||||
|
response=$(curl -sf \
|
||||||
|
--max-time 30 \
|
||||||
|
-H "X-Api-Key: $api_key" \
|
||||||
|
-w "\n%{http_code}" \
|
||||||
|
"${base_url}/api/${api_version}/${endpoint}" 2>/dev/null)
|
||||||
|
|
||||||
|
http_code=$(echo "$response" | tail -1)
|
||||||
|
body=$(echo "$response" | head -n -1)
|
||||||
|
|
||||||
|
if [[ "$http_code" != "200" ]]; then
|
||||||
|
error "$label API HTTP $http_code for: $endpoint"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
echo "$body"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Triggers an arr "command" endpoint with the given JSON payload, then polls every 10s until
|
||||||
|
# completed/failed/timeout, logging progress every 60s. Best-effort pre-flight — never treats
|
||||||
|
# an unreachable/timed-out scan as fatal, matching original per-script behavior.
|
||||||
|
# Usage: trigger_and_await_command "$LIDARR_URL" "$LIDARR_API_KEY" "v1" \
|
||||||
|
# '{"name": "DownloadedAlbumsScan"}' "${LIDARR_IMPORT_SCAN_TIMEOUT:-600}"
|
||||||
|
trigger_and_await_command() {
|
||||||
|
local base_url="$1" api_key="$2" api_version="$3" payload="$4" poll_timeout="${5:-600}"
|
||||||
|
|
||||||
|
local scan_response scan_cmd_id
|
||||||
|
scan_response=$(curl -sf --max-time 30 -X POST \
|
||||||
|
-H "X-Api-Key: $api_key" \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-d "$payload" \
|
||||||
|
"${base_url}/api/${api_version}/command" 2>/dev/null)
|
||||||
|
|
||||||
|
scan_cmd_id=$(echo "$scan_response" | jq -r '.id // empty' 2>/dev/null)
|
||||||
|
|
||||||
|
if [[ -z "$scan_cmd_id" ]]; then
|
||||||
|
warn "Could not trigger import scan — proceeding without pre-flight"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
info "Import scan queued (command ID: $scan_cmd_id) — waiting for completion..."
|
||||||
|
local polled=0 scan_status
|
||||||
|
while [[ "$polled" -lt "$poll_timeout" ]]; do
|
||||||
|
scan_status=$(curl -sf --max-time 10 \
|
||||||
|
-H "X-Api-Key: $api_key" \
|
||||||
|
"${base_url}/api/${api_version}/command/${scan_cmd_id}" 2>/dev/null | \
|
||||||
|
jq -r '.status // empty' 2>/dev/null)
|
||||||
|
case "$scan_status" in
|
||||||
|
completed) info "Import scan complete ✅"; return 0 ;;
|
||||||
|
failed) warn "Import scan reported failed — proceeding anyway"; return 0 ;;
|
||||||
|
esac
|
||||||
|
sleep 10
|
||||||
|
(( polled += 10 ))
|
||||||
|
[[ $(( polled % 60 )) -eq 0 ]] && log " Still scanning... (${polled}s elapsed)"
|
||||||
|
done
|
||||||
|
warn "Import scan timed out after ${poll_timeout}s — proceeding anyway"
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ── ARR VERSION CHECK ─────────────────────────────────────────────────────────────────────────
|
# ── ARR VERSION CHECK ─────────────────────────────────────────────────────────────────────────
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
@@ -1692,6 +2244,48 @@ check_arr_version() {
|
|||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# ==============================================================================================
|
||||||
|
# ── TMDB DISCOVERY SCORING ────────────────────────────────────────────────────────────────────
|
||||||
|
# ==============================================================================================
|
||||||
|
# Shared by playback_aware_radarr_discovery.sh and playback_aware_sonarr_discovery.sh — stage 2
|
||||||
|
# candidate scoring against TMDB rating/vote/seed-breadth data. Byte-for-byte identical between
|
||||||
|
# the two before this consolidation.
|
||||||
|
|
||||||
|
# Formats a TMDB vote_average×10 integer back to one decimal place (e.g. 78 → 7.8).
|
||||||
|
_fmt_rating() { local v="${1:-0}"; echo "${v::-1}.${v: -1}" 2>/dev/null || echo "$v"; }
|
||||||
|
|
||||||
|
# Stage 2: TMDB vote_average × 10 as integer (0-40)
|
||||||
|
_rating_score_s2() {
|
||||||
|
local v="$1"
|
||||||
|
if (( v >= 80 )); then echo 40
|
||||||
|
elif (( v >= 75 )); then echo 32
|
||||||
|
elif (( v >= 70 )); then echo 25
|
||||||
|
elif (( v >= 65 )); then echo 18
|
||||||
|
elif (( v >= 60 )); then echo 12
|
||||||
|
else echo 5
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# Stage 2: vote count (0-20)
|
||||||
|
_votes_score() {
|
||||||
|
local c="$1"
|
||||||
|
if (( c >= 10000 )); then echo 20
|
||||||
|
elif (( c >= 5000 )); then echo 15
|
||||||
|
elif (( c >= 1000 )); then echo 10
|
||||||
|
elif (( c >= 200 )); then echo 5
|
||||||
|
else echo 2
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# Stage 2: seed breadth (0-40)
|
||||||
|
_breadth_score() {
|
||||||
|
local seeds="$1"
|
||||||
|
if (( seeds >= 3 )); then echo 40
|
||||||
|
elif (( seeds == 2 )); then echo 25
|
||||||
|
else echo 10
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ── EMBY LIBRARY SCAN ─────────────────────────────────────────────────────────────────────────
|
# ── EMBY LIBRARY SCAN ─────────────────────────────────────────────────────────────────────────
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
@@ -1911,54 +2505,6 @@ check_remote_docker_daemon() {
|
|||||||
return 1
|
return 1
|
||||||
}
|
}
|
||||||
|
|
||||||
# ==============================================================================================
|
|
||||||
# ── UNRAID COMMAND VALIDATION ─────────────────────────────────────────────────────────────────
|
|
||||||
# ==============================================================================================
|
|
||||||
# Verifies a unRAID-specific command exists and produces expected output before use.
|
|
||||||
# Protects against commands moving, changing format, or disappearing after upgrades.
|
|
||||||
#
|
|
||||||
# If validation fails — notifies and returns 1. Caller exits its own section only.
|
|
||||||
# Other scripts that pass validation are unaffected.
|
|
||||||
#
|
|
||||||
# Usage:
|
|
||||||
# validate_unraid_cmd # "/usr/local/emhttp/plugins/dynamix/scripts/notify" # "--help" # "Usage" # "unRAID notify script"
|
|
||||||
#
|
|
||||||
# Arguments:
|
|
||||||
# $1 = full path to command
|
|
||||||
# $2 = test argument to pass (use "" for no argument)
|
|
||||||
# $3 = expected string in output (use "" to skip output check)
|
|
||||||
# $4 = human readable name for error messages
|
|
||||||
|
|
||||||
validate_unraid_cmd() {
|
|
||||||
local cmd_path="$1"
|
|
||||||
local test_arg="$2"
|
|
||||||
local expected_output="$3"
|
|
||||||
local cmd_name="${4:-$1}"
|
|
||||||
|
|
||||||
# Check command exists and is executable
|
|
||||||
if [[ ! -x "$cmd_path" ]]; then
|
|
||||||
error "unRAID command not found or not executable: $cmd_path"
|
|
||||||
error "$cmd_name may have moved or been removed — check after recent unRAID upgrade"
|
|
||||||
notify "$cmd_name not found on $(hostname) at $cmd_path — check after unRAID upgrade" "Command Validation" "warning"
|
|
||||||
return 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Check output matches expected pattern if provided
|
|
||||||
if [[ -n "$expected_output" ]]; then
|
|
||||||
local actual_output
|
|
||||||
actual_output=$(timeout 10 "$cmd_path" $test_arg 2>&1 || true)
|
|
||||||
if ! echo "$actual_output" | grep -q "$expected_output"; then
|
|
||||||
error "$cmd_name output format changed — expected '$expected_output' not found"
|
|
||||||
error "Command may have changed after unRAID upgrade — review $cmd_path"
|
|
||||||
notify "$cmd_name output format changed on $(hostname) — may need script update after unRAID upgrade" "Command Validation" "warning"
|
|
||||||
return 1
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
log "$cmd_name validated: $cmd_path ✅"
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ── STATUS DISPLAY ────────────────────────────────────────────────────────────────────────────
|
# ── STATUS DISPLAY ────────────────────────────────────────────────────────────────────────────
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
|
|||||||
+2
-1
@@ -135,7 +135,8 @@
|
|||||||
CONF_DIR="${SCRIPTS_DIR}/Configurations"
|
CONF_DIR="${SCRIPTS_DIR}/Configurations"
|
||||||
LOG_DIR="/var/log/varaverk"
|
LOG_DIR="/var/log/varaverk"
|
||||||
VV_CACHE_DIR="/tmp/vv_cache"
|
VV_CACHE_DIR="/tmp/vv_cache"
|
||||||
export CONF_DIR LOG_DIR VV_CACHE_DIR
|
CONF_RAM_CACHE_DIR="/tmp/.cache/vv/d" # tmpfs — cleared every reboot, repopulated by conf_sync.sh
|
||||||
|
export CONF_DIR LOG_DIR VV_CACHE_DIR CONF_RAM_CACHE_DIR
|
||||||
|
|
||||||
# ━━━ Cleanup ━━━
|
# ━━━ Cleanup ━━━
|
||||||
unset _conf _host_confs_loaded _adapter LOAD_CONFIG_DIR
|
unset _conf _host_confs_loaded _adapter LOAD_CONFIG_DIR
|
||||||
Reference in New Issue
Block a user