diff --git a/Docker_Essentials/docker_daily_restart.sh b/Docker_Essentials/docker_daily_restart.sh index bc6e289..786683f 100644 --- a/Docker_Essentials/docker_daily_restart.sh +++ b/Docker_Essentials/docker_daily_restart.sh @@ -29,6 +29,8 @@ fi success "Running as root" +acquire_lock + if ! command -v docker &>/dev/null; then error "Docker command not found — check PATH or Docker installation" notify "Docker daily restart failed — Docker not found on $(hostname)" "Docker Daily Restart" "warning" diff --git a/Docker_Essentials/docker_watchdog.sh b/Docker_Essentials/docker_watchdog.sh index b5acb52..229ab9e 100644 --- a/Docker_Essentials/docker_watchdog.sh +++ b/Docker_Essentials/docker_watchdog.sh @@ -53,6 +53,8 @@ fi success "Running as root" +acquire_lock + # Select correct per-host watchdog lists detect_hosts diff --git a/Docker_Essentials/docker_weekly_restart.sh b/Docker_Essentials/docker_weekly_restart.sh index bd89b3b..aa4f881 100644 --- a/Docker_Essentials/docker_weekly_restart.sh +++ b/Docker_Essentials/docker_weekly_restart.sh @@ -29,6 +29,8 @@ fi success "Running as root" +acquire_lock + if ! command -v docker &>/dev/null; then error "Docker command not found — check PATH or Docker installation" notify "Docker weekly restart failed — Docker not found on $(hostname)" "Docker Weekly Restart" "warning" diff --git a/Failover/failover.sh b/Failover/failover.sh index 65a75e0..ca85e4f 100644 --- a/Failover/failover.sh +++ b/Failover/failover.sh @@ -92,6 +92,9 @@ if [[ "$EUID" -ne 0 ]]; then fi success "Running as root" + +acquire_lock # failover.sh — strict single instance, notifies on collision + detect_hosts resolve_remote_ip diff --git a/Media/lidarr_cleanup.sh b/Media/lidarr_cleanup.sh index 6c5b851..e35cca4 100644 --- a/Media/lidarr_cleanup.sh +++ b/Media/lidarr_cleanup.sh @@ -81,6 +81,8 @@ if [[ ! -d "$LIDARR_MUSIC_ROOT" ]]; then exit 1 fi +acquire_lock "wait" + success "Config validated" # ----------------------------------------------------------------------------------------------- @@ -156,6 +158,11 @@ is_protected_file() { echo "" echo "━━━ $ICON_SYNC Fetching Lidarr Tracked Files ━━━" +check_api "$LIDARR_URL" "Lidarr" || { + notify "Lidarr cleanup aborted on $(hostname) — API unreachable" "Lidarr Cleanup" "warning" + exit 1 +} + info "Querying Lidarr API: $LIDARR_URL" TRACKFILE_RESPONSE=$(lidarr_api "trackfile") || { diff --git a/Media/media_cleaner.sh b/Media/media_cleaner.sh index 42901bf..e3afbd5 100644 --- a/Media/media_cleaner.sh +++ b/Media/media_cleaner.sh @@ -47,6 +47,8 @@ fi success "Running as root" +acquire_lock "wait" + if ! command -v find >/dev/null 2>&1; then error "find command not found — check findutils installation" exit 1 diff --git a/Media/media_shares_permissions.sh b/Media/media_shares_permissions.sh index cac5063..86b2a50 100644 --- a/Media/media_shares_permissions.sh +++ b/Media/media_shares_permissions.sh @@ -43,6 +43,8 @@ fi [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made" +acquire_lock "wait" + # ----------------------------------------------------------------------------------------------- # ━━━ $ICON_PERMS Media Permissions ━━━ # ----------------------------------------------------------------------------------------------- diff --git a/Media/radarr_cleanup.sh b/Media/radarr_cleanup.sh index 323cf79..ac70a6b 100644 --- a/Media/radarr_cleanup.sh +++ b/Media/radarr_cleanup.sh @@ -82,6 +82,8 @@ if [[ ! -d "$RADARR_MOVIES_ROOT" ]]; then exit 1 fi +acquire_lock "wait" + success "Config validated" # ----------------------------------------------------------------------------------------------- @@ -154,6 +156,11 @@ is_protected_file() { echo "" echo "━━━ $ICON_SYNC Fetching Radarr Tracked Files ━━━" +check_api "$RADARR_URL" "Radarr" || { + notify "Radarr cleanup aborted on $(hostname) — API unreachable" "Radarr Cleanup" "warning" + exit 1 +} + info "Querying Radarr API: $RADARR_URL" MOVIEFILE_RESPONSE=$(radarr_api "moviefile") || { diff --git a/Media/sonarr_cleanup.sh b/Media/sonarr_cleanup.sh index eb48a9f..aaed9be 100644 --- a/Media/sonarr_cleanup.sh +++ b/Media/sonarr_cleanup.sh @@ -82,6 +82,8 @@ if [[ ! -d "$SONARR_TV_ROOT" ]]; then exit 1 fi +acquire_lock "wait" + success "Config validated" # ----------------------------------------------------------------------------------------------- @@ -154,6 +156,11 @@ is_protected_file() { echo "" echo "━━━ $ICON_SYNC Fetching Sonarr Tracked Files ━━━" +check_api "$SONARR_URL" "Sonarr" || { + notify "Sonarr cleanup aborted on $(hostname) — API unreachable" "Sonarr Cleanup" "warning" + exit 1 +} + info "Querying Sonarr API: $SONARR_URL" EPISODEFILE_RESPONSE=$(sonarr_api "episodefile") || { diff --git a/Orchestrators/daily_sync.sh b/Orchestrators/daily_sync.sh index 6469418..bc0ce17 100644 --- a/Orchestrators/daily_sync.sh +++ b/Orchestrators/daily_sync.sh @@ -32,6 +32,8 @@ echo "━━━ $ICON_GEAR Setup ━━━" detect_hosts resolve_remote_ip +acquire_lock + # ----------------------------------------------------------------------------------------------- # ━━━ $ICON_SHIELD Pre-flight Checks ━━━ # ----------------------------------------------------------------------------------------------- diff --git a/Orchestrators/media_management.sh b/Orchestrators/media_management.sh index 3f0159a..eefd606 100644 --- a/Orchestrators/media_management.sh +++ b/Orchestrators/media_management.sh @@ -35,6 +35,8 @@ if [[ "$EUID" -ne 0 ]]; then fi success "Running as root" + +acquire_lock [[ "$DRY_RUN" == true ]] && warn "DRY RUN — passing --dry-run to all child scripts" # Validate job list diff --git a/Orchestrators/transcode_management.sh b/Orchestrators/transcode_management.sh index f916dc0..fd6badd 100644 --- a/Orchestrators/transcode_management.sh +++ b/Orchestrators/transcode_management.sh @@ -50,6 +50,8 @@ fi [[ "$DRY_RUN" == true ]] && warn "DRY RUN — passing to child scripts" +acquire_lock + # ----------------------------------------------------------------------------------------------- # HELPERS # ----------------------------------------------------------------------------------------------- diff --git a/Rsync/rsync.sh b/Rsync/rsync.sh index 8ca6588..e78c6c0 100644 --- a/Rsync/rsync.sh +++ b/Rsync/rsync.sh @@ -54,6 +54,9 @@ echo "" PROFILE_NAME=$(basename "$DIRECTORY" | tr '[:upper:]' '[:lower:]') info "$ICON_GEAR Loading profile: $PROFILE_NAME" +# Acquire per-profile lock and check global concurrent limit +acquire_rsync_lock "$PROFILE_NAME" + # Scalar overrides BW_LIMIT=${PROFILE_BW_LIMIT[$PROFILE_NAME]:-$BW_LIMIT} RETRY_COUNT=${PROFILE_RETRY_COUNT[$PROFILE_NAME]:-$RETRY_COUNT} diff --git a/Tools/emby_database_repair.sh b/Tools/emby_database_repair.sh index 104edeb..5619970 100644 --- a/Tools/emby_database_repair.sh +++ b/Tools/emby_database_repair.sh @@ -50,6 +50,8 @@ fi success "Running as root" +acquire_lock + if ! command -v sqlite3 >/dev/null 2>&1; then error "sqlite3 not found — install sqlite package" exit 1 diff --git a/Tools/zfs_pool_scrub.sh b/Tools/zfs_pool_scrub.sh index 75c0f4d..20cb172 100644 --- a/Tools/zfs_pool_scrub.sh +++ b/Tools/zfs_pool_scrub.sh @@ -42,6 +42,8 @@ fi success "Running as root" +acquire_lock + if ! command -v zpool >/dev/null 2>&1; then error "ZFS not available on this system" exit 1 diff --git a/Transcodes/transcode_cleanup.sh b/Transcodes/transcode_cleanup.sh index 07fa456..a30a043 100644 --- a/Transcodes/transcode_cleanup.sh +++ b/Transcodes/transcode_cleanup.sh @@ -42,6 +42,8 @@ fi success "Running as root" +acquire_lock "wait" + if ! command -v lsof >/dev/null 2>&1; then warn "lsof not available — active file check will be skipped, all aged files will be eligible" LSOF_AVAILABLE=false diff --git a/Transcodes/transcode_manager.sh b/Transcodes/transcode_manager.sh index 88afd4a..1f60c98 100644 --- a/Transcodes/transcode_manager.sh +++ b/Transcodes/transcode_manager.sh @@ -47,6 +47,8 @@ fi success "Running as root" +acquire_lock "wait" + # Select correct Emby instance for session display detect_hosts diff --git a/common.sh b/common.sh index 13a28de..08ccebc 100644 --- a/common.sh +++ b/common.sh @@ -21,38 +21,12 @@ # v1.5 — Full icon set expanded — each operation and state has its own distinct icon # All function output updated to use correct icon per context # Icons grouped and commented by category for clarity -# v1.6 — ICON_CONTAINERS added — 📦 anchors all container sections for visual consistency -# ICON_NOT_RUNNING changed to ⭕ — distinct from ICON_STOPPED 🔴 -# Section dividers updated from --- to ━━━ for cleaner log readability -# Summary passed/failed lines use ICON_SUCCESS and ICON_ERROR consistently -# v1.7 — ICON_MOVER added for mover operations -# ICON_CONTAINERS replaces ICON_DOCKER for docker/container operations -# validate_int added — reusable integer validation for any script -# v1.8 — ICON_PHP added for PHP-FPM operations -# v1.9 — ICON_REBOOT added for server reboot operations -# v2.0 — ICON_PLUGIN added for User Scripts plugin operations -# v2.1 — ICON_ZFS and ICON_MEM added for ZFS and memory diagnostics -# Diagnostics icon group added to icon block -# v2.2 — ICON_WATCHDOG added for Docker watchdog monitoring operations -# v2.3 — ICON_NOTIFY added for notification operations -# notify() added — shared notification function supporting unRAID native and Discord -# NOTIFY_UNRAID and DISCORD_WEBHOOK configured in Master.conf -# v2.4 — ICON_CLEAN, ICON_TRASH added for media cleaner operations -# ICON_PERMS, ICON_UNLOCKED added for media permissions operations -# ICON_REBOOT_SMART added for smart conditional reboot -# v2.5 — ICON_RAM added for ramdisk operations -# ICON_LINK added for symlink state and management -# Transcode scripts group added to ecosystem -# v2.6 — ICON_FAILOVER added for failover operations -# check_local_array added — verifies local /mnt/user is mounted and healthy -# check_remote_array added — verifies remote /mnt/user is mounted and healthy -# check_remote_docker added — verifies remote Docker daemon is responding -# ping_remote added — non-fatal ping returning status for failover use -# ping_internet added — non-fatal external ping for failover use -# v2.7 — ICON_WEBGUI added for WebGUI watchdog operations -# ICON_DOCKER_NET added for Docker network connect operations -# v2.8 — ICON_CERT added for SSL certificate monitoring operations -# v2.9 — ICON_MONITOR added for monitoring section headers +# v3.0 — Script locking system added — prevents concurrent execution conflicts +# acquire_lock() — create PID lock file, register EXIT trap +# release_lock() — remove lock file on exit +# acquire_rsync_lock() — per-profile lock + global concurrent limit +# release_rsync_lock() — decrement global counter, remove profile lock +# check_api() — pre-flight API reachability check # ICON_SMART added for drive SMART health operations # ICON_BANDWIDTH added for bandwidth tracking operations # ICON_DIGEST added for health digest operations @@ -579,6 +553,208 @@ get_rsync_opts() { fi } +# ----------------------------------------------------------------------------------------------- +# SCRIPT LOCKING — v3.0 +# Prevents multiple instances of the same script running simultaneously. +# All lock files live in /tmp/unraid_locks/ — auto-cleared on reboot. +# +# Usage in scripts: +# acquire_lock — strict: exit immediately if already running +# acquire_lock "wait" — wait mode: wait briefly then exit if still locked +# acquire_rsync_lock "$profile" — per-profile + global concurrent limit +# +# Stale lock detection — if lock file exists but PID is dead, clears and proceeds. +# Lock age warning — if lock is older than expected, warns but does not override. +# EXIT trap registered automatically — lock always released on exit, crash, or kill. +# ----------------------------------------------------------------------------------------------- + +LOCK_DIR="/tmp/unraid_locks" +RSYNC_COUNT_FILE="$LOCK_DIR/rsync_active_count" +RSYNC_MAX_CONCURRENT=3 +LOCK_WARN_AGE=300 # seconds — warn if lock older than this (5min default) +LOCK_WAIT_TIMEOUT=30 # seconds — how long "wait" mode waits before giving up + +# Internal — script name used as lock identifier +_lock_name() { + basename "${BASH_SOURCE[1]:-$0}" .sh +} + +# Internal — lock file path for this script +_lock_file() { + echo "$LOCK_DIR/${1:-$(_lock_name)}.lock" +} + +# Internal — release lock on exit +_release_on_exit() { + local lockfile="$1" + [[ -f "$lockfile" ]] && rm -f "$lockfile" +} + +# ----------------------------------------------------------------------------------------------- +# acquire_lock — acquire exclusive lock for this script +# Mode: strict (default) — exit immediately if locked +# wait — wait LOCK_WAIT_TIMEOUT seconds then exit +# Stale lock: if PID in lock file is dead → clear and acquire +# Age warning: if lock older than LOCK_WARN_AGE → warn +# ----------------------------------------------------------------------------------------------- +acquire_lock() { + local mode="${1:-strict}" + local script_name + script_name=$(basename "${BASH_SOURCE[1]:-$0}" .sh) + local lockfile + lockfile="$(_lock_file "$script_name")" + + mkdir -p "$LOCK_DIR" + + # Check for existing lock + if [[ -f "$lockfile" ]]; then + local existing_pid + existing_pid=$(cat "$lockfile" 2>/dev/null) + + # Stale lock detection — PID no longer running + if [[ -n "$existing_pid" ]] && ! kill -0 "$existing_pid" 2>/dev/null; then + warn "Stale lock detected for $script_name (PID $existing_pid gone) — clearing" + rm -f "$lockfile" + else + # Lock is active — check age + local lock_age + lock_age=$(( $(date +%s) - $(stat -c %Y "$lockfile" 2>/dev/null || echo 0) )) + if [[ "$lock_age" -gt "$LOCK_WARN_AGE" ]]; then + warn "$script_name has been running for ${lock_age}s — may be stuck (PID $existing_pid)" + fi + + if [[ "$mode" == "wait" ]]; then + info "Another instance of $script_name is running — waiting up to ${LOCK_WAIT_TIMEOUT}s" + local waited=0 + while [[ -f "$lockfile" ]] && [[ "$waited" -lt "$LOCK_WAIT_TIMEOUT" ]]; do + sleep 1 + ((waited++)) + # Re-check for stale + existing_pid=$(cat "$lockfile" 2>/dev/null) + if [[ -n "$existing_pid" ]] && ! kill -0 "$existing_pid" 2>/dev/null; then + warn "Lock became stale while waiting — clearing" + rm -f "$lockfile" + break + fi + done + if [[ -f "$lockfile" ]]; then + error "$script_name still locked after ${LOCK_WAIT_TIMEOUT}s — exiting" + exit 1 + fi + else + error "Another instance of $script_name is already running (PID $existing_pid) — exiting" + # Notify for critical scripts that should rarely overlap + case "$script_name" in + failover|transcode_management|media_management|daily_sync|system_watchdog) + notify "$script_name lock collision on $(hostname) — concurrent instance detected" "$script_name" "warning" + ;; + esac + exit 1 + fi + fi + fi + + # Acquire lock + echo $$ > "$lockfile" + + # Register EXIT trap to always release lock + trap "_release_on_exit '$lockfile'" EXIT + + log "Lock acquired: $script_name (PID $$)" +} + +# ----------------------------------------------------------------------------------------------- +# acquire_rsync_lock — per-profile lock + global concurrent limit +# Prevents same profile running twice and limits total concurrent rsync instances +# ----------------------------------------------------------------------------------------------- +acquire_rsync_lock() { + local profile="$1" + local profile_lock + profile_lock="$(_lock_file "rsync_${profile}")" + + mkdir -p "$LOCK_DIR" + + # Per-profile lock — same profile cannot run twice + if [[ -f "$profile_lock" ]]; then + local existing_pid + existing_pid=$(cat "$profile_lock" 2>/dev/null) + if [[ -n "$existing_pid" ]] && kill -0 "$existing_pid" 2>/dev/null; then + error "rsync profile '$profile' is already running (PID $existing_pid) — exiting" + exit 1 + else + warn "Stale rsync lock for profile '$profile' — clearing" + rm -f "$profile_lock" + fi + fi + + # Global concurrent limit + local current_count=0 + if [[ -f "$RSYNC_COUNT_FILE" ]]; then + current_count=$(cat "$RSYNC_COUNT_FILE" 2>/dev/null || echo 0) + # Validate count — clean up if stale + local actual_count=0 + for lf in "$LOCK_DIR"/rsync_*.lock; do + [[ -f "$lf" ]] || continue + local lpid + lpid=$(cat "$lf" 2>/dev/null) + kill -0 "$lpid" 2>/dev/null && ((actual_count++)) + done + if [[ "$actual_count" -ne "$current_count" ]]; then + log "rsync count corrected: $current_count → $actual_count" + current_count=$actual_count + echo "$current_count" > "$RSYNC_COUNT_FILE" + fi + fi + + if [[ "$current_count" -ge "$RSYNC_MAX_CONCURRENT" ]]; then + error "Maximum concurrent rsync limit ($RSYNC_MAX_CONCURRENT) reached — exiting" + info "Active rsync locks: $(ls "$LOCK_DIR"/rsync_*.lock 2>/dev/null | xargs -I{} basename {} .lock | tr '\n' ' ')" + exit 1 + fi + + # Acquire profile lock and increment counter + echo $$ > "$profile_lock" + echo $(( current_count + 1 )) > "$RSYNC_COUNT_FILE" + + # Register EXIT trap + trap "_release_rsync_on_exit '$profile_lock'" EXIT + + log "rsync lock acquired: profile '$profile' (PID $$, active: $(( current_count + 1 ))/$RSYNC_MAX_CONCURRENT)" +} + +# Internal — release rsync lock on exit +_release_rsync_on_exit() { + local profile_lock="$1" + [[ -f "$profile_lock" ]] && rm -f "$profile_lock" + # Decrement global counter + if [[ -f "$RSYNC_COUNT_FILE" ]]; then + local count + count=$(cat "$RSYNC_COUNT_FILE" 2>/dev/null || echo 1) + count=$(( count - 1 )) + [[ "$count" -lt 0 ]] && count=0 + echo "$count" > "$RSYNC_COUNT_FILE" + fi +} + +# ----------------------------------------------------------------------------------------------- +# check_api — pre-flight API reachability check +# Verifies API endpoint is reachable before attempting operations +# Usage: check_api "http://localhost:8989" "Sonarr" || exit 1 +# ----------------------------------------------------------------------------------------------- +check_api() { + local url="$1" + local service="${2:-API}" + local timeout="${3:-10}" + + if curl -sf --max-time "$timeout" "$url" >/dev/null 2>&1; then + log "$service API reachable: $url" + return 0 + else + error "$service API not reachable: $url" + return 1 + fi +} + # ----------------------------------------------------------------------------------------------- # STATUS DISPLAY # Prints current runtime configuration — triggered by --status flag. diff --git a/git_pull_execute.sh b/git_pull_execute.sh index 8be341c..74fb9da 100644 --- a/git_pull_execute.sh +++ b/git_pull_execute.sh @@ -35,6 +35,8 @@ fi success "Running as root" +acquire_lock + # Detect which server we're on and where Gitea is running detect_hosts diff --git a/unRAID_Essentials/system_watchdog.sh b/unRAID_Essentials/system_watchdog.sh index 17bf06e..0ae5044 100644 --- a/unRAID_Essentials/system_watchdog.sh +++ b/unRAID_Essentials/system_watchdog.sh @@ -55,6 +55,8 @@ fi success "Running as root" +acquire_lock + # Ensure state and persistent files exist touch "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null || { error "Cannot create state file: $SYS_WATCHDOG_STATE_FILE"