fixed lock logic. if a continuious app hits a lock it skips to the next instead of error

This commit is contained in:
2026-04-24 16:26:35 -04:00
parent 555fbb1b23
commit 6df0bafc09
4 changed files with 347 additions and 1634 deletions
+18 -3
View File
@@ -6,7 +6,7 @@
# -----------------------------------------------------------------------------------------------
# Changelog:
# v1.0 — Initial stable framework
# v1.1 — format_duration moved here from daily_sync.sh for shared use
# v1.1 — format_duration moved here from daily_sync_maintenance.sh for shared use
# SSH_KEY collision resolved — gitea key renamed GITEA_SSH_KEY in Master.conf
# Version and changelog tracking added
# v1.2 — Consistent function header comment blocks across all functions
@@ -92,6 +92,7 @@ ICON_STOPPED="🔴" # container confirmed stopped
ICON_START="▶️" # start command being issued
ICON_STARTED="💚" # container confirmed started
ICON_RUNNING="🟢" # container already running when checked
ICON_SKIP="⏭️" # skipping — already running healthy instance
ICON_NOT_RUNNING="⭕" # container already stopped when checked
# Transfer
@@ -695,6 +696,8 @@ _release_on_exit() {
# acquire_lock — acquire exclusive lock for this script
# Mode: strict (default) — exit immediately if locked
# wait — wait LOCK_WAIT_TIMEOUT seconds then exit
# continuous — for long-running scripts: skip gracefully if healthy,
# clear and restart if dead/stuck
# Stale lock: if PID in lock file is dead → clear and acquire
# Age warning: if lock older than LOCK_WARN_AGE → warn
# -----------------------------------------------------------------------------------------------
@@ -724,7 +727,19 @@ acquire_lock() {
warn "$script_name has been running for ${lock_age}s — may be stuck (PID $existing_pid)"
fi
if [[ "$mode" == "wait" ]]; then
if [[ "$mode" == "continuous" ]]; then
# Continuous scripts (watchdogs, failover) — healthy instance = skip gracefully
# Only force-restart if PID is stuck/unresponsive beyond LOCK_WARN_AGE
if [[ "$lock_age" -gt "$LOCK_WARN_AGE" ]]; then
warn "$script_name appears stuck (running ${lock_age}s) — clearing and restarting"
kill "$existing_pid" 2>/dev/null
sleep 2
rm -f "$lockfile"
else
log "$ICON_SKIP $script_name already running healthy (PID $existing_pid) — skipping"
exit 0
fi
elif [[ "$mode" == "wait" ]]; then
info "Another instance of $script_name is running — waiting up to ${LOCK_WAIT_TIMEOUT}s"
local waited=0
while [[ -f "$lockfile" ]] && [[ "$waited" -lt "$LOCK_WAIT_TIMEOUT" ]]; do
@@ -746,7 +761,7 @@ acquire_lock() {
error "Another instance of $script_name is already running (PID $existing_pid) — exiting"
# Notify for critical scripts that should rarely overlap
case "$script_name" in
failover|transcode_management|media_management|daily_sync|system_watchdog)
failover|transcode_management|media_management|daily_sync_maintenance|system_watchdog)
notify "$script_name lock collision on $(hostname) — concurrent instance detected" "$script_name" "warning"
;;
esac