FailedProxy/Varaverk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

system watchdog update to minitor for unheathy pids

Browse Source
This commit is contained in:
FailedProxy
2026-04-24 20:57:45 -04:00
parent 5f940a7181
commit 63101fdb83
2 changed files with 452 additions and 886 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common.sh
+34 -21
View File
@@ -697,10 +697,10 @@ _release_on_exit() {
# acquire_lock — acquire exclusive lock for this script
# Mode: strict (default) — exit immediately if locked
# wait — wait LOCK_WAIT_TIMEOUT seconds then exit
# continuous — for long-running scripts: skip gracefully if healthy,
# clear and restart if dead/stuck
# Stale lock: if PID in lock file is dead → clear and acquire
# Age warning: if lock older than LOCK_WARN_AGE → warn
# continuous — for long-running scripts: skip gracefully if healthy
# Lock file stores PID:scriptname — prevents PID reuse false positives
# Stale lock: if PID dead OR PID belongs to different process → clear and acquire
# Age warning: if lock older than LOCK_WARN_AGE → warn (skipped for continuous)
# -----------------------------------------------------------------------------------------------
acquire_lock() {
local mode="${1:-strict}"
@@ -713,15 +713,23 @@ acquire_lock() {
# Check for existing lock
if [[ -f "$lockfile" ]]; then
local existing_pid
existing_pid=$(cat "$lockfile" 2>/dev/null)
local lock_content existing_pid locked_name
lock_content=$(cat "$lockfile" 2>/dev/null)
existing_pid="${lock_content%%:*}"
locked_name="${lock_content##*:}"
# Stale lock detection — PID no longer running
if [[ -n "$existing_pid" ]] && ! kill -0 "$existing_pid" 2>/dev/null; then
# Stale lock — PID dead
if [[ -z "$existing_pid" ]] || ! kill -0 "$existing_pid" 2>/dev/null; then
warn "Stale lock detected for $script_name (PID $existing_pid gone) — clearing"
rm -f "$lockfile"
# PID reuse — PID alive but belongs to a different process
elif [[ "$locked_name" != "$script_name" ]]; then
warn "Lock PID $existing_pid reused by different process ($locked_name ≠ $script_name) — clearing stale lock"
rm -f "$lockfile"
else
# Lock is active — check age (skip warning for continuous scripts)
# Lock is genuinely active — check age (skip warning for continuous scripts)
local lock_age
lock_age=$(( $(date +%s) - $(stat -c %Y "$lockfile" 2>/dev/null || echo 0) ))
if [[ "$lock_age" -gt "$LOCK_WARN_AGE" ]] && [[ "$mode" != "continuous" ]]; then
@@ -729,8 +737,7 @@ acquire_lock() {
fi
if [[ "$mode" == "continuous" ]]; then
# Continuous scripts (watchdogs, failover) — healthy instance = always skip
# PID is alive and responding — this is correct behavior, not stuck
# Continuous scripts — healthy instance = always skip gracefully
log "$ICON_SKIP $script_name already running healthy (PID $existing_pid) — skipping"
exit 0
elif [[ "$mode" == "wait" ]]; then
@@ -739,12 +746,17 @@ acquire_lock() {
while [[ -f "$lockfile" ]] && [[ "$waited" -lt "$LOCK_WAIT_TIMEOUT" ]]; do
sleep 1
((waited++))
# Re-check for stale
existing_pid=$(cat "$lockfile" 2>/dev/null)
if [[ -n "$existing_pid" ]] && ! kill -0 "$existing_pid" 2>/dev/null; then
lock_content=$(cat "$lockfile" 2>/dev/null)
existing_pid="${lock_content%%:*}"
locked_name="${lock_content##*:}"
if [[ -z "$existing_pid" ]] || ! kill -0 "$existing_pid" 2>/dev/null; then
warn "Lock became stale while waiting — clearing"
rm -f "$lockfile"
break
elif [[ "$locked_name" != "$script_name" ]]; then
warn "Lock PID reused while waiting — clearing"
rm -f "$lockfile"
break
fi
done
if [[ -f "$lockfile" ]]; then
@@ -753,7 +765,6 @@ acquire_lock() {
fi
else
error "Another instance of $script_name is already running (PID $existing_pid) — exiting"
# Notify for critical scripts that should rarely overlap
case "$script_name" in
failover|transcode_management|media_management|daily_sync_maintenance|system_watchdog)
notify "$script_name lock collision on $(hostname) — concurrent instance detected" "$script_name" "warning"
@@ -764,8 +775,8 @@ acquire_lock() {
fi
fi
# Acquire lock
echo $$ > "$lockfile"
# Acquire lock — store PID:scriptname to prevent PID reuse false positives
echo "$$:$script_name" > "$lockfile"
# Register EXIT trap to always release lock
trap "_release_on_exit '$lockfile'" EXIT
@@ -786,9 +797,11 @@ acquire_rsync_lock() {
# Per-profile lock — same profile cannot run twice
if [[ -f "$profile_lock" ]]; then
local existing_pid
existing_pid=$(cat "$profile_lock" 2>/dev/null)
if [[ -n "$existing_pid" ]] && kill -0 "$existing_pid" 2>/dev/null; then
local lock_content existing_pid locked_name
lock_content=$(cat "$profile_lock" 2>/dev/null)
existing_pid="${lock_content%%:*}"
locked_name="${lock_content##*:}"
if [[ -n "$existing_pid" ]] && kill -0 "$existing_pid" 2>/dev/null && [[ "$locked_name" == "rsync_${profile}" ]]; then
error "rsync profile '$profile' is already running (PID $existing_pid) — exiting"
exit 1
else
@@ -823,7 +836,7 @@ acquire_rsync_lock() {
fi
# Acquire profile lock and increment counter
echo $$ > "$profile_lock"
echo "$$:rsync_${profile}" > "$profile_lock"
echo $(( current_count + 1 )) > "$RSYNC_COUNT_FILE"
# Register EXIT trap