system watchdog update to minitor for unheathy pids

common.sh

@@ -697,10 +697,10 @@ _release_on_exit() {
 # acquire_lock — acquire exclusive lock for this script
 # Mode: strict (default) — exit immediately if locked
 #       wait             — wait LOCK_WAIT_TIMEOUT seconds then exit
-#       continuous       — for long-running scripts: skip gracefully if healthy,
+#       continuous       — for long-running scripts: skip gracefully if healthy
-#                          clear and restart if dead/stuck
+# Lock file stores PID:scriptname — prevents PID reuse false positives
-# Stale lock: if PID in lock file is dead → clear and acquire
+# Stale lock: if PID dead OR PID belongs to different process → clear and acquire
-# Age warning: if lock older than LOCK_WARN_AGE → warn
+# Age warning: if lock older than LOCK_WARN_AGE → warn (skipped for continuous)
 # -----------------------------------------------------------------------------------------------
 acquire_lock() {
     local mode="${1:-strict}"
@@ -713,15 +713,23 @@ acquire_lock() {
 
     # Check for existing lock
     if [[ -f "$lockfile" ]]; then
-        local existing_pid
+        local lock_content existing_pid locked_name
-        existing_pid=$(cat "$lockfile" 2>/dev/null)
+        lock_content=$(cat "$lockfile" 2>/dev/null)
+        existing_pid="${lock_content%%:*}"
+        locked_name="${lock_content##*:}"
 
-        # Stale lock detection — PID no longer running
+        # Stale lock — PID dead
-        if [[ -n "$existing_pid" ]] && ! kill -0 "$existing_pid" 2>/dev/null; then
+        if [[ -z "$existing_pid" ]] || ! kill -0 "$existing_pid" 2>/dev/null; then
             warn "Stale lock detected for $script_name (PID $existing_pid gone) — clearing"
             rm -f "$lockfile"
+
+        # PID reuse — PID alive but belongs to a different process
+        elif [[ "$locked_name" != "$script_name" ]]; then
+            warn "Lock PID $existing_pid reused by different process ($locked_name ≠ $script_name) — clearing stale lock"
+            rm -f "$lockfile"
+
         else
-            # Lock is active — check age (skip warning for continuous scripts)
+            # Lock is genuinely active — check age (skip warning for continuous scripts)
             local lock_age
             lock_age=$(( $(date +%s) - $(stat -c %Y "$lockfile" 2>/dev/null || echo 0) ))
             if [[ "$lock_age" -gt "$LOCK_WARN_AGE" ]] && [[ "$mode" != "continuous" ]]; then
@@ -729,8 +737,7 @@ acquire_lock() {
             fi
 
             if [[ "$mode" == "continuous" ]]; then
-                # Continuous scripts (watchdogs, failover) — healthy instance = always skip
+                # Continuous scripts — healthy instance = always skip gracefully
-                # PID is alive and responding — this is correct behavior, not stuck
                 log "$ICON_SKIP $script_name already running healthy (PID $existing_pid) — skipping"
                 exit 0
             elif [[ "$mode" == "wait" ]]; then
@@ -739,12 +746,17 @@ acquire_lock() {
                 while [[ -f "$lockfile" ]] && [[ "$waited" -lt "$LOCK_WAIT_TIMEOUT" ]]; do
                     sleep 1
                     ((waited++))
-                    # Re-check for stale
+                    lock_content=$(cat "$lockfile" 2>/dev/null)
-                    existing_pid=$(cat "$lockfile" 2>/dev/null)
+                    existing_pid="${lock_content%%:*}"
-                    if [[ -n "$existing_pid" ]] && ! kill -0 "$existing_pid" 2>/dev/null; then
+                    locked_name="${lock_content##*:}"
+                    if [[ -z "$existing_pid" ]] || ! kill -0 "$existing_pid" 2>/dev/null; then
                         warn "Lock became stale while waiting — clearing"
                         rm -f "$lockfile"
                         break
+                    elif [[ "$locked_name" != "$script_name" ]]; then
+                        warn "Lock PID reused while waiting — clearing"
+                        rm -f "$lockfile"
+                        break
                     fi
                 done
                 if [[ -f "$lockfile" ]]; then
@@ -753,7 +765,6 @@ acquire_lock() {
                 fi
             else
                 error "Another instance of $script_name is already running (PID $existing_pid) — exiting"
-                # Notify for critical scripts that should rarely overlap
                 case "$script_name" in
                     failover|transcode_management|media_management|daily_sync_maintenance|system_watchdog)
                         notify "$script_name lock collision on $(hostname) — concurrent instance detected" "$script_name" "warning"
@@ -764,8 +775,8 @@ acquire_lock() {
         fi
     fi
 
-    # Acquire lock
+    # Acquire lock — store PID:scriptname to prevent PID reuse false positives
-    echo $$ > "$lockfile"
+    echo "$$:$script_name" > "$lockfile"
 
     # Register EXIT trap to always release lock
     trap "_release_on_exit '$lockfile'" EXIT
@@ -786,9 +797,11 @@ acquire_rsync_lock() {
 
     # Per-profile lock — same profile cannot run twice
     if [[ -f "$profile_lock" ]]; then
-        local existing_pid
+        local lock_content existing_pid locked_name
-        existing_pid=$(cat "$profile_lock" 2>/dev/null)
+        lock_content=$(cat "$profile_lock" 2>/dev/null)
-        if [[ -n "$existing_pid" ]] && kill -0 "$existing_pid" 2>/dev/null; then
+        existing_pid="${lock_content%%:*}"
+        locked_name="${lock_content##*:}"
+        if [[ -n "$existing_pid" ]] && kill -0 "$existing_pid" 2>/dev/null && [[ "$locked_name" == "rsync_${profile}" ]]; then
             error "rsync profile '$profile' is already running (PID $existing_pid) — exiting"
             exit 1
         else
@@ -823,7 +836,7 @@ acquire_rsync_lock() {
     fi
 
     # Acquire profile lock and increment counter
-    echo $$ > "$profile_lock"
+    echo "$$:rsync_${profile}" > "$profile_lock"
     echo $(( current_count + 1 )) > "$RSYNC_COUNT_FILE"
 
     # Register EXIT trap