Fix play state sync, add manual stop detection, minor bug fixes
play_state_sync: fix TVDB episode lookup for Jellyfin 10.x — AnyProviderIdEquals returns the entire library for TVDB queries; switch to season+episode search with ProviderIds.Tvdb validation to find the correct episode. Also fix pkey extraction that was pulling s7e2 instead of the TVDB ID from tvdb:ep:5618559:s7e2. docker_watchdog: add automatic manual-stop detection — containers stopped cleanly (exit 0/143) are tracked in docker_watchdog_manual_stop.db and skipped until restarted, removing the need to add manually-stopped containers to the exclusion list. Auto-clears when the container is seen running again. docker_daily_restart: remove bare `local` declarations outside a function that were printing an error for every container restarted.
This commit is contained in:
Gmer4Lfe
@@ -130,10 +130,11 @@
|
||||
# STATE FILES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# WATCHDOG_STATE_FILE — strike counts, daemon flags (STATE_DIR — survives reboots)
|
||||
# WATCHDOG_STATE_FILE — strike counts, daemon flags (STATE_DIR — survives reboots)
|
||||
# DOCKER_WATCHDOG_FAILED_FILE — container skip list (STATE_DIR — survives reboots)
|
||||
# WATCHDOG_CONTAINER_RESTART_LOG — restart history for loop detection (DATA_DIR)
|
||||
# RW_STATE_FILE — read-only: resource_watchdog RAM emergency flag
|
||||
# DOCKER_WATCHDOG_MANUAL_STOP_FILE — intentionally stopped containers (STATE_DIR — survives reboots)
|
||||
# WATCHDOG_CONTAINER_RESTART_LOG — restart history for loop detection (DATA_DIR)
|
||||
# RW_STATE_FILE — read-only: resource_watchdog RAM emergency flag
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
@@ -257,7 +258,7 @@ validate_unraid_cmd "/usr/local/emhttp/plugins/dynamix/scripts/notify" "
|
||||
|
||||
# Ensure state files exist
|
||||
touch "$WATCHDOG_STATE_FILE" "$WATCHDOG_CONTAINER_RESTART_LOG" \
|
||||
"$DOCKER_WATCHDOG_FAILED_FILE" 2>/dev/null
|
||||
"$DOCKER_WATCHDOG_FAILED_FILE" "$DOCKER_WATCHDOG_MANUAL_STOP_FILE" 2>/dev/null
|
||||
|
||||
# Timeout for all docker commands — configurable via WATCHDOG_DAEMON_TIMEOUT in master.conf
|
||||
DOCKER_TIMEOUT="${WATCHDOG_DAEMON_TIMEOUT:-20}"
|
||||
@@ -273,6 +274,8 @@ if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo "$ICON_CONTAINERS Required: ${WATCHDOG_REQUIRED_CONTAINERS[*]:-none}"
|
||||
echo "$ICON_WATCHDOG Scan all: $WATCHDOG_SCAN_ALL"
|
||||
echo "$ICON_WATCHDOG Ignore: ${WATCHDOG_SCAN_IGNORE[*]:-none}"
|
||||
_ms_list=$(cat "$DOCKER_WATCHDOG_MANUAL_STOP_FILE" 2>/dev/null | tr '\n' ' ' | xargs)
|
||||
echo "$ICON_WATCHDOG Manual-stop: ${_ms_list:-none}"
|
||||
echo "$ICON_WATCHDOG Schedule: every 15 min (cron via watchdog_orchestrator)"
|
||||
echo "$ICON_WATCHDOG Startup grace: ${WATCHDOG_STARTUP_GRACE}s"
|
||||
echo "$ICON_WATCHDOG Restart limit: $WATCHDOG_CONTAINER_RESTART_LIMIT in ${WATCHDOG_CONTAINER_RESTART_WINDOW}h"
|
||||
@@ -327,6 +330,28 @@ remove_from_skip_list() {
|
||||
warn "$1 recovered — removed from skip list ✅"
|
||||
}
|
||||
|
||||
# Check if container was intentionally stopped (exit 0/143 — clean/SIGTERM)
|
||||
is_manually_stopped() {
|
||||
grep -q "^${1}$" "$DOCKER_WATCHDOG_MANUAL_STOP_FILE" 2>/dev/null
|
||||
}
|
||||
|
||||
# Mark container as intentionally stopped — auto-cleared when seen running again
|
||||
add_to_manual_stop() {
|
||||
local container="$1" exit_code="$2"
|
||||
if ! is_manually_stopped "$container"; then
|
||||
echo "$container" >> "$DOCKER_WATCHDOG_MANUAL_STOP_FILE"
|
||||
log "$container — stopped cleanly (exit $exit_code) — skipping until restarted"
|
||||
fi
|
||||
}
|
||||
|
||||
# Remove container from manual-stop list — called when container is seen running again
|
||||
remove_from_manual_stop() {
|
||||
if is_manually_stopped "$1"; then
|
||||
sed -i "/^${1}$/d" "$DOCKER_WATCHDOG_MANUAL_STOP_FILE" 2>/dev/null
|
||||
log "$1 — running again — removed from manual-stop list ✅"
|
||||
fi
|
||||
}
|
||||
|
||||
# Log a restart event to the rolling restart history file
|
||||
log_restart() {
|
||||
local container="$1"
|
||||
@@ -631,6 +656,10 @@ CYCLE_START=$(date +%s)
|
||||
_skip_contents=$(cat "$DOCKER_WATCHDOG_FAILED_FILE" 2>/dev/null | tr '\n' ' ' | xargs)
|
||||
[[ -n "$_skip_contents" ]] && warn "$ICON_SKIP Skip list active: $_skip_contents — manual intervention needed"
|
||||
|
||||
local _manual_stop_contents
|
||||
_manual_stop_contents=$(cat "$DOCKER_WATCHDOG_MANUAL_STOP_FILE" 2>/dev/null | tr '\n' ' ' | xargs)
|
||||
[[ -n "$_manual_stop_contents" ]] && log "$ICON_SKIP Manual-stop list: $_manual_stop_contents"
|
||||
|
||||
# ── Docker daemon health check — first check every run ──────────────────────────────────
|
||||
# If daemon is hung all container operations will fail — check first, skip run if down
|
||||
if ! check_docker_daemon; then
|
||||
@@ -687,26 +716,36 @@ CYCLE_START=$(date +%s)
|
||||
fi
|
||||
|
||||
if [[ "$STATUS" == "true" ]]; then
|
||||
# Running — clear any strikes
|
||||
# Running — clear any strikes and manual-stop flag
|
||||
remove_from_manual_stop "$container"
|
||||
set_strikes "$container" 0 "$WATCHDOG_STATE_FILE"
|
||||
log "$ICON_RUNNING $container — running ✅"
|
||||
elif is_manually_stopped "$container"; then
|
||||
log "$container — manually stopped — skipping"
|
||||
else
|
||||
STRIKES=$(get_strikes "$container" "$WATCHDOG_STATE_FILE")
|
||||
STRIKES=$(( STRIKES + 1 ))
|
||||
set_strikes "$container" "$STRIKES" "$WATCHDOG_STATE_FILE"
|
||||
warn "$container — not running (strike $STRIKES/$SYS_WATCHDOG_STRIKE_LIMIT)"
|
||||
((T1_WARNINGS++))
|
||||
# Check if this was a clean/intentional stop before striking
|
||||
EXIT_CODE=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
|
||||
'{{.State.ExitCode}}' "$container" 2>/dev/null || echo "-1")
|
||||
if [[ "$EXIT_CODE" == "0" || "$EXIT_CODE" == "143" ]]; then
|
||||
add_to_manual_stop "$container" "$EXIT_CODE"
|
||||
else
|
||||
STRIKES=$(get_strikes "$container" "$WATCHDOG_STATE_FILE")
|
||||
STRIKES=$(( STRIKES + 1 ))
|
||||
set_strikes "$container" "$STRIKES" "$WATCHDOG_STATE_FILE"
|
||||
warn "$container — not running (strike $STRIKES/$SYS_WATCHDOG_STRIKE_LIMIT)"
|
||||
((T1_WARNINGS++))
|
||||
|
||||
if [[ "$STRIKES" -ge "$SYS_WATCHDOG_STRIKE_LIMIT" ]]; then
|
||||
result=0
|
||||
safe_restart "$container" "required container down" || result=$?
|
||||
case $result in
|
||||
0) set_strikes "$container" 0 "$WATCHDOG_STATE_FILE"
|
||||
((T1_RESTARTS++))
|
||||
queue_notify "$container was down and restarted on $(hostname)" "warning" ;;
|
||||
2) : ;; # Added to skip list — already notified
|
||||
*) queue_notify "$container failed to restart on $(hostname)" "warning" ;;
|
||||
esac
|
||||
if [[ "$STRIKES" -ge "$SYS_WATCHDOG_STRIKE_LIMIT" ]]; then
|
||||
result=0
|
||||
safe_restart "$container" "required container down" || result=$?
|
||||
case $result in
|
||||
0) set_strikes "$container" 0 "$WATCHDOG_STATE_FILE"
|
||||
((T1_RESTARTS++))
|
||||
queue_notify "$container was down and restarted on $(hostname)" "warning" ;;
|
||||
2) : ;; # Added to skip list — already notified
|
||||
*) queue_notify "$container failed to restart on $(hostname)" "warning" ;;
|
||||
esac
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
done
|
||||
@@ -811,6 +850,13 @@ CYCLE_START=$(date +%s)
|
||||
|
||||
ALL_CONTAINERS=$(timeout "$DOCKER_TIMEOUT" docker ps --format "{{.Names}}" 2>/dev/null)
|
||||
|
||||
# ── Auto-clear manual-stop list ───────────────────────────────────────────────────────
|
||||
# Any container now running was started intentionally — remove from manual-stop list
|
||||
while IFS= read -r container; do
|
||||
[[ -z "$container" ]] && continue
|
||||
remove_from_manual_stop "$container"
|
||||
done <<< "$ALL_CONTAINERS"
|
||||
|
||||
# ── Unhealthy containers ──────────────────────────────────────────────────────────────
|
||||
if [[ "$WATCHDOG_RESTART_UNHEALTHY" == "true" ]]; then
|
||||
UNHEALTHY=$(timeout "$DOCKER_TIMEOUT" docker ps \
|
||||
@@ -915,13 +961,13 @@ CYCLE_START=$(date +%s)
|
||||
fi
|
||||
|
||||
# ── Unexpected exits ──────────────────────────────────────────────────────────────────
|
||||
# Only non-zero exit codes — exit 0 is a clean stop, not a crash
|
||||
# Exit 0/143 = clean/SIGTERM — intentional stop, add to manual-stop list and skip.
|
||||
# All other non-zero exits = crash — restart via safe_restart.
|
||||
# Skips containers already covered by WATCHDOG_REQUIRED_CONTAINERS (handled in Tier 1)
|
||||
if [[ "$WATCHDOG_RESTART_CRASHED" == "true" ]]; then
|
||||
CRASHED=$(timeout "$DOCKER_TIMEOUT" docker ps -a \
|
||||
EXITED=$(timeout "$DOCKER_TIMEOUT" docker ps -a \
|
||||
--filter status=exited \
|
||||
--format "{{.Names}}|{{.Status}}" 2>/dev/null | \
|
||||
grep -v "Exited (0)")
|
||||
--format "{{.Names}}|{{.Status}}" 2>/dev/null)
|
||||
while IFS='|' read -r container status; do
|
||||
[[ -z "$container" ]] && continue
|
||||
[[ -n "${IGNORE_MAP[$container]:-}" ]] && continue
|
||||
@@ -932,6 +978,20 @@ CYCLE_START=$(date +%s)
|
||||
[[ "$container" == "$req" ]] && already_required=true && break
|
||||
done
|
||||
[[ "$already_required" == true ]] && continue
|
||||
|
||||
# Extract exit code from status string e.g. "Exited (143) 2 hours ago"
|
||||
EXIT_CODE="-1"
|
||||
[[ "$status" =~ Exited\ \(([0-9]+)\) ]] && EXIT_CODE="${BASH_REMATCH[1]}"
|
||||
|
||||
# Clean/intentional stop — add to manual-stop list and skip
|
||||
if [[ "$EXIT_CODE" == "0" || "$EXIT_CODE" == "143" ]]; then
|
||||
add_to_manual_stop "$container" "$EXIT_CODE"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Already known to be manually stopped from a prior cycle
|
||||
is_manually_stopped "$container" && continue
|
||||
|
||||
error "$container — $status (unexpected exit)"
|
||||
((T2_WARNINGS++))
|
||||
result=0
|
||||
@@ -940,7 +1000,7 @@ CYCLE_START=$(date +%s)
|
||||
((T2_RESTARTS++))
|
||||
queue_notify "$container crashed on $(hostname) ($status) — restarted" "warning"
|
||||
fi
|
||||
done <<< "$CRASHED"
|
||||
done <<< "$EXITED"
|
||||
fi
|
||||
|
||||
fi # WATCHDOG_SCAN_ALL
|
||||
|
||||
Reference in New Issue
Block a user