diff --git a/Docker_Essentials/docker_daily_restart.sh b/Docker_Essentials/docker_daily_restart.sh index 8fb8ce7..b862b78 100755 --- a/Docker_Essentials/docker_daily_restart.sh +++ b/Docker_Essentials/docker_daily_restart.sh @@ -293,9 +293,7 @@ LAST_RESTARTED="" for container in "${ORDERED_RESTART[@]}"; do [[ -z "$container" ]] && continue - local c_start c_start=$(date +%s) - local c_image c_image=$(docker inspect --format '{{.Config.Image}}' "$container" 2>/dev/null || echo "unknown") log "━━━ $ICON_CONTAINERS $container ($c_image) ━━━" diff --git a/Media/play_state_sync.sh b/Media/play_state_sync.sh index 8935d96..caf6b63 100755 --- a/Media/play_state_sync.sh +++ b/Media/play_state_sync.sh @@ -435,14 +435,29 @@ for lname in "${!USER_MAP[@]}"; do if [[ -z "$_iid" ]]; then _ptype="${_pkey%%:*}" _pval="${_pkey##*:}" + _search_field="" + case "$_ptype" in imdb) _search_field="imdb.${_pval}" ;; tmdb) _search_field="tmdb.${_pval##movie:}" ;; - tvdb) _search_field="tvdb.${_pval%%:*}" ;; - mb) _search_field="" ;; # skip music if not found + tvdb) + # AnyProviderIdEquals is unreliable for TVDB in Jellyfin 10.x — + # returns the entire library. Search by season+episode instead, + # then validate by ProviderIds.Tvdb. + # pkey format: tvdb:ep:TVDB_EP_ID:sSEASONePEP + _tvdb_ep_id="${_pkey#tvdb:ep:}" + _tvdb_ep_id="${_tvdb_ep_id%%:*}" + if [[ "$_pval" =~ ^s([0-9]+)e([0-9]+)$ ]]; then + _iid=$(_api_get "${SRV_URL[$_si]}" "${SRV_KEY[$_si]}" \ + "Items?IncludeItemTypes=Episode&ParentIndexNumber=${BASH_REMATCH[1]}&IndexNumber=${BASH_REMATCH[2]}&Recursive=true&Fields=ProviderIds&Limit=500" 2>/dev/null \ + | jq -r --arg tvdb "$_tvdb_ep_id" \ + '.Items[] | select(.ProviderIds.Tvdb == $tvdb) | .Id' 2>/dev/null | head -1) + fi + ;; + mb) : ;; # skip music if not found esac - if [[ -n "$_search_field" ]]; then + if [[ -z "$_iid" && -n "$_search_field" ]]; then _iid=$(_api_get "${SRV_URL[$_si]}" "${SRV_KEY[$_si]}" \ "Items?AnyProviderIdEquals=${_search_field}&Recursive=true&Fields=ProviderIds&Limit=1" 2>/dev/null \ | jq -r '.Items[0].Id // empty' 2>/dev/null) diff --git a/Watchdogs/docker_watchdog.sh b/Watchdogs/docker_watchdog.sh index 586d7c0..8a1b9c3 100755 --- a/Watchdogs/docker_watchdog.sh +++ b/Watchdogs/docker_watchdog.sh @@ -130,10 +130,11 @@ # STATE FILES # ============================================================================================== # -# WATCHDOG_STATE_FILE — strike counts, daemon flags (STATE_DIR — survives reboots) +# WATCHDOG_STATE_FILE — strike counts, daemon flags (STATE_DIR — survives reboots) # DOCKER_WATCHDOG_FAILED_FILE — container skip list (STATE_DIR — survives reboots) -# WATCHDOG_CONTAINER_RESTART_LOG — restart history for loop detection (DATA_DIR) -# RW_STATE_FILE — read-only: resource_watchdog RAM emergency flag +# DOCKER_WATCHDOG_MANUAL_STOP_FILE — intentionally stopped containers (STATE_DIR — survives reboots) +# WATCHDOG_CONTAINER_RESTART_LOG — restart history for loop detection (DATA_DIR) +# RW_STATE_FILE — read-only: resource_watchdog RAM emergency flag # # ============================================================================================== # CONFIGURATION @@ -257,7 +258,7 @@ validate_unraid_cmd "/usr/local/emhttp/plugins/dynamix/scripts/notify" " # Ensure state files exist touch "$WATCHDOG_STATE_FILE" "$WATCHDOG_CONTAINER_RESTART_LOG" \ - "$DOCKER_WATCHDOG_FAILED_FILE" 2>/dev/null + "$DOCKER_WATCHDOG_FAILED_FILE" "$DOCKER_WATCHDOG_MANUAL_STOP_FILE" 2>/dev/null # Timeout for all docker commands — configurable via WATCHDOG_DAEMON_TIMEOUT in master.conf DOCKER_TIMEOUT="${WATCHDOG_DAEMON_TIMEOUT:-20}" @@ -273,6 +274,8 @@ if [[ "$SHOW_STATUS" == true ]]; then echo "$ICON_CONTAINERS Required: ${WATCHDOG_REQUIRED_CONTAINERS[*]:-none}" echo "$ICON_WATCHDOG Scan all: $WATCHDOG_SCAN_ALL" echo "$ICON_WATCHDOG Ignore: ${WATCHDOG_SCAN_IGNORE[*]:-none}" + _ms_list=$(cat "$DOCKER_WATCHDOG_MANUAL_STOP_FILE" 2>/dev/null | tr '\n' ' ' | xargs) + echo "$ICON_WATCHDOG Manual-stop: ${_ms_list:-none}" echo "$ICON_WATCHDOG Schedule: every 15 min (cron via watchdog_orchestrator)" echo "$ICON_WATCHDOG Startup grace: ${WATCHDOG_STARTUP_GRACE}s" echo "$ICON_WATCHDOG Restart limit: $WATCHDOG_CONTAINER_RESTART_LIMIT in ${WATCHDOG_CONTAINER_RESTART_WINDOW}h" @@ -327,6 +330,28 @@ remove_from_skip_list() { warn "$1 recovered — removed from skip list ✅" } +# Check if container was intentionally stopped (exit 0/143 — clean/SIGTERM) +is_manually_stopped() { + grep -q "^${1}$" "$DOCKER_WATCHDOG_MANUAL_STOP_FILE" 2>/dev/null +} + +# Mark container as intentionally stopped — auto-cleared when seen running again +add_to_manual_stop() { + local container="$1" exit_code="$2" + if ! is_manually_stopped "$container"; then + echo "$container" >> "$DOCKER_WATCHDOG_MANUAL_STOP_FILE" + log "$container — stopped cleanly (exit $exit_code) — skipping until restarted" + fi +} + +# Remove container from manual-stop list — called when container is seen running again +remove_from_manual_stop() { + if is_manually_stopped "$1"; then + sed -i "/^${1}$/d" "$DOCKER_WATCHDOG_MANUAL_STOP_FILE" 2>/dev/null + log "$1 — running again — removed from manual-stop list ✅" + fi +} + # Log a restart event to the rolling restart history file log_restart() { local container="$1" @@ -631,6 +656,10 @@ CYCLE_START=$(date +%s) _skip_contents=$(cat "$DOCKER_WATCHDOG_FAILED_FILE" 2>/dev/null | tr '\n' ' ' | xargs) [[ -n "$_skip_contents" ]] && warn "$ICON_SKIP Skip list active: $_skip_contents — manual intervention needed" + local _manual_stop_contents + _manual_stop_contents=$(cat "$DOCKER_WATCHDOG_MANUAL_STOP_FILE" 2>/dev/null | tr '\n' ' ' | xargs) + [[ -n "$_manual_stop_contents" ]] && log "$ICON_SKIP Manual-stop list: $_manual_stop_contents" + # ── Docker daemon health check — first check every run ────────────────────────────────── # If daemon is hung all container operations will fail — check first, skip run if down if ! check_docker_daemon; then @@ -687,26 +716,36 @@ CYCLE_START=$(date +%s) fi if [[ "$STATUS" == "true" ]]; then - # Running — clear any strikes + # Running — clear any strikes and manual-stop flag + remove_from_manual_stop "$container" set_strikes "$container" 0 "$WATCHDOG_STATE_FILE" log "$ICON_RUNNING $container — running ✅" + elif is_manually_stopped "$container"; then + log "$container — manually stopped — skipping" else - STRIKES=$(get_strikes "$container" "$WATCHDOG_STATE_FILE") - STRIKES=$(( STRIKES + 1 )) - set_strikes "$container" "$STRIKES" "$WATCHDOG_STATE_FILE" - warn "$container — not running (strike $STRIKES/$SYS_WATCHDOG_STRIKE_LIMIT)" - ((T1_WARNINGS++)) + # Check if this was a clean/intentional stop before striking + EXIT_CODE=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \ + '{{.State.ExitCode}}' "$container" 2>/dev/null || echo "-1") + if [[ "$EXIT_CODE" == "0" || "$EXIT_CODE" == "143" ]]; then + add_to_manual_stop "$container" "$EXIT_CODE" + else + STRIKES=$(get_strikes "$container" "$WATCHDOG_STATE_FILE") + STRIKES=$(( STRIKES + 1 )) + set_strikes "$container" "$STRIKES" "$WATCHDOG_STATE_FILE" + warn "$container — not running (strike $STRIKES/$SYS_WATCHDOG_STRIKE_LIMIT)" + ((T1_WARNINGS++)) - if [[ "$STRIKES" -ge "$SYS_WATCHDOG_STRIKE_LIMIT" ]]; then - result=0 - safe_restart "$container" "required container down" || result=$? - case $result in - 0) set_strikes "$container" 0 "$WATCHDOG_STATE_FILE" - ((T1_RESTARTS++)) - queue_notify "$container was down and restarted on $(hostname)" "warning" ;; - 2) : ;; # Added to skip list — already notified - *) queue_notify "$container failed to restart on $(hostname)" "warning" ;; - esac + if [[ "$STRIKES" -ge "$SYS_WATCHDOG_STRIKE_LIMIT" ]]; then + result=0 + safe_restart "$container" "required container down" || result=$? + case $result in + 0) set_strikes "$container" 0 "$WATCHDOG_STATE_FILE" + ((T1_RESTARTS++)) + queue_notify "$container was down and restarted on $(hostname)" "warning" ;; + 2) : ;; # Added to skip list — already notified + *) queue_notify "$container failed to restart on $(hostname)" "warning" ;; + esac + fi fi fi done @@ -811,6 +850,13 @@ CYCLE_START=$(date +%s) ALL_CONTAINERS=$(timeout "$DOCKER_TIMEOUT" docker ps --format "{{.Names}}" 2>/dev/null) + # ── Auto-clear manual-stop list ─────────────────────────────────────────────────────── + # Any container now running was started intentionally — remove from manual-stop list + while IFS= read -r container; do + [[ -z "$container" ]] && continue + remove_from_manual_stop "$container" + done <<< "$ALL_CONTAINERS" + # ── Unhealthy containers ────────────────────────────────────────────────────────────── if [[ "$WATCHDOG_RESTART_UNHEALTHY" == "true" ]]; then UNHEALTHY=$(timeout "$DOCKER_TIMEOUT" docker ps \ @@ -915,13 +961,13 @@ CYCLE_START=$(date +%s) fi # ── Unexpected exits ────────────────────────────────────────────────────────────────── - # Only non-zero exit codes — exit 0 is a clean stop, not a crash + # Exit 0/143 = clean/SIGTERM — intentional stop, add to manual-stop list and skip. + # All other non-zero exits = crash — restart via safe_restart. # Skips containers already covered by WATCHDOG_REQUIRED_CONTAINERS (handled in Tier 1) if [[ "$WATCHDOG_RESTART_CRASHED" == "true" ]]; then - CRASHED=$(timeout "$DOCKER_TIMEOUT" docker ps -a \ + EXITED=$(timeout "$DOCKER_TIMEOUT" docker ps -a \ --filter status=exited \ - --format "{{.Names}}|{{.Status}}" 2>/dev/null | \ - grep -v "Exited (0)") + --format "{{.Names}}|{{.Status}}" 2>/dev/null) while IFS='|' read -r container status; do [[ -z "$container" ]] && continue [[ -n "${IGNORE_MAP[$container]:-}" ]] && continue @@ -932,6 +978,20 @@ CYCLE_START=$(date +%s) [[ "$container" == "$req" ]] && already_required=true && break done [[ "$already_required" == true ]] && continue + + # Extract exit code from status string e.g. "Exited (143) 2 hours ago" + EXIT_CODE="-1" + [[ "$status" =~ Exited\ \(([0-9]+)\) ]] && EXIT_CODE="${BASH_REMATCH[1]}" + + # Clean/intentional stop — add to manual-stop list and skip + if [[ "$EXIT_CODE" == "0" || "$EXIT_CODE" == "143" ]]; then + add_to_manual_stop "$container" "$EXIT_CODE" + continue + fi + + # Already known to be manually stopped from a prior cycle + is_manually_stopped "$container" && continue + error "$container — $status (unexpected exit)" ((T2_WARNINGS++)) result=0 @@ -940,7 +1000,7 @@ CYCLE_START=$(date +%s) ((T2_RESTARTS++)) queue_notify "$container crashed on $(hostname) ($status) — restarted" "warning" fi - done <<< "$CRASHED" + done <<< "$EXITED" fi fi # WATCHDOG_SCAN_ALL