FailedProxy/Varaverk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix play state sync, add manual stop detection, minor bug fixes

Browse Source 
play_state_sync: fix TVDB episode lookup for Jellyfin 10.x — AnyProviderIdEquals
returns the entire library for TVDB queries; switch to season+episode search with
ProviderIds.Tvdb validation to find the correct episode. Also fix pkey extraction
that was pulling s7e2 instead of the TVDB ID from tvdb:ep:5618559:s7e2.

docker_watchdog: add automatic manual-stop detection — containers stopped cleanly
(exit 0/143) are tracked in docker_watchdog_manual_stop.db and skipped until
restarted, removing the need to add manually-stopped containers to the exclusion
list. Auto-clears when the container is seen running again.

docker_daily_restart: remove bare `local` declarations outside a function that
were printing an error for every container restarted.
This commit is contained in:
Gmer4Lfe
2026-06-02 17:55:26 -04:00
parent 1a8d40d631
commit 8be54ab5ee
3 changed files with 103 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Docker_Essentials/docker_daily_restart.sh
-2
View File
@@ -293,9 +293,7 @@ LAST_RESTARTED=""
for container in "${ORDERED_RESTART[@]}"; do for container in "${ORDERED_RESTART[@]}"; do
[[ -z "$container" ]] && continue [[ -z "$container" ]] && continue
local c_start
c_start=$(date +%s) c_start=$(date +%s)
local c_image
c_image=$(docker inspect --format '{{.Config.Image}}' "$container" 2>/dev/null || echo "unknown") c_image=$(docker inspect --format '{{.Config.Image}}' "$container" 2>/dev/null || echo "unknown")
log "━━━ $ICON_CONTAINERS $container ($c_image) ━━━" log "━━━ $ICON_CONTAINERS $container ($c_image) ━━━"
Media/play_state_sync.sh
+18 -3
View File
@@ -435,14 +435,29 @@ for lname in "${!USER_MAP[@]}"; do
if [[ -z "$_iid" ]]; then if [[ -z "$_iid" ]]; then
_ptype="${_pkey%%:*}" _ptype="${_pkey%%:*}"
_pval="${_pkey##*:}" _pval="${_pkey##*:}"
_search_field=""
case "$_ptype" in case "$_ptype" in
imdb) _search_field="imdb.${_pval}" ;; imdb) _search_field="imdb.${_pval}" ;;
tmdb) _search_field="tmdb.${_pval##movie:}" ;; tmdb) _search_field="tmdb.${_pval##movie:}" ;;
tvdb) _search_field="tvdb.${_pval%%:*}" ;; tvdb)
mb) _search_field="" ;; # skip music if not found # AnyProviderIdEquals is unreliable for TVDB in Jellyfin 10.x —
# returns the entire library. Search by season+episode instead,
# then validate by ProviderIds.Tvdb.
# pkey format: tvdb:ep:TVDB_EP_ID:sSEASONePEP
_tvdb_ep_id="${_pkey#tvdb:ep:}"
_tvdb_ep_id="${_tvdb_ep_id%%:*}"
if [[ "$_pval" =~ ^s([0-9]+)e([0-9]+)$ ]]; then
_iid=$(_api_get "${SRV_URL[$_si]}" "${SRV_KEY[$_si]}" \
"Items?IncludeItemTypes=Episode&ParentIndexNumber=${BASH_REMATCH[1]}&IndexNumber=${BASH_REMATCH[2]}&Recursive=true&Fields=ProviderIds&Limit=500" 2>/dev/null \
| jq -r --arg tvdb "$_tvdb_ep_id" \
'.Items[] | select(.ProviderIds.Tvdb == $tvdb) | .Id' 2>/dev/null | head -1)
fi
;;
mb) : ;; # skip music if not found
esac esac
if [[ -n "$_search_field" ]]; then if [[ -z "$_iid" && -n "$_search_field" ]]; then
_iid=$(_api_get "${SRV_URL[$_si]}" "${SRV_KEY[$_si]}" \ _iid=$(_api_get "${SRV_URL[$_si]}" "${SRV_KEY[$_si]}" \
"Items?AnyProviderIdEquals=${_search_field}&Recursive=true&Fields=ProviderIds&Limit=1" 2>/dev/null \ "Items?AnyProviderIdEquals=${_search_field}&Recursive=true&Fields=ProviderIds&Limit=1" 2>/dev/null \
| jq -r '.Items[0].Id // empty' 2>/dev/null) | jq -r '.Items[0].Id // empty' 2>/dev/null)
Watchdogs/docker_watchdog.sh
+85 -25
View File
@@ -130,10 +130,11 @@
# STATE FILES # STATE FILES
# ============================================================================================== # ==============================================================================================
# #
# WATCHDOG_STATE_FILE — strike counts, daemon flags (STATE_DIR — survives reboots) # WATCHDOG_STATE_FILE — strike counts, daemon flags (STATE_DIR — survives reboots)
# DOCKER_WATCHDOG_FAILED_FILE — container skip list (STATE_DIR — survives reboots) # DOCKER_WATCHDOG_FAILED_FILE — container skip list (STATE_DIR — survives reboots)
# WATCHDOG_CONTAINER_RESTART_LOG — restart history for loop detection (DATA_DIR) # DOCKER_WATCHDOG_MANUAL_STOP_FILE — intentionally stopped containers (STATE_DIR — survives reboots)
# RW_STATE_FILE — read-only: resource_watchdog RAM emergency flag # WATCHDOG_CONTAINER_RESTART_LOG — restart history for loop detection (DATA_DIR)
# RW_STATE_FILE — read-only: resource_watchdog RAM emergency flag
# #
# ============================================================================================== # ==============================================================================================
# CONFIGURATION # CONFIGURATION
@@ -257,7 +258,7 @@ validate_unraid_cmd "/usr/local/emhttp/plugins/dynamix/scripts/notify" "
# Ensure state files exist # Ensure state files exist
touch "$WATCHDOG_STATE_FILE" "$WATCHDOG_CONTAINER_RESTART_LOG" \ touch "$WATCHDOG_STATE_FILE" "$WATCHDOG_CONTAINER_RESTART_LOG" \
"$DOCKER_WATCHDOG_FAILED_FILE" 2>/dev/null "$DOCKER_WATCHDOG_FAILED_FILE" "$DOCKER_WATCHDOG_MANUAL_STOP_FILE" 2>/dev/null
# Timeout for all docker commands — configurable via WATCHDOG_DAEMON_TIMEOUT in master.conf # Timeout for all docker commands — configurable via WATCHDOG_DAEMON_TIMEOUT in master.conf
DOCKER_TIMEOUT="${WATCHDOG_DAEMON_TIMEOUT:-20}" DOCKER_TIMEOUT="${WATCHDOG_DAEMON_TIMEOUT:-20}"
@@ -273,6 +274,8 @@ if [[ "$SHOW_STATUS" == true ]]; then
echo "$ICON_CONTAINERS Required: ${WATCHDOG_REQUIRED_CONTAINERS[*]:-none}" echo "$ICON_CONTAINERS Required: ${WATCHDOG_REQUIRED_CONTAINERS[*]:-none}"
echo "$ICON_WATCHDOG Scan all: $WATCHDOG_SCAN_ALL" echo "$ICON_WATCHDOG Scan all: $WATCHDOG_SCAN_ALL"
echo "$ICON_WATCHDOG Ignore: ${WATCHDOG_SCAN_IGNORE[*]:-none}" echo "$ICON_WATCHDOG Ignore: ${WATCHDOG_SCAN_IGNORE[*]:-none}"
_ms_list=$(cat "$DOCKER_WATCHDOG_MANUAL_STOP_FILE" 2>/dev/null | tr '\n' ' ' | xargs)
echo "$ICON_WATCHDOG Manual-stop: ${_ms_list:-none}"
echo "$ICON_WATCHDOG Schedule: every 15 min (cron via watchdog_orchestrator)" echo "$ICON_WATCHDOG Schedule: every 15 min (cron via watchdog_orchestrator)"
echo "$ICON_WATCHDOG Startup grace: ${WATCHDOG_STARTUP_GRACE}s" echo "$ICON_WATCHDOG Startup grace: ${WATCHDOG_STARTUP_GRACE}s"
echo "$ICON_WATCHDOG Restart limit: $WATCHDOG_CONTAINER_RESTART_LIMIT in ${WATCHDOG_CONTAINER_RESTART_WINDOW}h" echo "$ICON_WATCHDOG Restart limit: $WATCHDOG_CONTAINER_RESTART_LIMIT in ${WATCHDOG_CONTAINER_RESTART_WINDOW}h"
@@ -327,6 +330,28 @@ remove_from_skip_list() {
warn "$1 recovered — removed from skip list ✅" warn "$1 recovered — removed from skip list ✅"
} }
# Check if container was intentionally stopped (exit 0/143 — clean/SIGTERM)
is_manually_stopped() {
grep -q "^${1}$" "$DOCKER_WATCHDOG_MANUAL_STOP_FILE" 2>/dev/null
}
# Mark container as intentionally stopped — auto-cleared when seen running again
add_to_manual_stop() {
local container="$1" exit_code="$2"
if ! is_manually_stopped "$container"; then
echo "$container" >> "$DOCKER_WATCHDOG_MANUAL_STOP_FILE"
log "$container — stopped cleanly (exit $exit_code) — skipping until restarted"
fi
}
# Remove container from manual-stop list — called when container is seen running again
remove_from_manual_stop() {
if is_manually_stopped "$1"; then
sed -i "/^${1}$/d" "$DOCKER_WATCHDOG_MANUAL_STOP_FILE" 2>/dev/null
log "$1 — running again — removed from manual-stop list ✅"
fi
}
# Log a restart event to the rolling restart history file # Log a restart event to the rolling restart history file
log_restart() { log_restart() {
local container="$1" local container="$1"
@@ -631,6 +656,10 @@ CYCLE_START=$(date +%s)
_skip_contents=$(cat "$DOCKER_WATCHDOG_FAILED_FILE" 2>/dev/null | tr '\n' ' ' | xargs) _skip_contents=$(cat "$DOCKER_WATCHDOG_FAILED_FILE" 2>/dev/null | tr '\n' ' ' | xargs)
[[ -n "$_skip_contents" ]] && warn "$ICON_SKIP Skip list active: $_skip_contents — manual intervention needed" [[ -n "$_skip_contents" ]] && warn "$ICON_SKIP Skip list active: $_skip_contents — manual intervention needed"
local _manual_stop_contents
_manual_stop_contents=$(cat "$DOCKER_WATCHDOG_MANUAL_STOP_FILE" 2>/dev/null | tr '\n' ' ' | xargs)
[[ -n "$_manual_stop_contents" ]] && log "$ICON_SKIP Manual-stop list: $_manual_stop_contents"
# ── Docker daemon health check — first check every run ────────────────────────────────── # ── Docker daemon health check — first check every run ──────────────────────────────────
# If daemon is hung all container operations will fail — check first, skip run if down # If daemon is hung all container operations will fail — check first, skip run if down
if ! check_docker_daemon; then if ! check_docker_daemon; then
@@ -687,26 +716,36 @@ CYCLE_START=$(date +%s)
fi fi
if [[ "$STATUS" == "true" ]]; then if [[ "$STATUS" == "true" ]]; then
# Running — clear any strikes # Running — clear any strikes and manual-stop flag
remove_from_manual_stop "$container"
set_strikes "$container" 0 "$WATCHDOG_STATE_FILE" set_strikes "$container" 0 "$WATCHDOG_STATE_FILE"
log "$ICON_RUNNING $container — running ✅" log "$ICON_RUNNING $container — running ✅"
elif is_manually_stopped "$container"; then
log "$container — manually stopped — skipping"
else else
STRIKES=$(get_strikes "$container" "$WATCHDOG_STATE_FILE") # Check if this was a clean/intentional stop before striking
STRIKES=$(( STRIKES + 1 )) EXIT_CODE=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
set_strikes "$container" "$STRIKES" "$WATCHDOG_STATE_FILE" '{{.State.ExitCode}}' "$container" 2>/dev/null || echo "-1")
warn "$container — not running (strike $STRIKES/$SYS_WATCHDOG_STRIKE_LIMIT)" if [[ "$EXIT_CODE" == "0" || "$EXIT_CODE" == "143" ]]; then
((T1_WARNINGS++)) add_to_manual_stop "$container" "$EXIT_CODE"
else
STRIKES=$(get_strikes "$container" "$WATCHDOG_STATE_FILE")
STRIKES=$(( STRIKES + 1 ))
set_strikes "$container" "$STRIKES" "$WATCHDOG_STATE_FILE"
warn "$container — not running (strike $STRIKES/$SYS_WATCHDOG_STRIKE_LIMIT)"
((T1_WARNINGS++))
if [[ "$STRIKES" -ge "$SYS_WATCHDOG_STRIKE_LIMIT" ]]; then if [[ "$STRIKES" -ge "$SYS_WATCHDOG_STRIKE_LIMIT" ]]; then
result=0 result=0
safe_restart "$container" "required container down" || result=$? safe_restart "$container" "required container down" || result=$?
case $result in case $result in
0) set_strikes "$container" 0 "$WATCHDOG_STATE_FILE" 0) set_strikes "$container" 0 "$WATCHDOG_STATE_FILE"
((T1_RESTARTS++)) ((T1_RESTARTS++))
queue_notify "$container was down and restarted on $(hostname)" "warning" ;; queue_notify "$container was down and restarted on $(hostname)" "warning" ;;
2) : ;; # Added to skip list — already notified 2) : ;; # Added to skip list — already notified
*) queue_notify "$container failed to restart on $(hostname)" "warning" ;; *) queue_notify "$container failed to restart on $(hostname)" "warning" ;;
esac esac
fi
fi fi
fi fi
done done
@@ -811,6 +850,13 @@ CYCLE_START=$(date +%s)
ALL_CONTAINERS=$(timeout "$DOCKER_TIMEOUT" docker ps --format "{{.Names}}" 2>/dev/null) ALL_CONTAINERS=$(timeout "$DOCKER_TIMEOUT" docker ps --format "{{.Names}}" 2>/dev/null)
# ── Auto-clear manual-stop list ───────────────────────────────────────────────────────
# Any container now running was started intentionally — remove from manual-stop list
while IFS= read -r container; do
[[ -z "$container" ]] && continue
remove_from_manual_stop "$container"
done <<< "$ALL_CONTAINERS"
# ── Unhealthy containers ────────────────────────────────────────────────────────────── # ── Unhealthy containers ──────────────────────────────────────────────────────────────
if [[ "$WATCHDOG_RESTART_UNHEALTHY" == "true" ]]; then if [[ "$WATCHDOG_RESTART_UNHEALTHY" == "true" ]]; then
UNHEALTHY=$(timeout "$DOCKER_TIMEOUT" docker ps \ UNHEALTHY=$(timeout "$DOCKER_TIMEOUT" docker ps \
@@ -915,13 +961,13 @@ CYCLE_START=$(date +%s)
fi fi
# ── Unexpected exits ────────────────────────────────────────────────────────────────── # ── Unexpected exits ──────────────────────────────────────────────────────────────────
# Only non-zero exit codes — exit 0 is a clean stop, not a crash # Exit 0/143 = clean/SIGTERM — intentional stop, add to manual-stop list and skip.
# All other non-zero exits = crash — restart via safe_restart.
# Skips containers already covered by WATCHDOG_REQUIRED_CONTAINERS (handled in Tier 1) # Skips containers already covered by WATCHDOG_REQUIRED_CONTAINERS (handled in Tier 1)
if [[ "$WATCHDOG_RESTART_CRASHED" == "true" ]]; then if [[ "$WATCHDOG_RESTART_CRASHED" == "true" ]]; then
CRASHED=$(timeout "$DOCKER_TIMEOUT" docker ps -a \ EXITED=$(timeout "$DOCKER_TIMEOUT" docker ps -a \
--filter status=exited \ --filter status=exited \
--format "{{.Names}}|{{.Status}}" 2>/dev/null | \ --format "{{.Names}}|{{.Status}}" 2>/dev/null)
grep -v "Exited (0)")
while IFS='|' read -r container status; do while IFS='|' read -r container status; do
[[ -z "$container" ]] && continue [[ -z "$container" ]] && continue
[[ -n "${IGNORE_MAP[$container]:-}" ]] && continue [[ -n "${IGNORE_MAP[$container]:-}" ]] && continue
@@ -932,6 +978,20 @@ CYCLE_START=$(date +%s)
[[ "$container" == "$req" ]] && already_required=true && break [[ "$container" == "$req" ]] && already_required=true && break
done done
[[ "$already_required" == true ]] && continue [[ "$already_required" == true ]] && continue
# Extract exit code from status string e.g. "Exited (143) 2 hours ago"
EXIT_CODE="-1"
[[ "$status" =~ Exited\ \(([0-9]+)\) ]] && EXIT_CODE="${BASH_REMATCH[1]}"
# Clean/intentional stop — add to manual-stop list and skip
if [[ "$EXIT_CODE" == "0" || "$EXIT_CODE" == "143" ]]; then
add_to_manual_stop "$container" "$EXIT_CODE"
continue
fi
# Already known to be manually stopped from a prior cycle
is_manually_stopped "$container" && continue
error "$container — $status (unexpected exit)" error "$container — $status (unexpected exit)"
((T2_WARNINGS++)) ((T2_WARNINGS++))
result=0 result=0
@@ -940,7 +1000,7 @@ CYCLE_START=$(date +%s)
((T2_RESTARTS++)) ((T2_RESTARTS++))
queue_notify "$container crashed on $(hostname) ($status) — restarted" "warning" queue_notify "$container crashed on $(hostname) ($status) — restarted" "warning"
fi fi
done <<< "$CRASHED" done <<< "$EXITED"
fi fi
fi # WATCHDOG_SCAN_ALL fi # WATCHDOG_SCAN_ALL