FailedProxy/Varaverk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add Rsync page, upgrade monitor rsync card, partnership overhaul, API key periodic check, docker watchdog manual-stop detection

Browse Source
This commit is contained in:
Gmer4Lfe
2026-06-03 15:58:50 -04:00
parent 8be54ab5ee
commit de6fcc3997
13 changed files with 2546 additions and 343 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Watchdogs/docker_watchdog.sh
+78 -85
View File
@@ -130,11 +130,10 @@
# STATE FILES
# ==============================================================================================
#
# WATCHDOG_STATE_FILE — strike counts, daemon flags (STATE_DIR — survives reboots)
# WATCHDOG_STATE_FILE — strike counts, daemon flags (STATE_DIR — survives reboots)
# DOCKER_WATCHDOG_FAILED_FILE — container skip list (STATE_DIR — survives reboots)
# DOCKER_WATCHDOG_MANUAL_STOP_FILE — intentionally stopped containers (STATE_DIR — survives reboots)
# WATCHDOG_CONTAINER_RESTART_LOG — restart history for loop detection (DATA_DIR)
# RW_STATE_FILE — read-only: resource_watchdog RAM emergency flag
# WATCHDOG_CONTAINER_RESTART_LOG — restart history for loop detection (DATA_DIR)
# RW_STATE_FILE — read-only: resource_watchdog RAM emergency flag
#
# ==============================================================================================
# CONFIGURATION
@@ -150,6 +149,11 @@
# HTTP health check endpoints. Format: "ContainerName:http://host:port"
# Aliased by detect_hosts() → WATCHDOG_CONTAINER_URLS
#
# HOST*_WATCHDOG_CONTAINER_API_CHECKS
# API liveness checks — deeper than HTTP. Format: "ContainerName:URL|APIKey"
# Use endpoints that require a live DB round-trip (e.g. Emby /System/Info).
# Aliased by detect_hosts() → WATCHDOG_CONTAINER_API_CHECKS
#
# HOST*_WATCHDOG_REQUIRED_CONTAINERS
# Containers that must always be running. Aliased by detect_hosts() →
# WATCHDOG_REQUIRED_CONTAINERS
@@ -258,7 +262,7 @@ validate_unraid_cmd "/usr/local/emhttp/plugins/dynamix/scripts/notify" "
# Ensure state files exist
touch "$WATCHDOG_STATE_FILE" "$WATCHDOG_CONTAINER_RESTART_LOG" \
"$DOCKER_WATCHDOG_FAILED_FILE" "$DOCKER_WATCHDOG_MANUAL_STOP_FILE" 2>/dev/null
"$DOCKER_WATCHDOG_FAILED_FILE" 2>/dev/null
# Timeout for all docker commands — configurable via WATCHDOG_DAEMON_TIMEOUT in master.conf
DOCKER_TIMEOUT="${WATCHDOG_DAEMON_TIMEOUT:-20}"
@@ -274,8 +278,6 @@ if [[ "$SHOW_STATUS" == true ]]; then
echo "$ICON_CONTAINERS Required: ${WATCHDOG_REQUIRED_CONTAINERS[*]:-none}"
echo "$ICON_WATCHDOG Scan all: $WATCHDOG_SCAN_ALL"
echo "$ICON_WATCHDOG Ignore: ${WATCHDOG_SCAN_IGNORE[*]:-none}"
_ms_list=$(cat "$DOCKER_WATCHDOG_MANUAL_STOP_FILE" 2>/dev/null | tr '\n' ' ' | xargs)
echo "$ICON_WATCHDOG Manual-stop: ${_ms_list:-none}"
echo "$ICON_WATCHDOG Schedule: every 15 min (cron via watchdog_orchestrator)"
echo "$ICON_WATCHDOG Startup grace: ${WATCHDOG_STARTUP_GRACE}s"
echo "$ICON_WATCHDOG Restart limit: $WATCHDOG_CONTAINER_RESTART_LIMIT in ${WATCHDOG_CONTAINER_RESTART_WINDOW}h"
@@ -330,28 +332,6 @@ remove_from_skip_list() {
warn "$1 recovered — removed from skip list ✅"
}
# Check if container was intentionally stopped (exit 0/143 — clean/SIGTERM)
is_manually_stopped() {
grep -q "^${1}$" "$DOCKER_WATCHDOG_MANUAL_STOP_FILE" 2>/dev/null
}
# Mark container as intentionally stopped — auto-cleared when seen running again
add_to_manual_stop() {
local container="$1" exit_code="$2"
if ! is_manually_stopped "$container"; then
echo "$container" >> "$DOCKER_WATCHDOG_MANUAL_STOP_FILE"
log "$container — stopped cleanly (exit $exit_code) — skipping until restarted"
fi
}
# Remove container from manual-stop list — called when container is seen running again
remove_from_manual_stop() {
if is_manually_stopped "$1"; then
sed -i "/^${1}$/d" "$DOCKER_WATCHDOG_MANUAL_STOP_FILE" 2>/dev/null
log "$1 — running again — removed from manual-stop list ✅"
fi
}
# Log a restart event to the rolling restart history file
log_restart() {
local container="$1"
@@ -656,10 +636,6 @@ CYCLE_START=$(date +%s)
_skip_contents=$(cat "$DOCKER_WATCHDOG_FAILED_FILE" 2>/dev/null | tr '\n' ' ' | xargs)
[[ -n "$_skip_contents" ]] && warn "$ICON_SKIP Skip list active: $_skip_contents — manual intervention needed"
local _manual_stop_contents
_manual_stop_contents=$(cat "$DOCKER_WATCHDOG_MANUAL_STOP_FILE" 2>/dev/null | tr '\n' ' ' | xargs)
[[ -n "$_manual_stop_contents" ]] && log "$ICON_SKIP Manual-stop list: $_manual_stop_contents"
# ── Docker daemon health check — first check every run ──────────────────────────────────
# If daemon is hung all container operations will fail — check first, skip run if down
if ! check_docker_daemon; then
@@ -716,36 +692,26 @@ CYCLE_START=$(date +%s)
fi
if [[ "$STATUS" == "true" ]]; then
# Running — clear any strikes and manual-stop flag
remove_from_manual_stop "$container"
# Running — clear any strikes
set_strikes "$container" 0 "$WATCHDOG_STATE_FILE"
log "$ICON_RUNNING $container — running ✅"
elif is_manually_stopped "$container"; then
log "$container — manually stopped — skipping"
else
# Check if this was a clean/intentional stop before striking
EXIT_CODE=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
'{{.State.ExitCode}}' "$container" 2>/dev/null || echo "-1")
if [[ "$EXIT_CODE" == "0" || "$EXIT_CODE" == "143" ]]; then
add_to_manual_stop "$container" "$EXIT_CODE"
else
STRIKES=$(get_strikes "$container" "$WATCHDOG_STATE_FILE")
STRIKES=$(( STRIKES + 1 ))
set_strikes "$container" "$STRIKES" "$WATCHDOG_STATE_FILE"
warn "$container — not running (strike $STRIKES/$SYS_WATCHDOG_STRIKE_LIMIT)"
((T1_WARNINGS++))
STRIKES=$(get_strikes "$container" "$WATCHDOG_STATE_FILE")
STRIKES=$(( STRIKES + 1 ))
set_strikes "$container" "$STRIKES" "$WATCHDOG_STATE_FILE"
warn "$container — not running (strike $STRIKES/$SYS_WATCHDOG_STRIKE_LIMIT)"
((T1_WARNINGS++))
if [[ "$STRIKES" -ge "$SYS_WATCHDOG_STRIKE_LIMIT" ]]; then
result=0
safe_restart "$container" "required container down" || result=$?
case $result in
0) set_strikes "$container" 0 "$WATCHDOG_STATE_FILE"
((T1_RESTARTS++))
queue_notify "$container was down and restarted on $(hostname)" "warning" ;;
2) : ;; # Added to skip list — already notified
*) queue_notify "$container failed to restart on $(hostname)" "warning" ;;
esac
fi
if [[ "$STRIKES" -ge "$SYS_WATCHDOG_STRIKE_LIMIT" ]]; then
result=0
safe_restart "$container" "required container down" || result=$?
case $result in
0) set_strikes "$container" 0 "$WATCHDOG_STATE_FILE"
((T1_RESTARTS++))
queue_notify "$container was down and restarted on $(hostname)" "warning" ;;
2) : ;; # Added to skip list — already notified
*) queue_notify "$container failed to restart on $(hostname)" "warning" ;;
esac
fi
fi
done
@@ -843,6 +809,54 @@ CYCLE_START=$(date +%s)
done
fi
# ── API liveness checks ───────────────────────────────────────────────────────────────────
# Catches containers that serve HTTP 200 but are internally frozen (DB lock, deadlocked
# thread, etc.). Endpoint must require a live DB round-trip to respond successfully.
# Format per entry: "URL|APIKey"
if [[ ${#WATCHDOG_CONTAINER_API_CHECKS[@]} -gt 0 ]]; then
for container in "${!WATCHDOG_CONTAINER_API_CHECKS[@]}"; do
IFS='|' read -r _api_url _api_key <<< "${WATCHDOG_CONTAINER_API_CHECKS[$container]}"
# Skip entirely if key is absent or a placeholder — check is optional protection
if [[ -z "$_api_key" || "$_api_key" == "YOUR_API_KEY"* || "$_api_key" == "placeholder"* ]]; then
log "$container — API check skipped (no key configured)"
continue
fi
_api_http=$(curl -s --max-time "$CURL_TIMEOUT" \
-H "X-Emby-Token: ${_api_key}" \
-o /tmp/_varaverk_api_check \
-w "%{http_code}" \
"$_api_url" 2>/dev/null)
_resp=$(cat /tmp/_varaverk_api_check 2>/dev/null)
# 401/403 = wrong key — skip silently, don't penalise the container
if [[ "$_api_http" == "401" || "$_api_http" == "403" ]]; then
log "$container — API check skipped (HTTP $_api_http — key may be wrong or revoked)"
continue
fi
if echo "$_resp" | jq -e '.ServerName // .Id // .Version' >/dev/null 2>&1; then
set_strikes "${container}_api" 0 "$WATCHDOG_STATE_FILE"
else
API_STRIKES=$(get_strikes "${container}_api" "$WATCHDOG_STATE_FILE")
API_STRIKES=$(( API_STRIKES + 1 ))
set_strikes "${container}_api" "$API_STRIKES" "$WATCHDOG_STATE_FILE"
warn "$container — API unresponsive at $_api_url (strike $API_STRIKES/$RESP_FAIL_LIMIT)"
((T1_WARNINGS++))
if [[ "$API_STRIKES" -ge "$RESP_FAIL_LIMIT" ]]; then
result=0
safe_restart "$container" "API unresponsive at $_api_url" || result=$?
if [[ $result -eq 0 ]]; then
set_strikes "${container}_api" 0 "$WATCHDOG_STATE_FILE"
((T1_RESTARTS++))
queue_notify "$container API unresponsive at $_api_url on $(hostname) — restarted" "warning"
fi
fi
fi
done
fi
# ==========================================================================================
# ── TIER 2 — Global Health Scan ───────────────────────────────────────────────────────────
# ==========================================================================================
@@ -850,13 +864,6 @@ CYCLE_START=$(date +%s)
ALL_CONTAINERS=$(timeout "$DOCKER_TIMEOUT" docker ps --format "{{.Names}}" 2>/dev/null)
# ── Auto-clear manual-stop list ───────────────────────────────────────────────────────
# Any container now running was started intentionally — remove from manual-stop list
while IFS= read -r container; do
[[ -z "$container" ]] && continue
remove_from_manual_stop "$container"
done <<< "$ALL_CONTAINERS"
# ── Unhealthy containers ──────────────────────────────────────────────────────────────
if [[ "$WATCHDOG_RESTART_UNHEALTHY" == "true" ]]; then
UNHEALTHY=$(timeout "$DOCKER_TIMEOUT" docker ps \
@@ -961,13 +968,13 @@ CYCLE_START=$(date +%s)
fi
# ── Unexpected exits ──────────────────────────────────────────────────────────────────
# Exit 0/143 = clean/SIGTERM — intentional stop, add to manual-stop list and skip.
# All other non-zero exits = crash — restart via safe_restart.
# Only non-zero exit codes — exit 0 is a clean stop, not a crash
# Skips containers already covered by WATCHDOG_REQUIRED_CONTAINERS (handled in Tier 1)
if [[ "$WATCHDOG_RESTART_CRASHED" == "true" ]]; then
EXITED=$(timeout "$DOCKER_TIMEOUT" docker ps -a \
CRASHED=$(timeout "$DOCKER_TIMEOUT" docker ps -a \
--filter status=exited \
--format "{{.Names}}|{{.Status}}" 2>/dev/null)
--format "{{.Names}}|{{.Status}}" 2>/dev/null | \
grep -v "Exited (0)")
while IFS='|' read -r container status; do
[[ -z "$container" ]] && continue
[[ -n "${IGNORE_MAP[$container]:-}" ]] && continue
@@ -978,20 +985,6 @@ CYCLE_START=$(date +%s)
[[ "$container" == "$req" ]] && already_required=true && break
done
[[ "$already_required" == true ]] && continue
# Extract exit code from status string e.g. "Exited (143) 2 hours ago"
EXIT_CODE="-1"
[[ "$status" =~ Exited\ \(([0-9]+)\) ]] && EXIT_CODE="${BASH_REMATCH[1]}"
# Clean/intentional stop — add to manual-stop list and skip
if [[ "$EXIT_CODE" == "0" || "$EXIT_CODE" == "143" ]]; then
add_to_manual_stop "$container" "$EXIT_CODE"
continue
fi
# Already known to be manually stopped from a prior cycle
is_manually_stopped "$container" && continue
error "$container — $status (unexpected exit)"
((T2_WARNINGS++))
result=0
@@ -1000,7 +993,7 @@ CYCLE_START=$(date +%s)
((T2_RESTARTS++))
queue_notify "$container crashed on $(hostname) ($status) — restarted" "warning"
fi
done <<< "$EXITED"
done <<< "$CRASHED"
fi
fi # WATCHDOG_SCAN_ALL