added a global check system for docker to docker watchdog
This commit is contained in:
+90
-36
@@ -380,33 +380,36 @@ WEEKLY_RESTART_CONTAINERS=(
|
||||
)
|
||||
|
||||
# ━━━ Docker Watchdog ━━━
|
||||
# First line of defense for container health — runs every 15 minutes via cron.
|
||||
# Monitors memory usage, CPU usage and HTTP responsiveness per container.
|
||||
# Uses a strike system to avoid restarting on brief spikes — sustained issues trigger restart.
|
||||
# Works alongside system_watchdog.sh — containers first, system reboot is the last resort.
|
||||
|
||||
# Containers to monitor with their memory hard limits in MB.
|
||||
# Memory hard limit exceeded → immediate restart (no strike system for memory).
|
||||
# CPU and HTTP use strike system — see CPU_FAIL_LIMIT and RESP_FAIL_LIMIT below.
|
||||
# Two-tier self-healing container monitoring:
|
||||
# Tier 1 — strict monitoring of explicitly configured containers
|
||||
# Tier 2 — global health scan of ALL running containers
|
||||
#
|
||||
# Cross-cutting intelligence applies to both tiers:
|
||||
# Startup grace — skip restarts while system is still booting
|
||||
# Dependency order — restart database before app, not the other way around
|
||||
# Restart loop — stop restarting after limit hit → skip list → notify critical
|
||||
# Skip list — persistent across reboots, auto-clears when container recovers
|
||||
# Batch notify — one clean summary per run instead of one ping per event
|
||||
|
||||
# ── Tier 1 — Strict Monitoring ────────────────────────────────────────────────────────────
|
||||
|
||||
# Memory hard limits in MB — immediate restart if exceeded, no strike system
|
||||
# 20GB=20480 16GB=16384 14GB=14336 12GB=12288 10GB=10240
|
||||
# 8GB=8192 6GB=6144 4GB=4096 2GB=2048 1GB=1024
|
||||
declare -A WATCHDOG_CONTAINERS=(
|
||||
["Emby"]=16384 # 16GB — media server, transcoding can spike high
|
||||
["LidaTube"]=6144 # 6GB — YouTube downloader
|
||||
["Tdarr"]=6144 # 6GB — transcoding node
|
||||
["Code-Server"]=1024 # 1GB — VS Code server
|
||||
["Emby"]=16384
|
||||
["LidaTube"]=6144
|
||||
["Tdarr"]=6144
|
||||
["Code-Server"]=1024
|
||||
)
|
||||
|
||||
# Containers to check HTTP responsiveness via curl — omit a container to skip its HTTP check.
|
||||
# curl checks the URL and considers the container unresponsive if it times out or errors.
|
||||
|
||||
# HTTP responsiveness checks — omit container to skip its HTTP check
|
||||
declare -A WATCHDOG_CONTAINER_URLS=(
|
||||
["Emby"]="http://localhost:8096"
|
||||
)
|
||||
|
||||
# Containers that should always be running — monitored for unexpected stops.
|
||||
# Strike system used — tries restart on each strike up to SYS_WATCHDOG_STRIKE_LIMIT.
|
||||
# If restart fails after strike limit → added to persistent skip list on /boot/
|
||||
# Skip list auto-clears when container recovers after reboot or manual fix.
|
||||
|
||||
# Containers that must always be running — strike system, persistent skip list on /boot/
|
||||
# Skip list auto-clears when container recovers — no manual intervention for normal recovery
|
||||
WATCHDOG_REQUIRED_CONTAINERS=(
|
||||
"NginxProxyManager"
|
||||
"Lldap-Gmer4Lfe"
|
||||
@@ -416,23 +419,74 @@ WATCHDOG_REQUIRED_CONTAINERS=(
|
||||
"Authelia-Secondary"
|
||||
"Redis-Authelia-Secondary"
|
||||
)
|
||||
|
||||
# Strike state file — /tmp resets on reboot which is correct behaviour for strike tracking
|
||||
|
||||
# Strike thresholds
|
||||
WATCHDOG_STATE_FILE="/tmp/container_watchdog_state.db"
|
||||
|
||||
# CPU thresholds — normalised against total core count automatically at runtime.
|
||||
# A container using 85% of one core on a 16-core system = ~5.3% normalised.
|
||||
SOFT_CPU_THRESHOLD=80 # warn at this % of total system CPU
|
||||
HARD_CPU_THRESHOLD=85 # strike at this % of total system CPU
|
||||
CPU_FAIL_LIMIT=2 # consecutive strikes before container restart
|
||||
|
||||
# Memory soft threshold — warn when container reaches this % of its hard limit.
|
||||
# Hard limit exceeded triggers immediate restart regardless of strikes.
|
||||
SOFT_MEM_THRESHOLD=80
|
||||
|
||||
# HTTP responsiveness check settings
|
||||
RESP_FAIL_LIMIT=2 # consecutive failed curl checks before restart
|
||||
CURL_TIMEOUT=5 # seconds before curl gives up per check
|
||||
SOFT_CPU_THRESHOLD=80 # warn at this % of total system CPU
|
||||
HARD_CPU_THRESHOLD=85 # strike at this % of total system CPU
|
||||
CPU_FAIL_LIMIT=2 # consecutive CPU strikes before restart
|
||||
SOFT_MEM_THRESHOLD=80 # warn when container reaches this % of hard limit
|
||||
RESP_FAIL_LIMIT=2 # consecutive failed HTTP checks before restart
|
||||
CURL_TIMEOUT=5 # seconds before curl gives up per check
|
||||
|
||||
# ── Tier 2 — Global Health Scan ───────────────────────────────────────────────────────────
|
||||
|
||||
# Master toggle — false disables Tier 2 entirely
|
||||
WATCHDOG_SCAN_ALL=true
|
||||
|
||||
# Containers to skip in Tier 2 — add containers expected to be in a non-running state
|
||||
# or managed by other systems that should not be auto-restarted
|
||||
WATCHDOG_SCAN_IGNORE=(
|
||||
# "container-name"
|
||||
)
|
||||
|
||||
# Individual Tier 2 check toggles — disable checks that cause false positives
|
||||
WATCHDOG_RESTART_UNHEALTHY=true # restart containers with unhealthy Docker health status
|
||||
WATCHDOG_RESTART_DEAD=true # remove and restart containers in dead state
|
||||
WATCHDOG_RESTART_CRASHED=true # restart containers that exited with non-zero exit code
|
||||
WATCHDOG_NOTIFY_OOM=true # restart and notify when OOM killed by kernel
|
||||
WATCHDOG_NOTIFY_CRASHLOOP=true # notify when Docker restart count is climbing
|
||||
|
||||
# Crash loop threshold — notify critical if Docker has restarted container this many times
|
||||
WATCHDOG_CRASH_LIMIT=5
|
||||
|
||||
# ── Cross-cutting Intelligence ────────────────────────────────────────────────────────────
|
||||
|
||||
# Startup grace period — skip restarts while system is still booting
|
||||
# Prevents false positives while containers are coming up after array start
|
||||
WATCHDOG_STARTUP_GRACE=300 # seconds after boot before watchdog acts on failures
|
||||
|
||||
# Restart loop protection — stops hammering broken containers
|
||||
# Tracks watchdog-initiated restarts per container in a bounded /boot/ file
|
||||
# After limit hit → container added to skip list → notify critical → manual intervention
|
||||
# Skip list auto-clears when container is found running again
|
||||
WATCHDOG_CONTAINER_RESTART_LIMIT=3 # max watchdog restarts allowed in window
|
||||
WATCHDOG_CONTAINER_RESTART_WINDOW=1 # hours — rolling window for restart count
|
||||
WATCHDOG_CONTAINER_RESTART_LOG="/boot/config/container_restart_history.db"
|
||||
# /boot/ survives reboots — bounded, auto-purges old entries
|
||||
|
||||
# Dependency ordering — skip restarting a container if its dependency is also down
|
||||
# Dependency gets restarted first, dependent picked up on the next watchdog cycle
|
||||
# Format: ["dependent"]="dependency1 dependency2"
|
||||
declare -A WATCHDOG_DEPENDENCIES=(
|
||||
["Authelia"]="Mariadb-Authelia Redis-Authelia"
|
||||
["Authelia-Secondary"]="Mariadb-Authelia Redis-Authelia-Secondary"
|
||||
["NextCloud"]="Postgres-NextCloud"
|
||||
)
|
||||
|
||||
# Notification batching — one clean summary per run instead of one ping per event
|
||||
# true = batch all events into a single notification at end of run
|
||||
# false = send individual notification per event as it happens
|
||||
WATCHDOG_BATCH_NOTIFY=true
|
||||
|
||||
# ━━━ Docker Network Connect ━━━
|
||||
NETWORK_CONNECT_CONTAINERS=(
|
||||
"memcached"
|
||||
"Npm-CrowdSec"
|
||||
)
|
||||
NETWORK_CONNECT_NETWORKS=(
|
||||
"nextcloud-aio"
|
||||
)
|
||||
|
||||
# ━━━ Docker Network Connect ━━━
|
||||
# Connects containers to extra Docker networks on array start.
|
||||
|
||||
Reference in New Issue
Block a user