Bring script headers onto the template and close safeguard gaps
Headers claimed protections the code never had, and several destructive paths had no guard against a collapsed config value.
This commit is contained in:
Gmer4Lfe
@@ -68,6 +68,15 @@
|
||||
# Docker Presence Check
|
||||
# Verifies docker binary exists before execution.
|
||||
#
|
||||
# Docker Daemon Check
|
||||
# Verifies the daemon is responsive before enumerating containers. A hung
|
||||
# daemon returns an empty container list, which would otherwise be read as
|
||||
# "nothing to stop" and pass a shutdown that never happened.
|
||||
#
|
||||
# Host Detection
|
||||
# detect_hosts() identifies which server is running the script and sets
|
||||
# MY_ID / LOCAL_SERVER_NAME for logging and notifications.
|
||||
#
|
||||
# Lock Acquisition
|
||||
# acquire_lock prevents overlapping runs (e.g. array_stopping firing twice).
|
||||
#
|
||||
@@ -140,10 +149,20 @@ fi
|
||||
|
||||
detect_hosts
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be stopped"
|
||||
|
||||
DOCKER_TIMEOUT=30
|
||||
DOCKER_STOP_TIMEOUT=30 # grace period for SIGTERM before docker sends SIGKILL internally
|
||||
|
||||
# A hung daemon makes docker ps return nothing — indistinguishable from "no containers
|
||||
# running", which would silently report a clean shutdown that never happened.
|
||||
if ! timeout "$DOCKER_TIMEOUT" docker info >/dev/null 2>&1; then
|
||||
error "Docker daemon not responding — cannot verify container shutdown"
|
||||
notify "Container stop aborted on $(hostname) ($MY_ID) — Docker daemon not responding" \
|
||||
"Docker Container Stop" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be stopped"
|
||||
|
||||
_RETRY_COUNT="${RETRY_COUNT:-3}"
|
||||
|
||||
log "$ICON_GEAR Config: retries=$_RETRY_COUNT sleep=${SLEEP:-5}s grace=${DOCKER_STOP_TIMEOUT}s cmd-timeout=${DOCKER_TIMEOUT}s"
|
||||
@@ -152,7 +171,7 @@ log "$ICON_GEAR Config: retries=$_RETRY_COUNT sleep=${SLEEP:-5}s grace=${DOCKER_
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
mapfile -t RUNNING < <(docker ps --format '{{.Names}}' 2>/dev/null | sort)
|
||||
mapfile -t RUNNING < <(timeout "$DOCKER_TIMEOUT" docker ps --format '{{.Names}}' 2>/dev/null | sort)
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
@@ -166,7 +185,7 @@ fi
|
||||
# ==============================================================================================
|
||||
# ━━━ Stop Containers ━━━
|
||||
# ==============================================================================================
|
||||
mapfile -t RUNNING < <(docker ps --format '{{.Names}}' 2>/dev/null | sort)
|
||||
mapfile -t RUNNING < <(timeout "$DOCKER_TIMEOUT" docker ps --format '{{.Names}}' 2>/dev/null | sort)
|
||||
|
||||
echo ""
|
||||
echo "━━━ $ICON_CONTAINERS Docker Container Stop — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
@@ -187,7 +206,7 @@ FAILED=()
|
||||
for container in "${RUNNING[@]}"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
c_start=$(date +%s)
|
||||
c_image=$(docker inspect --format '{{.Config.Image}}' "$container" 2>/dev/null || echo "unknown")
|
||||
c_image=$(timeout "$DOCKER_TIMEOUT" docker inspect --format '{{.Config.Image}}' "$container" 2>/dev/null || echo "unknown")
|
||||
log "━━━ $ICON_CONTAINERS $container ($c_image) ━━━"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
|
||||
@@ -14,6 +14,33 @@
|
||||
# so there is no second list to maintain.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Containers are processed one at a time in dependency-safe order:
|
||||
#
|
||||
# 1. Build restart order
|
||||
# → build_restart_order() sorts DAILY_RESTART_CONTAINERS by WATCHDOG_DEPENDENCIES
|
||||
#
|
||||
# 2. Skip anything docker_update.sh already rebuilt this run
|
||||
# → a rebuild onto a new image already restarted it moments ago
|
||||
#
|
||||
# 3. Inspect container state
|
||||
# missing → skip, not an error
|
||||
# stopped → skip, stopped state is respected
|
||||
# running → restart
|
||||
#
|
||||
# 4. Restart with retry
|
||||
# → retry_docker wraps each attempt in a timeout, up to RETRY_COUNT
|
||||
#
|
||||
# 5. Verify it stayed running
|
||||
# → verify_running() settles for RESTART_VERIFY_WAIT then checks State.Running
|
||||
# → a container that crashes immediately is marked failed and notified
|
||||
#
|
||||
# 6. Prune dangling images
|
||||
# → restarts swap onto new images, leaving the old ones dangling
|
||||
#
|
||||
# ==============================================================================================
|
||||
# DESIGN PRINCIPLES
|
||||
# ==============================================================================================
|
||||
#
|
||||
@@ -37,6 +64,27 @@
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Root Enforcement
|
||||
# Docker operations require root privileges.
|
||||
#
|
||||
# Docker Presence Check
|
||||
# Verifies the docker binary exists before execution. Notifies on absence —
|
||||
# a missing binary during the maintenance window is worth knowing about.
|
||||
#
|
||||
# Docker Daemon Check
|
||||
# Verifies the daemon is responsive before any restart work. Every container
|
||||
# would otherwise fail its inspect and be logged as an unknown-status failure,
|
||||
# burying one daemon fault under a list of bogus per-container errors.
|
||||
#
|
||||
# Host Detection
|
||||
# detect_hosts() identifies which server is running the script and aliases
|
||||
# HOST*_DAILY_RESTART_CONTAINERS and HOST*_WATCHDOG_DEPENDENCIES to the
|
||||
# correct host's values.
|
||||
#
|
||||
# Empty List Guard
|
||||
# Exits cleanly with a pointer to the relevant conf key if
|
||||
# DAILY_RESTART_CONTAINERS is unconfigured for this host.
|
||||
#
|
||||
# Dependency Ordering
|
||||
# Containers restart in dependency-safe order using HOST*_WATCHDOG_DEPENDENCIES.
|
||||
# CONTAINER_DELAY seconds between dependency restart and dependent restart gives
|
||||
@@ -53,6 +101,11 @@
|
||||
# cannot cause this script to hang indefinitely. Timed-out commands retry
|
||||
# per RETRY_COUNT before marking as failed.
|
||||
#
|
||||
# Stale Rebuild-List Guard
|
||||
# The rebuilt-container list written by docker_update.sh is discarded if older
|
||||
# than DOCKER_UPDATE_REBUILT_STALE_HOURS. A stale file would otherwise suppress
|
||||
# real restarts based on an update run that never happened today.
|
||||
#
|
||||
# Lock Acquisition
|
||||
# acquire_lock() prevents concurrent execution if a previous run is still active.
|
||||
#
|
||||
@@ -134,6 +187,14 @@ fi
|
||||
# detect_hosts() sets MY_ID and aliases HOST*_DAILY_RESTART_CONTAINERS → DAILY_RESTART_CONTAINERS
|
||||
detect_hosts
|
||||
|
||||
# Without this, a hung daemon fails every container's inspect individually and the summary
|
||||
# reports a list of unknown-status failures instead of the one fault that caused them.
|
||||
if ! timeout "$DOCKER_TIMEOUT" docker info >/dev/null 2>&1; then
|
||||
error "Docker daemon not responding — skipping daily restart"
|
||||
notify "Daily restart skipped on $(hostname) — Docker daemon not responding" "Docker Daily Restart" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ${#DAILY_RESTART_CONTAINERS[@]} -eq 0 ]]; then
|
||||
warn "DAILY_RESTART_CONTAINERS is empty for $MY_ID — nothing to restart"
|
||||
warn "Check HOST${MY_ID#HOST}_DAILY_RESTART_CONTAINERS in host*.conf"
|
||||
@@ -213,7 +274,7 @@ LAST_RESTARTED=""
|
||||
for container in "${ORDERED_RESTART[@]}"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
c_start=$(date +%s)
|
||||
c_image=$(docker inspect --format '{{.Config.Image}}' "$container" 2>/dev/null || echo "unknown")
|
||||
c_image=$(timeout "$DOCKER_TIMEOUT" docker inspect --format '{{.Config.Image}}' "$container" 2>/dev/null || echo "unknown")
|
||||
log "━━━ $ICON_CONTAINERS $container ($c_image) ━━━"
|
||||
|
||||
if ! timeout "$DOCKER_TIMEOUT" docker inspect "$container" &>/dev/null; then
|
||||
@@ -242,7 +303,6 @@ for container in "${ORDERED_RESTART[@]}"; do
|
||||
RESTARTED+=("$container")
|
||||
else
|
||||
if retry_docker docker restart "$container"; then
|
||||
[[ "${RESTART_VERIFY_WAIT:-3}" -gt 0 ]] && sleep "${RESTART_VERIFY_WAIT:-3}"
|
||||
if verify_running "$container"; then
|
||||
echo "$ICON_STARTED $container restarted and running in $(format_duration $(( $(date +%s) - c_start ))) ✅"
|
||||
RESTARTED+=("$container")
|
||||
@@ -283,7 +343,7 @@ if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would prune dangling images"
|
||||
PRUNED_SUMMARY="(dry run)"
|
||||
else
|
||||
PRUNED_OUTPUT=$(docker image prune -f 2>&1)
|
||||
PRUNED_OUTPUT=$(timeout "$DOCKER_TIMEOUT" docker image prune -f 2>&1)
|
||||
[[ "$ENABLE_LOGGING" == "true" ]] && echo "$PRUNED_OUTPUT" | sed 's/^/ /'
|
||||
PRUNED_SUMMARY=$(echo "$PRUNED_OUTPUT" | grep -E "^Total reclaimed" || echo "nothing reclaimed")
|
||||
fi
|
||||
|
||||
@@ -51,6 +51,12 @@
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Root Enforcement
|
||||
# Docker network operations require root privileges.
|
||||
#
|
||||
# Docker Presence Check
|
||||
# Verifies the docker binary exists before any network operations.
|
||||
#
|
||||
# Lock Acquisition
|
||||
# Prevents concurrent execution via acquire_lock(). Safe to call from
|
||||
# array start hooks or manually without risk of overlap.
|
||||
@@ -71,8 +77,10 @@
|
||||
# Warns and exits cleanly if NETWORK_CONNECT_NETWORKS or
|
||||
# NETWORK_CONNECT_CONTAINERS are unconfigured.
|
||||
#
|
||||
# Command Validation
|
||||
# Validates unRAID notify script before use.
|
||||
# Missing Container Tolerance
|
||||
# A configured container that does not exist yet warns and is skipped rather
|
||||
# than failing the run. This script executes early at array start, before
|
||||
# every container has necessarily been created.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
@@ -130,13 +138,12 @@ fi
|
||||
# detect_hosts() sets MY_ID and aliases HOST*_NETWORK_CONNECT_* arrays
|
||||
detect_hosts
|
||||
|
||||
# Validate unRAID notify script — used for network creation alerts
|
||||
|
||||
# Docker daemon check — network operations are useless if daemon is hung
|
||||
DOCKER_TIMEOUT=15
|
||||
if ! timeout "$DOCKER_TIMEOUT" docker info >/dev/null 2>&1; then
|
||||
error "Docker daemon not responding — cannot manage networks"
|
||||
notify "docker_network_connect failed on $(hostname) — Docker daemon not responding" "Network Connect" "warning"
|
||||
notify "docker_network_connect failed on $(hostname) — Docker daemon not responding" \
|
||||
"Network Connect" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -291,7 +298,8 @@ if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no changes made"
|
||||
elif [[ ${#FAILED[@]} -gt 0 ]]; then
|
||||
echo "$ICON_ERROR Status: SOME OPERATIONS FAILED"
|
||||
notify "Docker network connect failed on $(hostname) — ${FAILED[*]}" "Network Connect" "warning"
|
||||
notify "Docker network connect failed on $(hostname) — ${FAILED[*]}" \
|
||||
"Network Connect" "warning"
|
||||
elif [[ ${#NETWORKS_CREATED[@]} -gt 0 ]]; then
|
||||
warn "Networks recreated — ${NETWORKS_CREATED[*]} — unRAID update likely wiped them"
|
||||
else
|
||||
|
||||
@@ -74,6 +74,28 @@
|
||||
# Root Enforcement
|
||||
# Docker operations require root privileges.
|
||||
#
|
||||
# Docker Presence Check
|
||||
# Verifies the docker binary exists before execution.
|
||||
#
|
||||
# Docker Daemon Check
|
||||
# Verifies the daemon is responsive before container discovery. Remainder mode
|
||||
# derives its entire target list from docker ps — against a hung daemon that
|
||||
# returns empty and the run silently reports "no containers to update".
|
||||
#
|
||||
# Timeout Protection
|
||||
# Inspect, discovery and image-query commands are wrapped in a timeout so a
|
||||
# hung daemon cannot stall the maintenance window. docker pull is deliberately
|
||||
# NOT wrapped — a large image legitimately takes longer than any sane timeout,
|
||||
# and killing it mid-layer wastes the transfer.
|
||||
#
|
||||
# Empty List Guards
|
||||
# Each mode exits cleanly with a pointer to the relevant conf key when its
|
||||
# container list is unconfigured for this host.
|
||||
#
|
||||
# Rebuild Failure Fallback
|
||||
# A container that fails to rebuild is excluded from the rebuilt-list handoff
|
||||
# file, so the follow-up restart script still gives it a normal restart pass.
|
||||
#
|
||||
# DAILY_CONTAINER_UPDATES / WEEKLY_CONTAINER_UPDATES Toggles
|
||||
# Each mode exits cleanly when disabled. Restart scripts run regardless —
|
||||
# update and restart are independent operations.
|
||||
@@ -192,6 +214,14 @@ fi
|
||||
|
||||
detect_hosts
|
||||
|
||||
# Remainder mode builds its whole target list from docker ps — a hung daemon returns
|
||||
# empty and the run would report "no containers to update" instead of failing.
|
||||
if ! timeout "$DOCKER_TIMEOUT" docker info >/dev/null 2>&1; then
|
||||
error "Docker daemon not responding — skipping image updates"
|
||||
notify "Docker update skipped on $(hostname) — Docker daemon not responding" "Docker Update" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Container Discovery ━━━
|
||||
# ==============================================================================================
|
||||
@@ -234,7 +264,7 @@ if [[ "$REMAINDER_MODE" == true ]]; then
|
||||
done
|
||||
unset _tier _tier_var _tier_arr _c
|
||||
|
||||
mapfile -t _all_running < <(docker ps --format '{{.Names}}' | sort)
|
||||
mapfile -t _all_running < <(timeout "$DOCKER_TIMEOUT" docker ps --format '{{.Names}}' | sort)
|
||||
TARGET_CONTAINERS=()
|
||||
for _c in "${_all_running[@]}"; do
|
||||
[[ -z "${_exclude[$_c]+x}" ]] && TARGET_CONTAINERS+=("$_c")
|
||||
@@ -328,13 +358,13 @@ for container in "${TARGET_CONTAINERS[@]}"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
log "━━━ $ICON_CONTAINERS $container ━━━"
|
||||
|
||||
if ! docker inspect "$container" &>/dev/null; then
|
||||
if ! timeout "$DOCKER_TIMEOUT" docker inspect "$container" &>/dev/null; then
|
||||
warn "$container — not found, skipping"
|
||||
SKIPPED+=("$container")
|
||||
continue
|
||||
fi
|
||||
|
||||
IMAGE=$(docker inspect --format='{{.Config.Image}}' "$container" 2>/dev/null)
|
||||
IMAGE=$(timeout "$DOCKER_TIMEOUT" docker inspect --format='{{.Config.Image}}' "$container" 2>/dev/null)
|
||||
if [[ -z "$IMAGE" ]]; then
|
||||
warn "$container — could not determine image, skipping"
|
||||
SKIPPED+=("$container")
|
||||
@@ -352,8 +382,8 @@ for container in "${TARGET_CONTAINERS[@]}"; do
|
||||
# Capture the image ID the container is currently running on, and the
|
||||
# image ID :latest points to before the pull. After pulling, we rebuild if
|
||||
# either a new digest landed OR the container is behind what :latest is now.
|
||||
CONTAINER_IMAGE_ID=$(docker inspect "$container" --format='{{.Image}}' 2>/dev/null || echo "")
|
||||
OLD_ID=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "")
|
||||
CONTAINER_IMAGE_ID=$(timeout "$DOCKER_TIMEOUT" docker inspect "$container" --format='{{.Image}}' 2>/dev/null || echo "")
|
||||
OLD_ID=$(timeout "$DOCKER_TIMEOUT" docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "")
|
||||
|
||||
log "$ICON_SYNC Pulling $IMAGE..."
|
||||
if [[ "$ENABLE_LOGGING" == "true" ]]; then
|
||||
@@ -363,7 +393,7 @@ for container in "${TARGET_CONTAINERS[@]}"; do
|
||||
docker pull "$IMAGE" >/dev/null 2>&1
|
||||
_pull_rc=$?
|
||||
fi
|
||||
NEW_ID=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "")
|
||||
NEW_ID=$(timeout "$DOCKER_TIMEOUT" docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "")
|
||||
|
||||
if [[ $_pull_rc -eq 0 ]]; then
|
||||
_pull_new=$( [[ -n "$OLD_ID" && "$OLD_ID" != "$NEW_ID" ]] && echo true || echo false)
|
||||
@@ -436,9 +466,9 @@ if [[ "$DRY_RUN" == true ]]; then
|
||||
PRUNED_SUMMARY="(dry run)"
|
||||
else
|
||||
for _old_id in "${OLD_IMAGE_IDS[@]}"; do
|
||||
docker rmi "$_old_id" >/dev/null 2>&1 || true
|
||||
timeout "$DOCKER_TIMEOUT" docker rmi "$_old_id" >/dev/null 2>&1 || true
|
||||
done
|
||||
PRUNED_OUTPUT=$(docker image prune -f 2>&1)
|
||||
PRUNED_OUTPUT=$(timeout "$DOCKER_TIMEOUT" docker image prune -f 2>&1)
|
||||
[[ "$ENABLE_LOGGING" == "true" ]] && echo "$PRUNED_OUTPUT" | sed 's/^/ /'
|
||||
PRUNED_SUMMARY=$(echo "$PRUNED_OUTPUT" | grep -E "^Total reclaimed" || echo "nothing reclaimed")
|
||||
fi
|
||||
|
||||
@@ -16,6 +16,33 @@
|
||||
# stopped → leave, missing → skip. Container state is always respected.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Identical to docker_daily_restart.sh, against WEEKLY_RESTART_CONTAINERS:
|
||||
#
|
||||
# 1. Build restart order
|
||||
# → build_restart_order() sorts WEEKLY_RESTART_CONTAINERS by WATCHDOG_DEPENDENCIES
|
||||
#
|
||||
# 2. Skip anything docker_update.sh --weekly already rebuilt this run
|
||||
# → a rebuild onto a new image already restarted it moments ago
|
||||
#
|
||||
# 3. Inspect container state
|
||||
# missing → skip, not an error
|
||||
# stopped → skip, stopped state is respected
|
||||
# running → restart
|
||||
#
|
||||
# 4. Restart with retry
|
||||
# → retry_docker wraps each attempt in a timeout, up to RETRY_COUNT
|
||||
#
|
||||
# 5. Verify it stayed running
|
||||
# → verify_running() settles for RESTART_VERIFY_WAIT then checks State.Running
|
||||
# → a container that crashes immediately is marked failed and notified
|
||||
#
|
||||
# 6. Prune dangling images
|
||||
# → restarts swap onto new images, leaving the old ones dangling
|
||||
#
|
||||
# ==============================================================================================
|
||||
# DESIGN PRINCIPLES
|
||||
# ==============================================================================================
|
||||
#
|
||||
@@ -39,6 +66,26 @@
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Root Enforcement
|
||||
# Docker operations require root privileges.
|
||||
#
|
||||
# Docker Presence Check
|
||||
# Verifies the docker binary exists before execution. Notifies on absence.
|
||||
#
|
||||
# Docker Daemon Check
|
||||
# Verifies the daemon is responsive before any restart work. Every container
|
||||
# would otherwise fail its inspect and be logged as an unknown-status failure,
|
||||
# burying one daemon fault under a list of bogus per-container errors.
|
||||
#
|
||||
# Host Detection
|
||||
# detect_hosts() identifies which server is running the script and aliases
|
||||
# HOST*_WEEKLY_RESTART_CONTAINERS and HOST*_WATCHDOG_DEPENDENCIES to the
|
||||
# correct host's values.
|
||||
#
|
||||
# Empty List Guard
|
||||
# Exits cleanly with a pointer to the relevant conf key if
|
||||
# WEEKLY_RESTART_CONTAINERS is unconfigured for this host.
|
||||
#
|
||||
# Dependency Ordering
|
||||
# Containers restart in dependency-safe order using HOST*_WATCHDOG_DEPENDENCIES.
|
||||
# CONTAINER_DELAY seconds between dependency restart and dependent restart.
|
||||
@@ -51,10 +98,10 @@
|
||||
# All docker commands wrapped in a 30 second timeout. A hung Docker daemon
|
||||
# cannot cause this script to hang indefinitely.
|
||||
#
|
||||
# Host Detection
|
||||
# detect_hosts() identifies which server is running the script and aliases
|
||||
# HOST*_WEEKLY_RESTART_CONTAINERS and HOST*_WATCHDOG_DEPENDENCIES to the
|
||||
# correct host's values.
|
||||
# Stale Rebuild-List Guard
|
||||
# The rebuilt-container list written by docker_update.sh --weekly is discarded
|
||||
# if older than DOCKER_UPDATE_REBUILT_STALE_HOURS. A stale file would otherwise
|
||||
# suppress real restarts based on an update run that never happened this week.
|
||||
#
|
||||
# Lock Acquisition
|
||||
# acquire_lock() prevents concurrent execution.
|
||||
@@ -84,6 +131,11 @@
|
||||
# CONTAINER_DELAY
|
||||
# Seconds to wait after restarting a dependency before starting its dependents
|
||||
#
|
||||
# RESTART_VERIFY_WAIT
|
||||
# Seconds verify_running() waits after docker restart before checking the
|
||||
# container is running. Gives the process time to initialise before the
|
||||
# state is sampled. (default: 3)
|
||||
#
|
||||
# DOCKER_UPDATE_REBUILT_WEEKLY_FILE / DOCKER_UPDATE_REBUILT_STALE_HOURS
|
||||
# List of containers docker_update.sh --weekly already rebuilt onto a new image
|
||||
# this run — read here so they're not restarted a second time. Discarded as
|
||||
@@ -132,6 +184,14 @@ fi
|
||||
# detect_hosts() sets MY_ID and aliases HOST*_WEEKLY_RESTART_CONTAINERS → WEEKLY_RESTART_CONTAINERS
|
||||
detect_hosts
|
||||
|
||||
# Without this, a hung daemon fails every container's inspect individually and the summary
|
||||
# reports a list of unknown-status failures instead of the one fault that caused them.
|
||||
if ! timeout "$DOCKER_TIMEOUT" docker info >/dev/null 2>&1; then
|
||||
error "Docker daemon not responding — skipping weekly restart"
|
||||
notify "Weekly restart skipped on $(hostname) — Docker daemon not responding" "Docker Weekly Restart" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ${#WEEKLY_RESTART_CONTAINERS[@]} -eq 0 ]]; then
|
||||
warn "WEEKLY_RESTART_CONTAINERS is empty for $MY_ID — nothing to restart"
|
||||
warn "Check HOST*_WEEKLY_RESTART_CONTAINERS in host*.conf"
|
||||
@@ -211,7 +271,7 @@ LAST_RESTARTED=""
|
||||
for container in "${ORDERED_RESTART[@]}"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
c_start=$(date +%s)
|
||||
c_image=$(docker inspect --format '{{.Config.Image}}' "$container" 2>/dev/null || echo "unknown")
|
||||
c_image=$(timeout "$DOCKER_TIMEOUT" docker inspect --format '{{.Config.Image}}' "$container" 2>/dev/null || echo "unknown")
|
||||
log "━━━ $ICON_CONTAINERS $container ($c_image) ━━━"
|
||||
|
||||
if ! timeout "$DOCKER_TIMEOUT" docker inspect "$container" &>/dev/null; then
|
||||
@@ -281,7 +341,7 @@ if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would prune dangling images"
|
||||
PRUNED_SUMMARY="(dry run)"
|
||||
else
|
||||
PRUNED_OUTPUT=$(docker image prune -f 2>&1)
|
||||
PRUNED_OUTPUT=$(timeout "$DOCKER_TIMEOUT" docker image prune -f 2>&1)
|
||||
[[ "$ENABLE_LOGGING" == "true" ]] && echo "$PRUNED_OUTPUT" | sed 's/^/ /'
|
||||
PRUNED_SUMMARY=$(echo "$PRUNED_OUTPUT" | grep -E "^Total reclaimed" || echo "nothing reclaimed")
|
||||
fi
|
||||
|
||||
@@ -61,6 +61,15 @@
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Root Enforcement
|
||||
# Failed-import purging deletes directories owned by container users.
|
||||
#
|
||||
# Dependency Check
|
||||
# Verifies curl and jq exist before any API work. jq backs the slskd connection
|
||||
# probe — without it the probe returns false forever, the script burns its full
|
||||
# 60 second reconnect wait, then skips every slskd section as "disconnected".
|
||||
# A missing dependency is reported as itself rather than as a phantom outage.
|
||||
#
|
||||
# Active Transfer Protection
|
||||
# slskd: skips users with InProgress or Queued transfers before any removal.
|
||||
# SABnzbd: age threshold enforced before deletion.
|
||||
@@ -70,6 +79,19 @@
|
||||
# Each section validates its downloader URL before API calls. Missing or
|
||||
# unreachable downloaders skip without affecting other sections.
|
||||
#
|
||||
# No Downloaders Guard
|
||||
# Exits cleanly when none of SLSKD_URL, SABNZBD_URL or QBIT_URL are set for
|
||||
# this host — nothing configured is not an error.
|
||||
#
|
||||
# Timeout Protection
|
||||
# Every curl carries --max-time. An unresponsive downloader cannot stall the
|
||||
# 30 minute maintenance cycle or overlap the next run.
|
||||
#
|
||||
# Deletion Scope Limit
|
||||
# qBittorrent removals pass deleteFiles=false — the torrent record is dropped
|
||||
# but files on disk are left for the arrs to manage. This script never deletes
|
||||
# media.
|
||||
#
|
||||
# Host Detection
|
||||
# detect_hosts() identifies which server is running the script and aliases
|
||||
# all HOST*_SLSKD_*, HOST*_SABNZBD_*, and HOST*_QBIT_* vars to the correct
|
||||
@@ -140,6 +162,17 @@ fi
|
||||
# Lock first — wait mode since this runs every 30min and previous may still be finishing
|
||||
acquire_lock "wait"
|
||||
|
||||
# jq backs the slskd connection probe. Missing, the probe never returns true and slskd
|
||||
# looks permanently disconnected — a 60s wait followed by silently skipped sections.
|
||||
for _dep in curl jq; do
|
||||
if ! command -v "$_dep" &>/dev/null; then
|
||||
error "$_dep not found — required for downloader API calls"
|
||||
notify "Downloaders reset failed on $(hostname) — $_dep not installed" "Downloaders Reset" "warning"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
unset _dep
|
||||
|
||||
# detect_hosts() sets MY_ID and aliases all HOST*_SLSKD_*, HOST*_SABNZBD_*, HOST*_QBIT_* vars
|
||||
detect_hosts
|
||||
|
||||
|
||||
Reference in New Issue
Block a user