massive update. Master conf split, now modular with a load sceriprt to drive all configs to scripts. with unraid scpecific safeguard tests , and improved standardized ux. including dynamic host detect, who am i who else it there. EVERY SINGLE SCRIPT UPDATED. DEBATING THAT THIS IS ACUALLY V2
This commit is contained in:
@@ -1,24 +1,64 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- Docker Daily Restart ---------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Restarts or starts specified Docker containers with retry logic.
|
||||
# Containers are configured in Master.conf under DAILY_RESTART_CONTAINERS.
|
||||
# Uses global RETRY_COUNT and SLEEP from Master.conf for retry behaviour.
|
||||
# Sends notifications on completion or failure via common.sh notify().
|
||||
# Supports --dry-run to preview what would be restarted without taking action.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ================================= Docker Daily Restart =======================================
|
||||
# ==============================================================================================
|
||||
# Restarts or starts all containers in HOST*_DAILY_RESTART_CONTAINERS.
|
||||
# Called by daily_sync_maintenance.sh via DAILY_MAINTENANCE_SCRIPTS every night at 1am.
|
||||
# Can also be run manually for ad hoc restarts.
|
||||
#
|
||||
# ── WHY DAILY RESTARTS ────────────────────────────────────────────────────────────────────────
|
||||
# Some containers degrade over time without a restart:
|
||||
# Dispatcharr — Live TV scheduler accumulates state and slows down
|
||||
# NginxProxyManager — connection table grows, occasional stale proxy entries
|
||||
# Authelia — session cache benefits from periodic clearing
|
||||
# Daily restart is intentional maintenance, not just housekeeping.
|
||||
#
|
||||
# ── BEHAVIOUR ─────────────────────────────────────────────────────────────────────────────────
|
||||
# Running containers → docker restart (graceful stop + start)
|
||||
# Stopped containers → left stopped — was down intentionally, do not bring back up
|
||||
# Missing containers → logged and skipped — not treated as fatal
|
||||
# Each action uses RETRY_COUNT + SLEEP from master.conf for retry logic.
|
||||
#
|
||||
# The "was running → restart, was stopped → leave stopped" rule is consistent
|
||||
# across the entire ecosystem — container state is always respected.
|
||||
#
|
||||
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
|
||||
# Dependency ordering — containers restart in dependency-safe order using
|
||||
# HOST*_WATCHDOG_DEPENDENCIES from master_host*.conf. If Authelia depends on
|
||||
# Mariadb + Redis, those restart first with CONTAINER_DELAY before Authelia starts.
|
||||
#
|
||||
# Restart verification — after each restart, container state is checked after a short
|
||||
# settle period. If the container fails to stay running it is marked as failed and
|
||||
# a notification is sent rather than silently passing.
|
||||
#
|
||||
# Timeout protection — all docker commands are wrapped in a 30 second timeout.
|
||||
# A hung Docker daemon cannot cause this script to hang indefinitely.
|
||||
# Timed-out commands are retried per RETRY_COUNT before marking as failed.
|
||||
#
|
||||
# ── CONFIGURATION (master_host*.conf) ─────────────────────────────────────────────────────────
|
||||
# HOST*_DAILY_RESTART_CONTAINERS — list of containers to restart daily
|
||||
# Set by detect_hosts() alias → DAILY_RESTART_CONTAINERS used by this script
|
||||
#
|
||||
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||||
# RETRY_COUNT — retry attempts before giving up on a container
|
||||
# SLEEP — seconds between retry attempts
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# docker_daily_restart.sh — normal restart
|
||||
# docker_daily_restart.sh --dry-run — preview without restarting
|
||||
# docker_daily_restart.sh --log — verbose output
|
||||
# docker_daily_restart.sh --status — show config and exit
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
|
||||
@@ -26,7 +66,6 @@ if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Running as root"
|
||||
|
||||
acquire_lock
|
||||
@@ -36,30 +75,41 @@ if ! command -v docker &>/dev/null; then
|
||||
notify "Docker daily restart failed — Docker not found on $(hostname)" "Docker Daily Restart" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Docker found"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Status ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# detect_hosts() sets MY_ID and aliases HOST*_DAILY_RESTART_CONTAINERS → DAILY_RESTART_CONTAINERS
|
||||
detect_hosts
|
||||
|
||||
if [[ ${#DAILY_RESTART_CONTAINERS[@]} -eq 0 ]]; then
|
||||
warn "DAILY_RESTART_CONTAINERS is empty for $MY_ID — nothing to restart"
|
||||
warn "Check HOST${MY_ID#HOST}_DAILY_RESTART_CONTAINERS in master_host*.conf"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_CONTAINERS Containers: ${DAILY_RESTART_CONTAINERS[*]}"
|
||||
echo "$ICON_RETRY Retries: $RETRY_COUNT"
|
||||
echo "$ICON_TIME Sleep: ${SLEEP}s between retries"
|
||||
echo "$ICON_NOTIFY Notifications: unRAID=${NOTIFY_UNRAID:-false} Discord=$([[ -n "${DISCORD_WEBHOOK:-}" ]] && echo enabled || echo disabled)"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "$ICON_RETRY Retries: $RETRY_COUNT"
|
||||
echo "$ICON_TIME Sleep: ${SLEEP}s between retries"
|
||||
echo "$ICON_NOTIFY Notify: unRAID=${NOTIFY_UNRAID:-false} Discord=$([[ -n "${MY_DISCORD_WEBHOOK:-}" ]] && echo enabled || echo disabled)"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be restarted"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# FUNCTIONS
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ── FUNCTIONS ─────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# Retries a docker command up to RETRY_COUNT times with SLEEP seconds between attempts.
|
||||
# Usage: retry_docker docker restart ContainerName
|
||||
retry_docker() {
|
||||
local attempt=1
|
||||
|
||||
@@ -80,9 +130,132 @@ retry_docker() {
|
||||
return 1
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_CONTAINERS Daily Restart ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ── FUNCTIONS ─────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# Wraps docker commands with a 30 second timeout.
|
||||
# Prevents a hung Docker daemon from causing the script to hang indefinitely.
|
||||
# Usage: docker_cmd docker restart ContainerName
|
||||
DOCKER_TIMEOUT=30
|
||||
docker_cmd() {
|
||||
timeout "$DOCKER_TIMEOUT" "$@"
|
||||
local exit_code=$?
|
||||
if [[ "$exit_code" -eq 124 ]]; then
|
||||
error "Docker command timed out after ${DOCKER_TIMEOUT}s: $*"
|
||||
return 1
|
||||
fi
|
||||
return "$exit_code"
|
||||
}
|
||||
|
||||
# Verifies a container is still running after restart.
|
||||
# Gives the container a short settle period before checking.
|
||||
# Returns 0 if running, 1 if crashed or stopped.
|
||||
RESTART_VERIFY_WAIT=5 # seconds to wait before checking state post-restart
|
||||
verify_running() {
|
||||
local container="$1"
|
||||
sleep "$RESTART_VERIFY_WAIT"
|
||||
local state
|
||||
state=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
|
||||
if [[ "$state" != "true" ]]; then
|
||||
error "$container failed to stay running after restart — may have crashed"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# Builds a dependency-safe restart order from DAILY_RESTART_CONTAINERS.
|
||||
# Containers that are dependencies of others restart first.
|
||||
# Returns ordered list in ORDERED_RESTART array.
|
||||
build_restart_order() {
|
||||
ORDERED_RESTART=()
|
||||
local remaining=("${DAILY_RESTART_CONTAINERS[@]}")
|
||||
local placed=()
|
||||
|
||||
# First pass — add dependency containers that appear in our list
|
||||
for container in "${remaining[@]}"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
local is_dependency=false
|
||||
# Check if this container is a dependency of any other in our list
|
||||
for dep_string in "${WATCHDOG_DEPENDENCIES[@]:-}"; do
|
||||
if [[ "$dep_string" == *"$container"* ]]; then
|
||||
is_dependency=true
|
||||
break
|
||||
fi
|
||||
done
|
||||
# Also check associative array format
|
||||
for dependent in "${!WATCHDOG_DEPENDENCIES[@]:-}"; do
|
||||
if [[ "${WATCHDOG_DEPENDENCIES[$dependent]}" == *"$container"* ]]; then
|
||||
is_dependency=true
|
||||
break
|
||||
fi
|
||||
done
|
||||
if [[ "$is_dependency" == true ]]; then
|
||||
# Check not already placed
|
||||
local already=false
|
||||
for p in "${placed[@]:-}"; do [[ "$p" == "$container" ]] && already=true && break; done
|
||||
if [[ "$already" == false ]]; then
|
||||
ORDERED_RESTART+=("$container")
|
||||
placed+=("$container")
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
# Second pass — add remaining containers (dependents and independents)
|
||||
for container in "${remaining[@]}"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
local already=false
|
||||
for p in "${placed[@]:-}"; do [[ "$p" == "$container" ]] && already=true && break; done
|
||||
if [[ "$already" == false ]]; then
|
||||
ORDERED_RESTART+=("$container")
|
||||
placed+=("$container")
|
||||
fi
|
||||
done
|
||||
|
||||
log "Restart order: ${ORDERED_RESTART[*]}"
|
||||
}
|
||||
|
||||
# Checks if a container is a dependent of the previously restarted container.
|
||||
# If so, waits CONTAINER_DELAY before restarting to allow dependency to settle.
|
||||
# Usage: check_dependency_delay "$container" "$last_restarted"
|
||||
check_dependency_delay() {
|
||||
local container="$1"
|
||||
local last="$2"
|
||||
[[ -z "$last" ]] && return
|
||||
|
||||
local deps="${WATCHDOG_DEPENDENCIES[$container]:-}"
|
||||
if [[ -n "$deps" ]] && [[ "$deps" == *"$last"* ]]; then
|
||||
info "Waiting ${CONTAINER_DELAY}s — $container depends on $last..."
|
||||
sleep "$CONTAINER_DELAY"
|
||||
fi
|
||||
}
|
||||
|
||||
# Retries a docker command up to RETRY_COUNT times with SLEEP seconds between attempts.
|
||||
# Uses docker_cmd wrapper for timeout protection on each attempt.
|
||||
# Usage: retry_docker docker restart ContainerName
|
||||
retry_docker() {
|
||||
local attempt=1
|
||||
|
||||
while [[ "$attempt" -le "$RETRY_COUNT" ]]; do
|
||||
info "$ICON_RETRY Attempt $attempt of $RETRY_COUNT: $*"
|
||||
|
||||
if docker_cmd "$@"; then
|
||||
success "Succeeded on attempt $attempt"
|
||||
return 0
|
||||
else
|
||||
warn "Attempt $attempt failed"
|
||||
(( attempt++ ))
|
||||
[[ "$attempt" -le "$RETRY_COUNT" ]] && sleep "$SLEEP"
|
||||
fi
|
||||
done
|
||||
|
||||
error "Command failed after $RETRY_COUNT attempts: $*"
|
||||
return 1
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Daily Restart ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_CONTAINERS Daily Restart — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo "$ICON_CONTAINERS Containers: ${DAILY_RESTART_CONTAINERS[*]}"
|
||||
@@ -92,30 +265,49 @@ echo ""
|
||||
START=$(date +%s)
|
||||
FAILED=()
|
||||
RESTARTED=()
|
||||
STARTED=()
|
||||
SKIPPED=()
|
||||
|
||||
for container in "${DAILY_RESTART_CONTAINERS[@]}"; do
|
||||
# Build dependency-safe restart order
|
||||
build_restart_order
|
||||
echo "$ICON_GEAR Restart order: ${ORDERED_RESTART[*]}"
|
||||
echo ""
|
||||
|
||||
LAST_RESTARTED=""
|
||||
|
||||
for container in "${ORDERED_RESTART[@]}"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
echo "━━━ $ICON_CONTAINERS $container ━━━"
|
||||
|
||||
if ! docker inspect "$container" &>/dev/null; then
|
||||
error "$container does not exist — skipping"
|
||||
FAILED+=("$container")
|
||||
if ! timeout "$DOCKER_TIMEOUT" docker inspect "$container" &>/dev/null; then
|
||||
warn "$container does not exist — skipping"
|
||||
echo ""
|
||||
continue
|
||||
fi
|
||||
|
||||
STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
|
||||
STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
|
||||
|
||||
case "$STATUS" in
|
||||
true)
|
||||
echo "$ICON_RUNNING $container is running — restarting..."
|
||||
|
||||
# Wait if this container depends on the last one restarted
|
||||
check_dependency_delay "$container" "$LAST_RESTARTED"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would restart $container"
|
||||
RESTARTED+=("$container")
|
||||
else
|
||||
if retry_docker docker restart "$container"; then
|
||||
echo "$ICON_STARTED $container restarted"
|
||||
RESTARTED+=("$container")
|
||||
# Verify container stayed running after restart
|
||||
if verify_running "$container"; then
|
||||
echo "$ICON_STARTED $container restarted and running ✅"
|
||||
RESTARTED+=("$container")
|
||||
LAST_RESTARTED="$container"
|
||||
else
|
||||
error "$container restarted but crashed immediately"
|
||||
notify "$container crashed after restart on $(hostname)" "Docker Daily Restart" "warning"
|
||||
FAILED+=("$container")
|
||||
fi
|
||||
else
|
||||
error "Failed to restart $container after $RETRY_COUNT attempts"
|
||||
notify "$container failed to restart on $(hostname)" "Docker Daily Restart" "warning"
|
||||
@@ -124,20 +316,10 @@ for container in "${DAILY_RESTART_CONTAINERS[@]}"; do
|
||||
fi
|
||||
;;
|
||||
false)
|
||||
echo "$ICON_NOT_RUNNING $container is stopped — starting..."
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would start $container"
|
||||
else
|
||||
if retry_docker docker start "$container"; then
|
||||
echo "$ICON_STARTED $container started"
|
||||
STARTED+=("$container")
|
||||
else
|
||||
error "Failed to start $container after $RETRY_COUNT attempts"
|
||||
notify "$container failed to start on $(hostname)" "Docker Daily Restart" "warning"
|
||||
FAILED+=("$container")
|
||||
fi
|
||||
fi
|
||||
# Container was stopped — leave it stopped
|
||||
# Intentionally stopped containers are not restarted
|
||||
echo "$ICON_NOT_RUNNING $container is stopped — skipping (respecting stopped state)"
|
||||
SKIPPED+=("$container")
|
||||
;;
|
||||
*)
|
||||
error "Unknown status for $container: $STATUS"
|
||||
@@ -150,20 +332,21 @@ done
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Summary ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo "━━━━━ $ICON_SUMMARY DAILY RESTART SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
||||
[[ ${#RESTARTED[@]} -gt 0 ]] && echo "$ICON_STARTED Restarted: ${RESTARTED[*]}"
|
||||
[[ ${#STARTED[@]} -gt 0 ]] && echo "$ICON_STARTED Started: ${STARTED[*]}"
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}"
|
||||
[[ ${#RESTARTED[@]} -gt 0 ]] && echo "$ICON_STARTED Restarted: ${RESTARTED[*]}"
|
||||
[[ ${#SKIPPED[@]} -gt 0 ]] && echo "$ICON_NOT_RUNNING Skipped: ${SKIPPED[*]} (were stopped)"
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "$ICON_WARN Status: DRY RUN — no changes made"
|
||||
elif [[ ${#FAILED[@]} -eq 0 ]]; then
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS ALL DONE"
|
||||
notify "Daily restart complete — ${#RESTARTED[@]} restarted, ${#STARTED[@]} started on $(hostname)" "Docker Daily Restart" "normal"
|
||||
notify "Daily restart complete — ${#RESTARTED[@]} restarted, ${#SKIPPED[@]} skipped (stopped) on $(hostname)" "Docker Daily Restart" "normal"
|
||||
else
|
||||
echo "$ICON_ERROR Status: $ICON_ERROR ${#FAILED[@]} container(s) failed"
|
||||
notify "Daily restart completed with errors on $(hostname) — failed: ${FAILED[*]}" "Docker Daily Restart" "warning"
|
||||
|
||||
Reference in New Issue
Block a user