Gmer4Lfe
0b992a9054
Consistent two-tier output model across the entire folder: - Per-container banners, action lines, and list details → log (ENABLE_LOGGING=true only) - Section headers, summaries, counts, and status conclusions → echo (always visible) - Warnings and errors always visible regardless of log setting - Blank echo lines inside loops removed All scripts: added Lock Acquisition and Host Detection entries to OPERATIONAL SAFEGUARDS. Setup banners removed from docker_watchdog.sh, docker_weekly_restart.sh, downloaders_reset.sh (noise before any work happens). docker_watchdog.sh: removed success "Running as root" / "Docker found" setup lines; info() → log() for daemon-recovered and per-cycle header; removed per-cycle echo separator. docker_update.sh + docker_update_remaining.sh: docker pull stdout suppressed when ENABLE_LOGGING=false to prevent orphaned Status: lines appearing without container context; restructured pull block to use PIPESTATUS for exit code capture. downloaders_reset.sh: 11× info() → log() for all per-item API operation lines. docker_weekly_restart.sh: removed duplicate restart-order echo (build_restart_order already logs it internally). Manual-Docker_Essentials.md: documented the two-tier output model and watchdog silent-when-healthy exception in the flag reference section.
339 lines
13 KiB
Bash
339 lines
13 KiB
Bash
#!/bin/bash
|
|
# ==============================================================================================
|
|
# ================================= Docker Weekly Restart ======================================
|
|
# ==============================================================================================
|
|
#
|
|
# PURPOSE
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Restarts configured containers every Sunday at 2:30am as proactive maintenance.
|
|
#
|
|
# Called by weekly_sync_maintenance.sh via WEEKLY_MAINTENANCE_SCRIPTS. Runs after
|
|
# the sync window has completed and already restarted its own critical containers
|
|
# (Emby, auth stack). Targets a separate set of less-critical services that benefit
|
|
# from a weekly clean start but do not need to be stopped for the sync itself.
|
|
#
|
|
# Same behavioural rules as docker_daily_restart.sh: running → restart,
|
|
# stopped → leave, missing → skip. Container state is always respected.
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL SAFEGUARDS
|
|
# ==============================================================================================
|
|
#
|
|
# Dependency Ordering
|
|
# Containers restart in dependency-safe order using HOST*_WATCHDOG_DEPENDENCIES.
|
|
# CONTAINER_DELAY seconds between dependency restart and dependent restart.
|
|
#
|
|
# Restart Verification
|
|
# Container state checked after a settle period. A container that crashes
|
|
# immediately after restart is marked failed with a notification sent.
|
|
#
|
|
# Timeout Protection
|
|
# All docker commands wrapped in a 30 second timeout. A hung Docker daemon
|
|
# cannot cause this script to hang indefinitely.
|
|
#
|
|
# Host Detection
|
|
# detect_hosts() identifies which server is running the script and aliases
|
|
# HOST*_WEEKLY_RESTART_CONTAINERS and HOST*_WATCHDOG_DEPENDENCIES to the
|
|
# correct host's values.
|
|
#
|
|
# Lock Acquisition
|
|
# acquire_lock() prevents concurrent execution.
|
|
#
|
|
# ==============================================================================================
|
|
# CONFIGURATION
|
|
# ==============================================================================================
|
|
#
|
|
# master_host*.conf
|
|
#
|
|
# HOST*_WEEKLY_RESTART_CONTAINERS
|
|
# Containers restarted weekly. Aliased by detect_hosts() →
|
|
# WEEKLY_RESTART_CONTAINERS
|
|
#
|
|
# HOST*_WATCHDOG_DEPENDENCIES
|
|
# Dependency ordering shared with docker_watchdog.sh. Aliased by
|
|
# detect_hosts() → WATCHDOG_DEPENDENCIES
|
|
#
|
|
# master.conf
|
|
#
|
|
# RETRY_COUNT
|
|
# Retry attempts before giving up on a container
|
|
#
|
|
# SLEEP
|
|
# Seconds between retry attempts
|
|
#
|
|
# CONTAINER_DELAY
|
|
# Seconds to wait after restarting a dependency before starting its dependents
|
|
#
|
|
# ==============================================================================================
|
|
# RUNTIME MODES
|
|
# ==============================================================================================
|
|
#
|
|
# docker_weekly_restart.sh
|
|
# Restart all containers in WEEKLY_RESTART_CONTAINERS
|
|
#
|
|
# docker_weekly_restart.sh --dry-run
|
|
# Preview which containers would be restarted and which would be skipped
|
|
#
|
|
# docker_weekly_restart.sh --status
|
|
# Show configured restart list, container states, and dependency ordering
|
|
#
|
|
# docker_weekly_restart.sh --log
|
|
# Verbose per-container execution output
|
|
#
|
|
# ==============================================================================================
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
source "$SCRIPT_DIR/../load_config.sh"
|
|
|
|
parse_args "$@"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Setup ━━━
|
|
# ==============================================================================================
|
|
if [[ "$EUID" -ne 0 ]]; then
|
|
error "Must be run as root"
|
|
exit 1
|
|
fi
|
|
acquire_lock
|
|
|
|
if ! command -v docker &>/dev/null; then
|
|
error "Docker command not found — check PATH or Docker installation"
|
|
notify "Docker weekly restart failed — Docker not found on $(hostname)" "Docker Weekly Restart" "warning"
|
|
exit 1
|
|
fi
|
|
|
|
# detect_hosts() sets MY_ID and aliases HOST*_WEEKLY_RESTART_CONTAINERS → WEEKLY_RESTART_CONTAINERS
|
|
detect_hosts
|
|
|
|
if [[ ${#WEEKLY_RESTART_CONTAINERS[@]} -eq 0 ]]; then
|
|
warn "WEEKLY_RESTART_CONTAINERS is empty for $MY_ID — nothing to restart"
|
|
warn "Check HOST*_WEEKLY_RESTART_CONTAINERS in master_host*.conf"
|
|
exit 0
|
|
fi
|
|
|
|
log "$MY_ID ($LOCAL_SERVER_NAME) — ${#WEEKLY_RESTART_CONTAINERS[@]} containers configured"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Status ━━━
|
|
# ==============================================================================================
|
|
if [[ "$SHOW_STATUS" == true ]]; then
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_CONTAINERS Containers: ${WEEKLY_RESTART_CONTAINERS[*]}"
|
|
echo "$ICON_RETRY Retries: $RETRY_COUNT"
|
|
echo "$ICON_TIME Sleep: ${SLEEP}s between retries"
|
|
echo "$ICON_NOTIFY Notify: unRAID=${NOTIFY_UNRAID:-false} Discord=$([[ -n "${MY_DISCORD_WEBHOOK:-}" ]] && echo enabled || echo disabled)"
|
|
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
fi
|
|
|
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be restarted"
|
|
|
|
# ==============================================================================================
|
|
# ── FUNCTIONS ─────────────────────────────────────────────────────────────────────────────────
|
|
# ==============================================================================================
|
|
|
|
# Wraps docker commands with a 30 second timeout.
|
|
# Prevents a hung Docker daemon from causing the script to hang indefinitely.
|
|
DOCKER_TIMEOUT=30
|
|
docker_cmd() {
|
|
timeout "$DOCKER_TIMEOUT" "$@"
|
|
local exit_code=$?
|
|
if [[ "$exit_code" -eq 124 ]]; then
|
|
error "Docker command timed out after ${DOCKER_TIMEOUT}s: $*"
|
|
return 1
|
|
fi
|
|
return "$exit_code"
|
|
}
|
|
|
|
# Retries a docker command up to RETRY_COUNT times with SLEEP seconds between attempts.
|
|
# Uses docker_cmd wrapper for timeout protection on each attempt.
|
|
retry_docker() {
|
|
local attempt=1
|
|
|
|
while [[ "$attempt" -le "$RETRY_COUNT" ]]; do
|
|
log "$ICON_RETRY Attempt $attempt of $RETRY_COUNT: $*"
|
|
|
|
if docker_cmd "$@"; then
|
|
log "Succeeded on attempt $attempt"
|
|
return 0
|
|
else
|
|
warn "Attempt $attempt failed"
|
|
(( attempt++ ))
|
|
[[ "$attempt" -le "$RETRY_COUNT" ]] && sleep "$SLEEP"
|
|
fi
|
|
done
|
|
|
|
error "Command failed after $RETRY_COUNT attempts: $*"
|
|
return 1
|
|
}
|
|
|
|
# Verifies a container is still running after restart.
|
|
# Gives the container a short settle period before checking.
|
|
RESTART_VERIFY_WAIT=5
|
|
verify_running() {
|
|
local container="$1"
|
|
sleep "$RESTART_VERIFY_WAIT"
|
|
local state
|
|
state=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
|
|
if [[ "$state" != "true" ]]; then
|
|
error "$container failed to stay running after restart — may have crashed"
|
|
return 1
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
# Builds a dependency-safe restart order from WEEKLY_RESTART_CONTAINERS.
|
|
# Containers that are dependencies of others restart first.
|
|
build_restart_order() {
|
|
ORDERED_RESTART=()
|
|
local remaining=("${WEEKLY_RESTART_CONTAINERS[@]}")
|
|
local placed=()
|
|
|
|
# First pass — add dependency containers that appear in our list
|
|
for container in "${remaining[@]}"; do
|
|
[[ -z "$container" ]] && continue
|
|
local is_dependency=false
|
|
for dependent in "${!WATCHDOG_DEPENDENCIES[@]}"; do
|
|
if [[ "${WATCHDOG_DEPENDENCIES[$dependent]}" == *"$container"* ]]; then
|
|
is_dependency=true
|
|
break
|
|
fi
|
|
done
|
|
if [[ "$is_dependency" == true ]]; then
|
|
local already=false
|
|
for p in "${placed[@]}"; do [[ "$p" == "$container" ]] && already=true && break; done
|
|
if [[ "$already" == false ]]; then
|
|
ORDERED_RESTART+=("$container")
|
|
placed+=("$container")
|
|
fi
|
|
fi
|
|
done
|
|
|
|
# Second pass — add remaining containers (dependents and independents)
|
|
for container in "${remaining[@]}"; do
|
|
[[ -z "$container" ]] && continue
|
|
local already=false
|
|
for p in "${placed[@]}"; do [[ "$p" == "$container" ]] && already=true && break; done
|
|
if [[ "$already" == false ]]; then
|
|
ORDERED_RESTART+=("$container")
|
|
placed+=("$container")
|
|
fi
|
|
done
|
|
|
|
log "Restart order: ${ORDERED_RESTART[*]}"
|
|
}
|
|
|
|
# Waits CONTAINER_DELAY if this container depends on the last restarted one.
|
|
check_dependency_delay() {
|
|
local container="$1"
|
|
local last="$2"
|
|
[[ -z "$last" ]] && return
|
|
local deps="${WATCHDOG_DEPENDENCIES[$container]:-}"
|
|
if [[ -n "$deps" ]] && [[ "$deps" == *"$last"* ]]; then
|
|
log "Waiting ${CONTAINER_DELAY}s — $container depends on $last..."
|
|
sleep "$CONTAINER_DELAY"
|
|
fi
|
|
}
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Weekly Restart ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_CONTAINERS Weekly Restart — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
|
log "$ICON_CONTAINERS Containers: ${WEEKLY_RESTART_CONTAINERS[*]}"
|
|
log "$ICON_RETRY Retries: $RETRY_COUNT"
|
|
|
|
START=$(date +%s)
|
|
FAILED=()
|
|
RESTARTED=()
|
|
SKIPPED=()
|
|
|
|
# Build dependency-safe restart order
|
|
build_restart_order
|
|
|
|
LAST_RESTARTED=""
|
|
|
|
for container in "${ORDERED_RESTART[@]}"; do
|
|
[[ -z "$container" ]] && continue
|
|
log "━━━ $ICON_CONTAINERS $container ━━━"
|
|
|
|
if ! timeout "$DOCKER_TIMEOUT" docker inspect "$container" &>/dev/null; then
|
|
warn "$container does not exist — skipping"
|
|
continue
|
|
fi
|
|
|
|
STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
|
|
|
|
case "$STATUS" in
|
|
true)
|
|
log "$ICON_RUNNING $container is running — restarting..."
|
|
|
|
# Wait if this container depends on the last one restarted
|
|
check_dependency_delay "$container" "$LAST_RESTARTED"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would restart $container"
|
|
RESTARTED+=("$container")
|
|
else
|
|
if retry_docker docker restart "$container"; then
|
|
if verify_running "$container"; then
|
|
log "$ICON_STARTED $container restarted and running ✅"
|
|
RESTARTED+=("$container")
|
|
LAST_RESTARTED="$container"
|
|
else
|
|
error "$container restarted but crashed immediately"
|
|
notify "$container crashed after restart on $(hostname)" "Docker Weekly Restart" "warning"
|
|
FAILED+=("$container")
|
|
fi
|
|
else
|
|
error "Failed to restart $container after $RETRY_COUNT attempts"
|
|
notify "$container failed to restart on $(hostname)" "Docker Weekly Restart" "warning"
|
|
FAILED+=("$container")
|
|
fi
|
|
fi
|
|
;;
|
|
false)
|
|
# Container was stopped — leave it stopped
|
|
log "$ICON_NOT_RUNNING $container is stopped — skipping (respecting stopped state)"
|
|
SKIPPED+=("$container")
|
|
;;
|
|
*)
|
|
error "Unknown status for $container: $STATUS"
|
|
FAILED+=("$container")
|
|
;;
|
|
esac
|
|
|
|
done
|
|
|
|
END=$(date +%s)
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Summary ━━━
|
|
# ==============================================================================================
|
|
echo "━━━━━ $ICON_SUMMARY WEEKLY RESTART SUMMARY ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
|
if [[ ${#RESTARTED[@]} -gt 0 ]]; then
|
|
echo "$ICON_STARTED Restarted: ${#RESTARTED[@]}"
|
|
log " ${RESTARTED[*]}"
|
|
fi
|
|
[[ ${#SKIPPED[@]} -gt 0 ]] && log "$ICON_NOT_RUNNING Skipped: ${#SKIPPED[@]} (were stopped)"
|
|
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
echo "$ICON_WARN Status: DRY RUN — no changes made"
|
|
elif [[ ${#FAILED[@]} -eq 0 ]]; then
|
|
echo "$ICON_DONE Status: $ICON_SUCCESS ALL DONE"
|
|
notify "Weekly restart complete — ${#RESTARTED[@]} restarted, ${#SKIPPED[@]} skipped (stopped) on $(hostname)" "Docker Weekly Restart" "normal"
|
|
else
|
|
echo "$ICON_ERROR Status: $ICON_ERROR ${#FAILED[@]} container(s) failed"
|
|
notify "Weekly restart completed with errors on $(hostname) — failed: ${FAILED[*]}" "Docker Weekly Restart" "warning"
|
|
fi
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
|
|
[[ ${#FAILED[@]} -gt 0 ]] && exit 1
|
|
exit 0 |