Varaverk/Docker_Essentials/docker_daily_restart.sh

#!/bin/bash
# ==============================================================================================
# ================================= Docker Daily Restart =======================================
# ==============================================================================================
# Restarts or starts all containers in HOST*_DAILY_RESTART_CONTAINERS.
# Called by daily_sync_maintenance.sh via DAILY_MAINTENANCE_SCRIPTS every night at 1am.
# Can also be run manually for ad hoc restarts.
#
# ── WHY DAILY RESTARTS ────────────────────────────────────────────────────────────────────────
# Some containers degrade over time without a restart:
#   Dispatcharr — Live TV scheduler accumulates state and slows down
#   NginxProxyManager — connection table grows, occasional stale proxy entries
#   Authelia — session cache benefits from periodic clearing
# Daily restart is intentional maintenance, not just housekeeping.
#
# ── BEHAVIOUR ─────────────────────────────────────────────────────────────────────────────────
# Running containers → docker restart (graceful stop + start)
# Stopped containers → left stopped — was down intentionally, do not bring back up
# Missing containers → logged and skipped — not treated as fatal
# Each action uses RETRY_COUNT + SLEEP from master.conf for retry logic.
#
# The "was running → restart, was stopped → leave stopped" rule is consistent
# across the entire ecosystem — container state is always respected.
#
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
# Dependency ordering — containers restart in dependency-safe order using
#   HOST*_WATCHDOG_DEPENDENCIES from master_host*.conf. If Authelia depends on
#   Mariadb + Redis, those restart first with CONTAINER_DELAY before Authelia starts.
#
# Restart verification — after each restart, container state is checked after a short
#   settle period. If the container fails to stay running it is marked as failed and
#   a notification is sent rather than silently passing.
#
# Timeout protection — all docker commands are wrapped in a 30 second timeout.
#   A hung Docker daemon cannot cause this script to hang indefinitely.
#   Timed-out commands are retried per RETRY_COUNT before marking as failed.
#
# ── CONFIGURATION (master_host*.conf) ─────────────────────────────────────────────────────────
#   HOST*_DAILY_RESTART_CONTAINERS — list of containers to restart daily
#   Set by detect_hosts() alias → DAILY_RESTART_CONTAINERS used by this script
#
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
#   RETRY_COUNT  — retry attempts before giving up on a container
#   SLEEP        — seconds between retry attempts
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
#   docker_daily_restart.sh              — normal restart
#   docker_daily_restart.sh --dry-run    — preview without restarting
#   docker_daily_restart.sh --log        — verbose output
#   docker_daily_restart.sh --status     — show config and exit
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

source "$SCRIPT_DIR/../load_config.sh"

parse_args "$@"

# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_GEAR Setup ━━━"

if [[ "$EUID" -ne 0 ]]; then
    error "Must be run as root"
    exit 1
fi
acquire_lock

if ! command -v docker &>/dev/null; then
    error "Docker command not found — check PATH or Docker installation"
    notify "Docker daily restart failed — Docker not found on $(hostname)" "Docker Daily Restart" "warning"
    exit 1
fi

# detect_hosts() sets MY_ID and aliases HOST*_DAILY_RESTART_CONTAINERS → DAILY_RESTART_CONTAINERS
detect_hosts

if [[ ${#DAILY_RESTART_CONTAINERS[@]} -eq 0 ]]; then
    warn "DAILY_RESTART_CONTAINERS is empty for $MY_ID — nothing to restart"
    warn "Check HOST${MY_ID#HOST}_DAILY_RESTART_CONTAINERS in master_host*.conf"
    exit 0
fi

echo "  $MY_ID ($LOCAL_SERVER_NAME) — ${#DAILY_RESTART_CONTAINERS[@]} containers configured"

# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
    echo ""
    echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
    echo "$ICON_HOST       Identity:  $MY_ID ($LOCAL_SERVER_NAME)"
    echo "$ICON_CONTAINERS Containers: ${DAILY_RESTART_CONTAINERS[*]}"
    echo "$ICON_RETRY      Retries:   $RETRY_COUNT"
    echo "$ICON_TIME       Sleep:     ${SLEEP}s between retries"
    echo "$ICON_NOTIFY     Notify:    unRAID=${NOTIFY_UNRAID:-false} Discord=$([[ -n "${MY_DISCORD_WEBHOOK:-}" ]] && echo enabled || echo disabled)"
    echo "$ICON_GEAR       Dry Run:   $DRY_RUN"
    echo "━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi

[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be restarted"

# ==============================================================================================
# ── FUNCTIONS ─────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================

# Wraps docker commands with a 30 second timeout.
# Prevents a hung Docker daemon from causing the script to hang indefinitely.
# Usage: docker_cmd docker restart ContainerName
DOCKER_TIMEOUT=30
docker_cmd() {
    timeout "$DOCKER_TIMEOUT" "$@"
    local exit_code=$?
    if [[ "$exit_code" -eq 124 ]]; then
        error "Docker command timed out after ${DOCKER_TIMEOUT}s: $*"
        return 1
    fi
    return "$exit_code"
}

# Verifies a container is still running after restart.
# Gives the container a short settle period before checking.
# Returns 0 if running, 1 if crashed or stopped.
RESTART_VERIFY_WAIT=5   # seconds to wait before checking state post-restart
verify_running() {
    local container="$1"
    sleep "$RESTART_VERIFY_WAIT"
    local state
    state=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
    if [[ "$state" != "true" ]]; then
        error "$container failed to stay running after restart — may have crashed"
        return 1
    fi
    return 0
}

# Builds a dependency-safe restart order from DAILY_RESTART_CONTAINERS.
# Containers that are dependencies of others restart first.
# Returns ordered list in ORDERED_RESTART array.
build_restart_order() {
    ORDERED_RESTART=()
    local remaining=("${DAILY_RESTART_CONTAINERS[@]}")
    local placed=()

    # First pass — add dependency containers that appear in our list
    for container in "${remaining[@]}"; do
        [[ -z "$container" ]] && continue
        local is_dependency=false
        # Check if this container is a dependency of any other in our list
        for dep_string in "${WATCHDOG_DEPENDENCIES[@]}"; do
            if [[ "$dep_string" == *"$container"* ]]; then
                is_dependency=true
                break
            fi
        done
        # Also check associative array format
        for dependent in "${!WATCHDOG_DEPENDENCIES[@]}"; do
            if [[ "${WATCHDOG_DEPENDENCIES[$dependent]}" == *"$container"* ]]; then
                is_dependency=true
                break
            fi
        done
        if [[ "$is_dependency" == true ]]; then
            # Check not already placed
            local already=false
            for p in "${placed[@]}"; do [[ "$p" == "$container" ]] && already=true && break; done
            if [[ "$already" == false ]]; then
                ORDERED_RESTART+=("$container")
                placed+=("$container")
            fi
        fi
    done

    # Second pass — add remaining containers (dependents and independents)
    for container in "${remaining[@]}"; do
        [[ -z "$container" ]] && continue
        local already=false
        for p in "${placed[@]}"; do [[ "$p" == "$container" ]] && already=true && break; done
        if [[ "$already" == false ]]; then
            ORDERED_RESTART+=("$container")
            placed+=("$container")
        fi
    done

    log "Restart order: ${ORDERED_RESTART[*]}"
}

# Checks if a container is a dependent of the previously restarted container.
# If so, waits CONTAINER_DELAY before restarting to allow dependency to settle.
# Usage: check_dependency_delay "$container" "$last_restarted"
check_dependency_delay() {
    local container="$1"
    local last="$2"
    [[ -z "$last" ]] && return

    local deps="${WATCHDOG_DEPENDENCIES[$container]:-}"
    if [[ -n "$deps" ]] && [[ "$deps" == *"$last"* ]]; then
        echo "  Waiting ${CONTAINER_DELAY}s — $container depends on $last..."
        sleep "$CONTAINER_DELAY"
    fi
}

# Retries a docker command up to RETRY_COUNT times with SLEEP seconds between attempts.
# Uses docker_cmd wrapper for timeout protection on each attempt.
# Usage: retry_docker docker restart ContainerName
retry_docker() {
    local attempt=1

    while [[ "$attempt" -le "$RETRY_COUNT" ]]; do
        info "$ICON_RETRY Attempt $attempt of $RETRY_COUNT: $*"

        if docker_cmd "$@"; then
            log "Succeeded on attempt $attempt"
            return 0
        else
            warn "Attempt $attempt failed"
            (( attempt++ ))
            [[ "$attempt" -le "$RETRY_COUNT" ]] && sleep "$SLEEP"
        fi
    done

    error "Command failed after $RETRY_COUNT attempts: $*"
    return 1
}

# ==============================================================================================
# ━━━ Daily Restart ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_CONTAINERS Daily Restart — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
echo "$ICON_CONTAINERS Containers: ${DAILY_RESTART_CONTAINERS[*]}"
echo "$ICON_RETRY      Retries:    $RETRY_COUNT"
echo ""

START=$(date +%s)
FAILED=()
RESTARTED=()
SKIPPED=()

# Build dependency-safe restart order
build_restart_order
echo "$ICON_GEAR Restart order: ${ORDERED_RESTART[*]}"
echo ""

LAST_RESTARTED=""

for container in "${ORDERED_RESTART[@]}"; do
    [[ -z "$container" ]] && continue
    echo "━━━ $ICON_CONTAINERS $container ━━━"

    if ! timeout "$DOCKER_TIMEOUT" docker inspect "$container" &>/dev/null; then
        warn "$container does not exist — skipping"
        echo ""
        continue
    fi

    STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)

    case "$STATUS" in
        true)
            echo "$ICON_RUNNING $container is running — restarting..."

            # Wait if this container depends on the last one restarted
            check_dependency_delay "$container" "$LAST_RESTARTED"

            if [[ "$DRY_RUN" == true ]]; then
                warn "DRY RUN — would restart $container"
                RESTARTED+=("$container")
            else
                if retry_docker docker restart "$container"; then
                    # Verify container stayed running after restart
                    if verify_running "$container"; then
                        echo "$ICON_STARTED $container restarted and running ✅"
                        RESTARTED+=("$container")
                        LAST_RESTARTED="$container"
                    else
                        error "$container restarted but crashed immediately"
                        notify "$container crashed after restart on $(hostname)" "Docker Daily Restart" "warning"
                        FAILED+=("$container")
                    fi
                else
                    error "Failed to restart $container after $RETRY_COUNT attempts"
                    notify "$container failed to restart on $(hostname)" "Docker Daily Restart" "warning"
                    FAILED+=("$container")
                fi
            fi
            ;;
        false)
            # Container was stopped — leave it stopped
            # Intentionally stopped containers are not restarted
            echo "$ICON_NOT_RUNNING $container is stopped — skipping (respecting stopped state)"
            SKIPPED+=("$container")
            ;;
        *)
            error "Unknown status for $container: $STATUS"
            FAILED+=("$container")
            ;;
    esac

    echo ""
done

END=$(date +%s)

# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
echo "━━━━━ $ICON_SUMMARY DAILY RESTART SUMMARY ━━━━━"
echo "$ICON_HOST     Identity:  $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_TIME     Duration:  $(format_duration $((END - START)))"
[[ ${#RESTARTED[@]} -gt 0 ]] && echo "$ICON_STARTED      Restarted: ${RESTARTED[*]}"
[[ ${#SKIPPED[@]} -gt 0 ]]   && echo "$ICON_NOT_RUNNING  Skipped:   ${SKIPPED[*]} (were stopped)"
[[ ${#FAILED[@]} -gt 0 ]]    && echo "$ICON_ERROR        Failed:    ${FAILED[*]}"

if [[ "$DRY_RUN" == true ]]; then
    echo "$ICON_WARN     Status:    DRY RUN — no changes made"
elif [[ ${#FAILED[@]} -eq 0 ]]; then
    echo "$ICON_DONE     Status:    $ICON_SUCCESS ALL DONE"
    notify "Daily restart complete — ${#RESTARTED[@]} restarted, ${#SKIPPED[@]} skipped (stopped) on $(hostname)" "Docker Daily Restart" "normal"
else
    echo "$ICON_ERROR    Status:    $ICON_ERROR ${#FAILED[@]} container(s) failed"
    notify "Daily restart completed with errors on $(hostname) — failed: ${FAILED[*]}" "Docker Daily Restart" "warning"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

[[ ${#FAILED[@]} -gt 0 ]] && exit 1
exit 0