diff --git a/Docker_Essentials/Manual-Docker_Essentials.md b/Docker_Essentials/Manual-Docker_Essentials.md index d6448d0..ed3aad8 100644 --- a/Docker_Essentials/Manual-Docker_Essentials.md +++ b/Docker_Essentials/Manual-Docker_Essentials.md @@ -528,7 +528,17 @@ All scripts support these standard flags: |------|-------------| | `--dry-run` | Preview actions without making changes. Shows exactly what would happen. | | `--status` | Show current config, container states, and relevant runtime info, then exit. | -| `--log` | Verbose output — full detail for every container checked and every decision made. | +| `--log` | Verbose mode — adds per-container banners, action lines, pull output, and list details. | + +**Output tiers:** Scripts have two output levels. Without `--log`, each script processes +silently and always concludes with a summary block: identity, duration, counts, and +a status line. Per-container detail — individual container names, pull Status lines, +skip reasons — only appears with `--log`. Warnings and errors are always visible +regardless of `--log`. + +`docker_watchdog.sh` is the exception — it runs 96 cycles/day as a background daemon +and is silent on clean cycles by design. Its summary block only fires when there are +restarts or warnings. Use `--log` to see per-cycle detail on clean cycles. ### `docker_watchdog.sh --status` shows: Strike counts for all monitored containers, current skip list contents, whether grace diff --git a/Docker_Essentials/docker_container_stop.sh b/Docker_Essentials/docker_container_stop.sh index e5bb137..d829c6d 100755 --- a/Docker_Essentials/docker_container_stop.sh +++ b/Docker_Essentials/docker_container_stop.sh @@ -68,6 +68,9 @@ # Docker Presence Check # Verifies docker binary exists before execution. # +# Lock Acquisition +# acquire_lock prevents overlapping runs (e.g. array_stopping firing twice). +# # Timeout Protection # All docker commands wrapped in timeout protection to prevent daemon hangs # from stalling shutdown indefinitely. @@ -167,7 +170,7 @@ echo "" echo "━━━ $ICON_CONTAINERS Docker Container Stop — $(date '+%Y-%m-%d %H:%M:%S') ━━━" if [[ ${#RUNNING[@]} -eq 0 ]]; then - log "No running containers — nothing to do" + echo "No running containers — nothing to do" exit 0 fi @@ -180,12 +183,11 @@ FAILED=() for container in "${RUNNING[@]}"; do [[ -z "$container" ]] && continue - echo "━━━ $ICON_CONTAINERS $container ━━━" + log "━━━ $ICON_CONTAINERS $container ━━━" if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would stop $container" STOPPED+=("$container") - echo "" continue fi @@ -202,7 +204,7 @@ for container in "${RUNNING[@]}"; do -f '{{.State.Running}}' "$container" 2>/dev/null || echo "unknown") if [[ "$STATE" != "true" ]]; then - echo "$ICON_DONE $container stopped ✅" + log "$ICON_DONE $container stopped ✅" STOPPED+=("$container") success=true break @@ -230,8 +232,6 @@ for container in "${RUNNING[@]}"; do FAILED+=("$container") fi fi - - echo "" done END=$(date +%s) @@ -243,8 +243,8 @@ echo "━━━━━ $ICON_SUMMARY DOCKER CONTAINER STOP SUMMARY ━━━━ echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" echo "$ICON_CONTAINERS Scope: ${#RUNNING[@]} running → ${#STOPPED[@]} stopped" -[[ ${#STOPPED[@]} -gt 0 ]] && echo "$ICON_DONE Stopped: ${STOPPED[*]}" -[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}" +[[ ${#STOPPED[@]} -gt 0 ]] && log "$ICON_DONE Stopped: ${STOPPED[*]}" +[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}" if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — no containers stopped" diff --git a/Docker_Essentials/docker_daily_restart.sh b/Docker_Essentials/docker_daily_restart.sh index 3ed911d..04fb85c 100644 --- a/Docker_Essentials/docker_daily_restart.sh +++ b/Docker_Essentials/docker_daily_restart.sh @@ -109,9 +109,6 @@ parse_args "$@" # ============================================================================================== # ━━━ Setup ━━━ # ============================================================================================== -echo "" -echo "━━━ $ICON_GEAR Setup ━━━" - if [[ "$EUID" -ne 0 ]]; then error "Must be run as root" exit 1 @@ -133,8 +130,6 @@ if [[ ${#DAILY_RESTART_CONTAINERS[@]} -eq 0 ]]; then exit 0 fi -echo " $MY_ID ($LOCAL_SERVER_NAME) — ${#DAILY_RESTART_CONTAINERS[@]} containers configured" - # ============================================================================================== # ━━━ Status ━━━ # ============================================================================================== @@ -260,7 +255,7 @@ retry_docker() { local attempt=1 while [[ "$attempt" -le "$RETRY_COUNT" ]]; do - info "$ICON_RETRY Attempt $attempt of $RETRY_COUNT: $*" + log "$ICON_RETRY Attempt $attempt of $RETRY_COUNT: $*" if docker_cmd "$@"; then log "Succeeded on attempt $attempt" @@ -281,9 +276,9 @@ retry_docker() { # ============================================================================================== echo "" echo "━━━ $ICON_CONTAINERS Daily Restart — $(date '+%Y-%m-%d %H:%M:%S') ━━━" -echo "$ICON_CONTAINERS Containers: ${DAILY_RESTART_CONTAINERS[*]}" -echo "$ICON_RETRY Retries: $RETRY_COUNT" -echo "" +echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME) — ${#DAILY_RESTART_CONTAINERS[@]} container(s)" +log "$ICON_CONTAINERS Containers: ${DAILY_RESTART_CONTAINERS[*]}" +log "$ICON_RETRY Retries: $RETRY_COUNT" START=$(date +%s) FAILED=() @@ -292,18 +287,16 @@ SKIPPED=() # Build dependency-safe restart order build_restart_order -echo "$ICON_GEAR Restart order: ${ORDERED_RESTART[*]}" -echo "" +log "$ICON_GEAR Restart order: ${ORDERED_RESTART[*]}" LAST_RESTARTED="" for container in "${ORDERED_RESTART[@]}"; do [[ -z "$container" ]] && continue - echo "━━━ $ICON_CONTAINERS $container ━━━" + log "━━━ $ICON_CONTAINERS $container ━━━" if ! timeout "$DOCKER_TIMEOUT" docker inspect "$container" &>/dev/null; then warn "$container does not exist — skipping" - echo "" continue fi @@ -311,7 +304,7 @@ for container in "${ORDERED_RESTART[@]}"; do case "$STATUS" in true) - echo "$ICON_RUNNING $container is running — restarting..." + log "$ICON_RUNNING $container is running — restarting..." # Wait if this container depends on the last one restarted check_dependency_delay "$container" "$LAST_RESTARTED" @@ -323,7 +316,7 @@ for container in "${ORDERED_RESTART[@]}"; do if retry_docker docker restart "$container"; then # Verify container stayed running after restart if verify_running "$container"; then - echo "$ICON_STARTED $container restarted and running ✅" + log "$ICON_STARTED $container restarted and running ✅" RESTARTED+=("$container") LAST_RESTARTED="$container" else @@ -339,9 +332,7 @@ for container in "${ORDERED_RESTART[@]}"; do fi ;; false) - # Container was stopped — leave it stopped - # Intentionally stopped containers are not restarted - echo "$ICON_NOT_RUNNING $container is stopped — skipping (respecting stopped state)" + log "$ICON_NOT_RUNNING $container is stopped — skipping" SKIPPED+=("$container") ;; *) @@ -349,8 +340,6 @@ for container in "${ORDERED_RESTART[@]}"; do FAILED+=("$container") ;; esac - - echo "" done END=$(date +%s) @@ -361,17 +350,18 @@ END=$(date +%s) echo "━━━━━ $ICON_SUMMARY DAILY RESTART SUMMARY ━━━━━" echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_TIME Duration: $(format_duration $((END - START)))" -[[ ${#RESTARTED[@]} -gt 0 ]] && echo "$ICON_STARTED Restarted: ${RESTARTED[*]}" -[[ ${#SKIPPED[@]} -gt 0 ]] && echo "$ICON_NOT_RUNNING Skipped: ${SKIPPED[*]} (were stopped)" -[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}" +echo "$ICON_CONTAINERS Scope: ${#RESTARTED[@]} restarted, ${#SKIPPED[@]} skipped, ${#FAILED[@]} failed" +[[ ${#RESTARTED[@]} -gt 0 ]] && log "$ICON_STARTED Restarted: ${RESTARTED[*]}" +[[ ${#SKIPPED[@]} -gt 0 ]] && log "$ICON_NOT_RUNNING Skipped: ${SKIPPED[*]}" +[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}" if [[ "$DRY_RUN" == true ]]; then - echo "$ICON_WARN Status: DRY RUN — no changes made" + warn "DRY RUN — no changes made" elif [[ ${#FAILED[@]} -eq 0 ]]; then - echo "$ICON_DONE Status: $ICON_SUCCESS ALL DONE" - notify "Daily restart complete — ${#RESTARTED[@]} restarted, ${#SKIPPED[@]} skipped (stopped) on $(hostname)" "Docker Daily Restart" "normal" + echo "$ICON_DONE Status: ALL DONE ✅" + notify "Daily restart complete — ${#RESTARTED[@]} restarted, ${#SKIPPED[@]} skipped on $(hostname)" "Docker Daily Restart" "normal" else - echo "$ICON_ERROR Status: $ICON_ERROR ${#FAILED[@]} container(s) failed" + echo "$ICON_ERROR Status: ${#FAILED[@]} container(s) failed" notify "Daily restart completed with errors on $(hostname) — failed: ${FAILED[*]}" "Docker Daily Restart" "warning" fi echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" diff --git a/Docker_Essentials/docker_network_connect.sh b/Docker_Essentials/docker_network_connect.sh index dc90d7f..7736179 100755 --- a/Docker_Essentials/docker_network_connect.sh +++ b/Docker_Essentials/docker_network_connect.sh @@ -51,6 +51,14 @@ # OPERATIONAL SAFEGUARDS # ============================================================================================== # +# Lock Acquisition +# Prevents concurrent execution via acquire_lock(). Safe to call from +# array start hooks or manually without risk of overlap. +# +# Host Detection +# detect_hosts() identifies which server is running the script and aliases +# HOST*_NETWORK_CONNECT_* arrays to the correct host's values. +# # Docker Daemon Check # Verifies daemon is responsive before any network operations. Network # commands against a hung daemon hang indefinitely. @@ -282,7 +290,7 @@ elif [[ ${#FAILED[@]} -gt 0 ]]; then elif [[ ${#NETWORKS_CREATED[@]} -gt 0 ]]; then warn "Networks recreated — ${NETWORKS_CREATED[*]} — unRAID update likely wiped them" else - log "All networks healthy — no action needed" + echo "All networks healthy — no action needed" fi echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" diff --git a/Docker_Essentials/docker_update.sh b/Docker_Essentials/docker_update.sh index 1d36ba9..6155681 100644 --- a/Docker_Essentials/docker_update.sh +++ b/Docker_Essentials/docker_update.sh @@ -55,6 +55,14 @@ # OPERATIONAL SAFEGUARDS # ============================================================================================== # +# Lock Acquisition +# Prevents concurrent execution via acquire_lock(). Safe to call from +# maintenance scripts without risk of overlap. +# +# Host Detection +# detect_hosts() identifies which server is running the script and aliases +# HOST*_DAILY_RESTART_CONTAINERS to the correct host's values. +# # Root Enforcement # Docker operations require root privileges. # @@ -191,7 +199,7 @@ if [[ "$REMAINDER_MODE" == true ]]; then unset _all_running _exclude _c else if [[ "${DAILY_CONTAINER_UPDATES:-true}" != "true" ]]; then - log "DAILY_CONTAINER_UPDATES=false — skipping container updates" + echo "DAILY_CONTAINER_UPDATES=false — skipping container updates" exit 0 fi @@ -227,7 +235,7 @@ fi [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no images will be pulled" if [[ ${#TARGET_CONTAINERS[@]} -eq 0 ]]; then - log "No containers to update" + echo "No containers to update" exit 0 fi @@ -240,7 +248,7 @@ if [[ "$REMAINDER_MODE" == true ]]; then echo "$ICON_CONTAINERS Updating ${#TARGET_CONTAINERS[@]} container(s) (not in daily or weekly sync)" else echo "━━━ $ICON_CONTAINERS Docker Update — $(date '+%Y-%m-%d %H:%M:%S') ━━━" - echo "$ICON_CONTAINERS Containers: ${TARGET_CONTAINERS[*]}" + log "$ICON_CONTAINERS Containers: ${TARGET_CONTAINERS[*]}" fi echo "" @@ -252,12 +260,11 @@ SKIPPED=() for container in "${TARGET_CONTAINERS[@]}"; do [[ -z "$container" ]] && continue - echo "━━━ $ICON_CONTAINERS $container ━━━" + log "━━━ $ICON_CONTAINERS $container ━━━" if ! docker inspect "$container" &>/dev/null; then warn "$container — not found, skipping" SKIPPED+=("$container") - echo "" continue fi @@ -265,7 +272,6 @@ for container in "${TARGET_CONTAINERS[@]}"; do if [[ -z "$IMAGE" ]]; then warn "$container — could not determine image, skipping" SKIPPED+=("$container") - echo "" continue fi @@ -274,19 +280,25 @@ for container in "${TARGET_CONTAINERS[@]}"; do if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would pull: $IMAGE" UPDATED+=("$container") - echo "" continue fi # Capture image ID before pull to detect whether an update landed OLD_ID=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "") - echo "$ICON_SYNC Pulling $IMAGE..." - if docker pull "$IMAGE" 2>&1 | grep -E "^(Status:|Digest:|Error|error)" | sed 's/^/ /'; then - NEW_ID=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "") + log "$ICON_SYNC Pulling $IMAGE..." + if [[ "$ENABLE_LOGGING" == "true" ]]; then + docker pull "$IMAGE" 2>&1 | grep -E "^(Status:|Digest:|Error|error)" | sed 's/^/ /' + _pull_rc=${PIPESTATUS[0]} + else + docker pull "$IMAGE" >/dev/null 2>&1 + _pull_rc=$? + fi + NEW_ID=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "") + if [[ $_pull_rc -eq 0 ]]; then if [[ -n "$OLD_ID" ]] && [[ "$OLD_ID" != "$NEW_ID" ]]; then - echo "$ICON_DONE $container — updated ✅" + log "$ICON_DONE $container — updated ✅" UPDATED+=("$container") else log "$container — already up to date" @@ -297,7 +309,6 @@ for container in "${TARGET_CONTAINERS[@]}"; do FAILED+=("$container") fi - echo "" done END=$(date +%s) @@ -312,15 +323,18 @@ else fi echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" -[[ ${#UPDATED[@]} -gt 0 ]] && echo "$ICON_DONE Updated: ${UPDATED[*]}" -[[ ${#UP_TO_DATE[@]} -gt 0 ]] && echo "$ICON_RUNNING Up to date: ${UP_TO_DATE[*]}" -[[ ${#SKIPPED[@]} -gt 0 ]] && echo "$ICON_WARN Skipped: ${SKIPPED[*]}" +if [[ ${#UPDATED[@]} -gt 0 ]]; then + echo "$ICON_DONE Updated: ${#UPDATED[@]}" + log " ${UPDATED[*]}" +fi +[[ ${#UP_TO_DATE[@]} -gt 0 ]] && log "$ICON_RUNNING Up to date: ${#UP_TO_DATE[@]}" +[[ ${#SKIPPED[@]} -gt 0 ]] && log "$ICON_WARN Skipped: ${#SKIPPED[@]}" [[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}" if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — no images pulled" elif [[ ${#FAILED[@]} -eq 0 ]]; then - log "$ICON_DONE Status: done ✅ — ${#UPDATED[@]} updated, ${#UP_TO_DATE[@]} current" + echo "$ICON_DONE Status: done ✅ — ${#UPDATED[@]} updated, ${#UP_TO_DATE[@]} current" else warn "Status: ${#FAILED[@]} pull(s) failed — restart will proceed with existing images" fi diff --git a/Docker_Essentials/docker_update_remaining.sh b/Docker_Essentials/docker_update_remaining.sh index ee05fb6..2a90a77 100644 --- a/Docker_Essentials/docker_update_remaining.sh +++ b/Docker_Essentials/docker_update_remaining.sh @@ -21,6 +21,15 @@ # OPERATIONAL SAFEGUARDS # ============================================================================================== # +# Lock Acquisition +# Prevents concurrent execution via acquire_lock(). Safe to call from +# weekly maintenance scripts without risk of overlap. +# +# Host Detection +# detect_hosts() identifies which server is running the script and aliases +# HOST*_DAILY_RESTART_CONTAINERS and HOST*_WEEKLY_RESTART_CONTAINERS to +# the correct host's values for exclusion. +# # Root Enforcement # Docker operations require root privileges. # @@ -95,7 +104,7 @@ fi detect_hosts if [[ "${WEEKLY_REMAINING_UPDATES:-true}" != "true" ]]; then - log "WEEKLY_REMAINING_UPDATES=false — skipping remaining container updates" + echo "WEEKLY_REMAINING_UPDATES=false — skipping remaining container updates" exit 0 fi @@ -132,7 +141,7 @@ if [[ "$SHOW_STATUS" == true ]]; then fi if [[ ${#REMAINING[@]} -eq 0 ]]; then - log "No remaining containers to update — all running containers are covered by daily/weekly lists" + echo "No remaining containers to update — all running containers are covered by daily/weekly lists" exit 0 fi @@ -184,8 +193,8 @@ verify_running() { # ============================================================================================== echo "" echo "━━━ $ICON_CONTAINERS Docker Update (Remaining) — $(date '+%Y-%m-%d %H:%M:%S') ━━━" -echo "$ICON_CONTAINERS Containers: ${REMAINING[*]}" -echo "$ICON_CONTAINERS Excluded (managed elsewhere): ${!EXCLUDED[*]}" +log "$ICON_CONTAINERS Containers: ${REMAINING[*]}" +log "$ICON_CONTAINERS Excluded (managed elsewhere): ${!EXCLUDED[*]}" echo "" START=$(date +%s) @@ -195,13 +204,12 @@ FAILED=() for container in "${REMAINING[@]}"; do [[ -z "$container" ]] && continue - echo "━━━ $ICON_CONTAINERS $container ━━━" + log "━━━ $ICON_CONTAINERS $container ━━━" IMAGE=$(docker inspect --format='{{.Config.Image}}' "$container" 2>/dev/null) if [[ -z "$IMAGE" ]]; then warn "$container — could not determine image, skipping" FAILED+=("$container") - echo "" continue fi @@ -210,18 +218,24 @@ for container in "${REMAINING[@]}"; do if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would pull: $IMAGE" UPDATED+=("$container") - echo "" continue fi OLD_ID=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "") - echo "$ICON_SYNC Pulling $IMAGE..." - if docker pull "$IMAGE" 2>&1 | grep -E "^(Status:|Digest:|Error|error)" | sed 's/^/ /'; then - NEW_ID=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "") + log "$ICON_SYNC Pulling $IMAGE..." + if [[ "$ENABLE_LOGGING" == "true" ]]; then + docker pull "$IMAGE" 2>&1 | grep -E "^(Status:|Digest:|Error|error)" | sed 's/^/ /' + _pull_rc=${PIPESTATUS[0]} + else + docker pull "$IMAGE" >/dev/null 2>&1 + _pull_rc=$? + fi + NEW_ID=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "") + if [[ $_pull_rc -eq 0 ]]; then if [[ -n "$OLD_ID" ]] && [[ "$OLD_ID" != "$NEW_ID" ]]; then - echo "$ICON_DONE $container — updated ✅" + log "$ICON_DONE $container — updated ✅" UPDATED+=("$container") else log "$container — already up to date" @@ -232,7 +246,6 @@ for container in "${REMAINING[@]}"; do FAILED+=("$container") fi - echo "" done # ============================================================================================== @@ -245,33 +258,30 @@ SKIPPED_STOPPED=() if [[ ${#UPDATED[@]} -gt 0 ]]; then echo "" echo "━━━ $ICON_CONTAINERS Restarting Updated Containers — $(date '+%Y-%m-%d %H:%M:%S') ━━━" - echo "$ICON_CONTAINERS Containers with new image: ${UPDATED[*]}" - echo "" + log "$ICON_CONTAINERS Containers with new image: ${UPDATED[*]}" for container in "${UPDATED[@]}"; do [[ -z "$container" ]] && continue - echo "━━━ $ICON_CONTAINERS $container ━━━" + log "━━━ $ICON_CONTAINERS $container ━━━" STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null) if [[ "$STATUS" != "true" ]]; then - echo "$ICON_NOT_RUNNING $container is stopped — skipping restart (respecting stopped state)" + log "$ICON_NOT_RUNNING $container is stopped — skipping restart (respecting stopped state)" SKIPPED_STOPPED+=("$container") - echo "" continue fi if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would restart $container" RESTARTED+=("$container") - echo "" continue fi - echo "$ICON_RUNNING $container is running — restarting on new image..." + log "$ICON_RUNNING $container is running — restarting on new image..." if retry_docker docker restart "$container"; then if verify_running "$container"; then - echo "$ICON_DONE $container restarted and running ✅" + log "$ICON_DONE $container restarted and running ✅" RESTARTED+=("$container") else error "$container restarted but crashed immediately" @@ -283,7 +293,6 @@ if [[ ${#UPDATED[@]} -gt 0 ]]; then notify "$container failed to restart after update on $(hostname)" "Docker Update Remaining" "warning" RESTART_FAILED+=("$container") fi - echo "" done else log "No containers received a new image — nothing to restart" @@ -300,7 +309,7 @@ if [[ "$DRY_RUN" == true ]]; then PRUNED_SUMMARY="(dry run)" else PRUNED_OUTPUT=$(docker image prune -f 2>&1) - echo "$PRUNED_OUTPUT" | sed 's/^/ /' + [[ "$ENABLE_LOGGING" == "true" ]] && echo "$PRUNED_OUTPUT" | sed 's/^/ /' PRUNED_SUMMARY=$(echo "$PRUNED_OUTPUT" | grep -E "^Total reclaimed" || echo "nothing reclaimed") fi @@ -314,11 +323,17 @@ echo "━━━━━ $ICON_SUMMARY DOCKER UPDATE (REMAINING) SUMMARY ━━━ echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" echo "$ICON_CONTAINERS Scope: ${#ALL_RUNNING[@]} running — ${#EXCLUDED[@]} managed = ${#REMAINING[@]} checked" -[[ ${#UPDATED[@]} -gt 0 ]] && echo "$ICON_DONE New image: ${UPDATED[*]}" -[[ ${#UP_TO_DATE[@]} -gt 0 ]] && echo "$ICON_RUNNING Up to date: ${UP_TO_DATE[*]}" +if [[ ${#UPDATED[@]} -gt 0 ]]; then + echo "$ICON_DONE New image: ${#UPDATED[@]}" + log " ${UPDATED[*]}" +fi +[[ ${#UP_TO_DATE[@]} -gt 0 ]] && log "$ICON_RUNNING Up to date: ${#UP_TO_DATE[@]}" [[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Pull failed: ${FAILED[*]}" -[[ ${#RESTARTED[@]} -gt 0 ]] && echo "$ICON_DONE Restarted: ${RESTARTED[*]}" -[[ ${#SKIPPED_STOPPED[@]} -gt 0 ]] && echo "$ICON_WARN Not running: ${SKIPPED_STOPPED[*]} (skipped restart)" +if [[ ${#RESTARTED[@]} -gt 0 ]]; then + echo "$ICON_DONE Restarted: ${#RESTARTED[@]}" + log " ${RESTARTED[*]}" +fi +[[ ${#SKIPPED_STOPPED[@]} -gt 0 ]] && log "$ICON_WARN Not running: ${#SKIPPED_STOPPED[@]} (skipped restart)" [[ ${#RESTART_FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Restart fail:${RESTART_FAILED[*]}" echo "$ICON_SYNC Pruned: ${PRUNED_SUMMARY:-none}" @@ -326,7 +341,7 @@ ALL_FAILED=$(( ${#FAILED[@]} + ${#RESTART_FAILED[@]} )) if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — no changes made" elif [[ "$ALL_FAILED" -eq 0 ]]; then - log "$ICON_DONE Status: done ✅ — ${#RESTARTED[@]} restarted, ${#UP_TO_DATE[@]} current" + echo "$ICON_DONE Status: done ✅ — ${#RESTARTED[@]} restarted, ${#UP_TO_DATE[@]} current" else warn "Status: $ALL_FAILED error(s) — ${#FAILED[@]} pull failure(s), ${#RESTART_FAILED[@]} restart failure(s)" fi diff --git a/Docker_Essentials/docker_watchdog.sh b/Docker_Essentials/docker_watchdog.sh index 180eacd..3fd61a9 100755 --- a/Docker_Essentials/docker_watchdog.sh +++ b/Docker_Essentials/docker_watchdog.sh @@ -91,6 +91,14 @@ # OPERATIONAL SAFEGUARDS # ============================================================================================== # +# Lock Acquisition +# Prevents concurrent execution via acquire_lock(). Safe at array start — +# only one watchdog instance runs at a time. +# +# Host Detection +# detect_hosts() identifies which server is running the script and aliases +# all HOST*_WATCHDOG_* arrays to the correct host's values. +# # Startup Grace Period # Restart actions suppressed for WATCHDOG_STARTUP_GRACE seconds after the # watchdog starts. Checks still run and log — only restarts are suppressed. @@ -224,14 +232,10 @@ parse_args "$@" # ============================================================================================== # ━━━ Setup — runs once at start ━━━ # ============================================================================================== -echo "" -echo "━━━ $ICON_GEAR Setup ━━━" - if [[ "$EUID" -ne 0 ]]; then error "Must be run as root" exit 1 fi -success "Running as root" # Skip if another instance is running — no pile-up during long operations acquire_lock @@ -243,7 +247,6 @@ if ! command -v docker >/dev/null 2>&1; then error "Docker not found — cannot start watchdog" exit 1 fi -success "Docker found" [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be restarted" @@ -519,7 +522,7 @@ check_docker_daemon() { # docker info is more definitive than docker ps for daemon health if timeout "$DOCKER_TIMEOUT" docker info >/dev/null 2>&1; then if [[ "$WATCHDOG_DAEMON_STRIKES" -gt 0 ]]; then - info "$ICON_STARTED Docker daemon recovered — clearing strikes" + log "$ICON_STARTED Docker daemon recovered — clearing strikes" queue_notify "Docker daemon recovered on $(hostname)" "normal" WATCHDOG_DAEMON_STRIKES=0 WATCHDOG_DAEMON_RESTARTED=false @@ -558,7 +561,7 @@ check_docker_daemon() { WATCHDOG_DAEMON_RESTARTED=true set_strikes "daemon_restarted_flag" "true" "$WATCHDOG_STATE_FILE" if /etc/rc.d/rc.docker restart >/dev/null 2>&1; then - info "Docker daemon restart issued — waiting ${WATCHDOG_DAEMON_RESTART_WAIT}s..." + log "Docker daemon restart issued — waiting ${WATCHDOG_DAEMON_RESTART_WAIT}s..." sleep "$WATCHDOG_DAEMON_RESTART_WAIT" if timeout "$DOCKER_TIMEOUT" docker info >/dev/null 2>&1; then @@ -586,8 +589,7 @@ check_docker_daemon() { # ============================================================================================== # ━━━ Single-Pass Monitoring Run ━━━ # ============================================================================================== -info "$ICON_WATCHDOG Docker watchdog — $MY_ID — $(date '+%H:%M:%S')" -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +log "$ICON_WATCHDOG Docker watchdog — $MY_ID — $(date '+%H:%M:%S')" CYCLE_START=$(date +%s) diff --git a/Docker_Essentials/docker_weekly_restart.sh b/Docker_Essentials/docker_weekly_restart.sh index a188675..8ce0c8f 100644 --- a/Docker_Essentials/docker_weekly_restart.sh +++ b/Docker_Essentials/docker_weekly_restart.sh @@ -31,6 +31,11 @@ # All docker commands wrapped in a 30 second timeout. A hung Docker daemon # cannot cause this script to hang indefinitely. # +# Host Detection +# detect_hosts() identifies which server is running the script and aliases +# HOST*_WEEKLY_RESTART_CONTAINERS and HOST*_WATCHDOG_DEPENDENCIES to the +# correct host's values. +# # Lock Acquisition # acquire_lock() prevents concurrent execution. # @@ -86,9 +91,6 @@ parse_args "$@" # ============================================================================================== # ━━━ Setup ━━━ # ============================================================================================== -echo "" -echo "━━━ $ICON_GEAR Setup ━━━" - if [[ "$EUID" -ne 0 ]]; then error "Must be run as root" exit 1 @@ -110,7 +112,7 @@ if [[ ${#WEEKLY_RESTART_CONTAINERS[@]} -eq 0 ]]; then exit 0 fi -echo " $MY_ID ($LOCAL_SERVER_NAME) — ${#WEEKLY_RESTART_CONTAINERS[@]} containers configured" +log "$MY_ID ($LOCAL_SERVER_NAME) — ${#WEEKLY_RESTART_CONTAINERS[@]} containers configured" # ============================================================================================== # ━━━ Status ━━━ @@ -232,7 +234,7 @@ check_dependency_delay() { [[ -z "$last" ]] && return local deps="${WATCHDOG_DEPENDENCIES[$container]:-}" if [[ -n "$deps" ]] && [[ "$deps" == *"$last"* ]]; then - info "Waiting ${CONTAINER_DELAY}s — $container depends on $last..." + log "Waiting ${CONTAINER_DELAY}s — $container depends on $last..." sleep "$CONTAINER_DELAY" fi } @@ -242,9 +244,8 @@ check_dependency_delay() { # ============================================================================================== echo "" echo "━━━ $ICON_CONTAINERS Weekly Restart — $(date '+%Y-%m-%d %H:%M:%S') ━━━" -echo "$ICON_CONTAINERS Containers: ${WEEKLY_RESTART_CONTAINERS[*]}" -echo "$ICON_RETRY Retries: $RETRY_COUNT" -echo "" +log "$ICON_CONTAINERS Containers: ${WEEKLY_RESTART_CONTAINERS[*]}" +log "$ICON_RETRY Retries: $RETRY_COUNT" START=$(date +%s) FAILED=() @@ -253,18 +254,15 @@ SKIPPED=() # Build dependency-safe restart order build_restart_order -echo "$ICON_GEAR Restart order: ${ORDERED_RESTART[*]}" -echo "" LAST_RESTARTED="" for container in "${ORDERED_RESTART[@]}"; do [[ -z "$container" ]] && continue - echo "━━━ $ICON_CONTAINERS $container ━━━" + log "━━━ $ICON_CONTAINERS $container ━━━" if ! timeout "$DOCKER_TIMEOUT" docker inspect "$container" &>/dev/null; then warn "$container does not exist — skipping" - echo "" continue fi @@ -272,7 +270,7 @@ for container in "${ORDERED_RESTART[@]}"; do case "$STATUS" in true) - echo "$ICON_RUNNING $container is running — restarting..." + log "$ICON_RUNNING $container is running — restarting..." # Wait if this container depends on the last one restarted check_dependency_delay "$container" "$LAST_RESTARTED" @@ -283,7 +281,7 @@ for container in "${ORDERED_RESTART[@]}"; do else if retry_docker docker restart "$container"; then if verify_running "$container"; then - echo "$ICON_STARTED $container restarted and running ✅" + log "$ICON_STARTED $container restarted and running ✅" RESTARTED+=("$container") LAST_RESTARTED="$container" else @@ -300,8 +298,7 @@ for container in "${ORDERED_RESTART[@]}"; do ;; false) # Container was stopped — leave it stopped - # Intentionally stopped containers are not restarted - echo "$ICON_NOT_RUNNING $container is stopped — skipping (respecting stopped state)" + log "$ICON_NOT_RUNNING $container is stopped — skipping (respecting stopped state)" SKIPPED+=("$container") ;; *) @@ -310,7 +307,6 @@ for container in "${ORDERED_RESTART[@]}"; do ;; esac - echo "" done END=$(date +%s) @@ -321,8 +317,11 @@ END=$(date +%s) echo "━━━━━ $ICON_SUMMARY WEEKLY RESTART SUMMARY ━━━━━" echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_TIME Duration: $(format_duration $((END - START)))" -[[ ${#RESTARTED[@]} -gt 0 ]] && echo "$ICON_STARTED Restarted: ${RESTARTED[*]}" -[[ ${#SKIPPED[@]} -gt 0 ]] && echo "$ICON_NOT_RUNNING Skipped: ${SKIPPED[*]} (were stopped)" +if [[ ${#RESTARTED[@]} -gt 0 ]]; then + echo "$ICON_STARTED Restarted: ${#RESTARTED[@]}" + log " ${RESTARTED[*]}" +fi +[[ ${#SKIPPED[@]} -gt 0 ]] && log "$ICON_NOT_RUNNING Skipped: ${#SKIPPED[@]} (were stopped)" [[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}" if [[ "$DRY_RUN" == true ]]; then diff --git a/Docker_Essentials/downloaders_reset.sh b/Docker_Essentials/downloaders_reset.sh index 13d2c1b..39f4bac 100644 --- a/Docker_Essentials/downloaders_reset.sh +++ b/Docker_Essentials/downloaders_reset.sh @@ -70,6 +70,11 @@ # Each section validates its downloader URL before API calls. Missing or # unreachable downloaders skip without affecting other sections. # +# Host Detection +# detect_hosts() identifies which server is running the script and aliases +# all HOST*_SLSKD_*, HOST*_SABNZBD_*, and HOST*_QBIT_* vars to the correct +# host's values. Downloaders not configured on this host skip automatically. +# # Lock Acquisition # acquire_lock "wait" — waits for previous run to finish since this runs every # 15 minutes and prior execution may still be completing. @@ -127,9 +132,6 @@ parse_args "$@" # ============================================================================================== # ━━━ Setup ━━━ # ============================================================================================== -echo "" -echo "━━━ $ICON_GEAR Downloaders Reset ━━━" - if [[ "$EUID" -ne 0 ]]; then error "Must be run as root" exit 1 @@ -252,7 +254,7 @@ if [[ -n "$SLSKD_URL" ]] && [[ -n "$SLSKD_API_KEY" ]] && [[ "$SLSKD_CONNECTED" = if [[ "$COUNT" -eq 0 ]]; then success "No stuck searches found ✅" else - info "Found $COUNT stuck search(es)" + log "Found $COUNT stuck search(es)" SUCCESS=0; FAIL=0 while IFS= read -r ID; do [[ -z "$ID" ]] && continue @@ -265,7 +267,7 @@ if [[ -n "$SLSKD_URL" ]] && [[ -n "$SLSKD_API_KEY" ]] && [[ "$SLSKD_CONNECTED" = "$SLSKD_URL/api/v0/searches/$ID" \ -H "X-Api-Key: $SLSKD_API_KEY") if [[ "$RESULT" == "200" || "$RESULT" == "204" ]]; then - info "$ICON_TRASH Cleared search: $ID" + log "$ICON_TRASH Cleared search: $ID" ((SUCCESS++)) else error "Failed: $ID (HTTP $RESULT)" @@ -313,7 +315,7 @@ if [[ -n "$SLSKD_URL" ]] && [[ -n "$SLSKD_API_KEY" ]] && [[ "$SLSKD_CONNECTED" = # Skip users with any active or queued transfers — never interrupt downloads ACTIVE=$(echo "$USER_DATA" | grep -c '"state":"InProgress"\|"state":"Queued"') if [[ "${ACTIVE:-0}" -gt 0 ]]; then - info "$ICON_SKIP Skipping $USER — has active/queued transfer(s)" + log "$ICON_SKIP Skipping $USER — has active/queued transfer(s)" ((SKIPPED++)) continue fi @@ -325,7 +327,7 @@ if [[ -n "$SLSKD_URL" ]] && [[ -n "$SLSKD_API_KEY" ]] && [[ "$SLSKD_CONNECTED" = grep -o '"id":"[^"]*"' | sed 's/"id":"//;s/"//') if [[ -z "$FILE_IDS" ]]; then - info "$ICON_SKIP Skipping $USER — no terminal-state transfers" + log "$ICON_SKIP Skipping $USER — no terminal-state transfers" ((SKIPPED++)) continue fi @@ -350,7 +352,7 @@ if [[ -n "$SLSKD_URL" ]] && [[ -n "$SLSKD_API_KEY" ]] && [[ "$SLSKD_CONNECTED" = fi done <<< "$FILE_IDS" - info "$ICON_TRASH Cleared $F_SUCCESS transfer(s) for: $USER ($F_FAIL failed)" + log "$ICON_TRASH Cleared $F_SUCCESS transfer(s) for: $USER ($F_FAIL failed)" ((SUCCESS += F_SUCCESS)) ((FAIL += F_FAIL)) done <<< "$USERNAMES" @@ -383,7 +385,7 @@ if [[ -n "$SLSKD_FAILED_IMPORTS_DIR" ]]; then if [[ "$IMPORT_COUNT" -eq 0 ]]; then success "No expired failed imports found ✅" else - info "Found $IMPORT_COUNT expired failed import(s)" + log "Found $IMPORT_COUNT expired failed import(s)" if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would delete:" echo "$OLD_IMPORTS" @@ -434,7 +436,7 @@ if [[ -n "$SABNZBD_URL" ]] && [[ -n "$SABNZBD_API_KEY" ]]; then curl -sf --max-time 10 \ "$SABNZBD_URL/api?mode=history&name=delete&value=$NZO_ID&apikey=$SABNZBD_API_KEY" \ >/dev/null - info "$ICON_TRASH Deleted: $NZO_ID" + log "$ICON_TRASH Deleted: $NZO_ID" ((DELETED++)) fi done <<< "$COMPLETED_IDS" @@ -480,7 +482,7 @@ if [[ -n "$SABNZBD_URL" ]] && [[ -n "$SABNZBD_API_KEY" ]]; then curl -sf --max-time 10 \ "$SABNZBD_URL/api?mode=history&name=delete&value=$NZO_ID&apikey=$SABNZBD_API_KEY" \ >/dev/null - info "$ICON_TRASH Deleted: $NZO_ID" + log "$ICON_TRASH Deleted: $NZO_ID" ((DELETED++)) fi done <<< "$FAILED_IDS" @@ -531,7 +533,7 @@ if [[ -n "$SABNZBD_URL" ]] && [[ -n "$SABNZBD_API_KEY" ]]; then curl -sf --max-time 10 \ "$SABNZBD_URL/api?mode=queue&name=delete&value=$NZO_ID&apikey=$SABNZBD_API_KEY" \ >/dev/null - info "$ICON_TRASH Removed stalled ($STATUS): $NZO_ID" + log "$ICON_TRASH Removed stalled ($STATUS): $NZO_ID" ((DELETED++)) fi done <<< "$STALLED_IDS" @@ -556,7 +558,7 @@ if [[ -n "$QBIT_URL" ]] && [[ -n "$QBIT_USERNAME" ]]; then echo "" echo "━━━ 🔍 qBittorrent — Failsafe (older than ${QBIT_FAILSAFE_MIN_DAYS} days) ━━━" [[ "$QBIT_FAILSAFE_MIN_RATIO" != "0" ]] && \ - info "Ratio requirement: >= ${QBIT_FAILSAFE_MIN_RATIO}" + log "Ratio requirement: >= ${QBIT_FAILSAFE_MIN_RATIO}" QBIT_COOKIE=$(curl -sf --max-time 10 -c - \ "$QBIT_URL/api/v2/auth/login" \ @@ -603,7 +605,7 @@ if [[ -n "$QBIT_URL" ]] && [[ -n "$QBIT_USERNAME" ]]; then "$QBIT_URL/api/v2/torrents/delete" \ -H "Cookie: $QBIT_COOKIE" \ --data "hashes=$HASH&deleteFiles=false" >/dev/null - info "$ICON_TRASH Deleted: $NAME (${AGE_DAYS}d old, ratio: $RATIO)" + log "$ICON_TRASH Deleted: $NAME (${AGE_DAYS}d old, ratio: $RATIO)" ((DELETED++)) fi done < <(echo "$TORRENTS" | tr '}' '\n')