unRAID_Essentials echo/log audit pass + exit traps

Exit traps: resource_watchdog registers/clears trap for stopped containers; server_reboot registers trap to restart Docker service on abort; system_watchdog records containers before bulk stop and registers trap to restart if reboot aborts. echo/log audit: status conclusions, nothing-to-do, and clean-cycle confirmations → echo across clear_logs, docker_syslog_filter, mover_stop, php_fpm_max_children, rsync_stop, user_scripts_stop. resource_watchdog: disabled flag, restore-level messages, pressure status → echo. system_watchdog: clean-cycle summary → echo. Manual-Unraid_Essentials.md: added Output Tiers section.
2026-05-21 17:21:20 -04:00
parent 86b31256ed
commit 3770bf80e7
10 changed files with 78 additions and 20 deletions
@@ -25,6 +25,22 @@ adding or reordering scripts at array start.

 ---

+## Output Tiers
+
+All scripts use a two-tier output model: `echo` lines are always visible; `log`
+lines only appear when `--log` is passed.
+
+**Daemon scripts** (`system_watchdog.sh`, `resource_watchdog.sh`, `webgui_restart.sh`):
+run on every cycle. Without `--log`, only state transitions, warnings, errors, and
+the clean-cycle conclusion line are visible. Per-check detail suppressed.
+
+**One-shot scripts** (`clear_logs.sh`, `docker_syslog_filter.sh`, `inotify_tuning.sh`,
+`mover_stop.sh`, `php_fpm_max_children.sh`, `rsync_stop.sh`, `server_reboot.sh`,
+`user_scripts_stop.sh`): without `--log`, section headers, per-step results, and the
+final summary are visible. Per-item detail suppressed.
+
+---
+
 ## ARRAY_START_SCRIPTS Order

 > **The order of scripts in ARRAY_START_SCRIPTS matters for three of these
@@ -277,12 +277,12 @@ if [[ "$TOTAL_CLEARED" -gt 0 || ${#FAILED[@]} -gt 0 ]]; then
        notify "Log clear failed on $(hostname) ($MY_ID) — ${FAILED[*]}" \
            "Clear Logs" "warning"
    else
-        log "$ICON_DONE Status: done — ${TOTAL_FREED_H} freed"
+        echo "$ICON_DONE Status: done — ${TOTAL_FREED_H} freed"
    fi
    echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
 else
    # All logs under threshold — completely silent
-    log "All logs under threshold — nothing to clear"
+    echo "All logs under threshold — nothing to clear"
 fi

 [[ ${#FAILED[@]} -gt 0 ]] && exit 1
@@ -189,12 +189,12 @@ fi
 if [[ "$DRY_RUN" == true ]]; then
    warn "DRY RUN — would restart rsyslog"
 else
-    log "Restarting rsyslog..."
+    echo "Restarting rsyslog..."
    if /etc/rc.d/rc.rsyslogd restart >/dev/null 2>&1; then
        sleep 2
        # Verify rsyslog actually running after restart
        if pgrep -x rsyslogd >/dev/null 2>&1; then
-            log "rsyslog restarted and running ✅"
+            echo "rsyslog restarted and running ✅"
        else
            error "rsyslog not running after restart"
            notify "rsyslog failed to start after filter update on $(hostname) ($MY_ID)" \
@@ -224,7 +224,7 @@ echo ""
 if [[ "$DRY_RUN" == true ]]; then
    warn "DRY RUN — no changes made"
 else
-    log "$ICON_DONE Status: done — Docker veth noise suppressed ✅"
+    echo "$ICON_DONE Status: done — Docker veth noise suppressed ✅"
 fi
 echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

@@ -127,7 +127,7 @@ fi
 START=$(date +%s)

 if ! pgrep -f "emhttp.*Mover" >/dev/null 2>&1; then
-    log "Mover is not running — nothing to do"
+    echo "Mover is not running — nothing to do"
    exit 0
 fi

@@ -137,7 +137,7 @@ warn "Mover is running (PID $MOVER_PID) — stopping in ${MOVER_STOP_TIMEOUT}s"
 # ── Warn users via wall ───────────────────────────────────────────────────────────────────────
 if [[ "$DRY_RUN" == false ]]; then
    wall "$ICON_WARN $MY_ID ($LOCAL_SERVER_NAME) — unRAID Mover stopping in ${MOVER_STOP_TIMEOUT}s"
-    log "Wall message sent — waiting ${MOVER_STOP_TIMEOUT}s..."
+    echo "Wall message sent — waiting ${MOVER_STOP_TIMEOUT}s..."
    sleep "$MOVER_STOP_TIMEOUT"
 else
    warn "DRY RUN — would send wall warning and wait ${MOVER_STOP_TIMEOUT}s"
@@ -187,6 +187,6 @@ echo ""
 if [[ "$DRY_RUN" == true ]]; then
    warn "DRY RUN — no changes made"
 else
-    log "$ICON_DONE Status: done — mover stopped ✅"
+    echo "$ICON_DONE Status: done — mover stopped ✅"
 fi
 echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
@@ -238,7 +238,7 @@ echo "$ICON_GEAR   Config file:  $PHP_CONF"
 echo "$ICON_PHP    Applied:      pm.max_children = $PHP_MAX_CHILDREN"
 echo "$ICON_TIME   Duration:     $(format_duration $(( END - START )))"
 echo ""
-log "$ICON_DONE Status: done ✅"
+echo "$ICON_DONE Status: done ✅"
 echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

 exit 0
@@ -118,7 +118,7 @@ if [[ "$EUID" -ne 0 ]]; then
 fi

 if [[ "${RW_ENABLED:-true}" != "true" ]]; then
-    log "Resource Manager disabled (RW_ENABLED=false)"
+    echo "Resource Manager disabled (RW_ENABLED=false)"
    exit 0
 fi

@@ -138,6 +138,21 @@ touch "$RW_STATE_FILE" 2>/dev/null || {
    exit 1
 }

+# ── Exit Trap — restart containers stopped this run if script crashes ──────────────────────────
+declare -a _RW_TRAP_STOPPED=()
+
+_rw_trap_restart_stopped() {
+    [[ ${#_RW_TRAP_STOPPED[@]} -eq 0 ]] && return
+    for c in "${_RW_TRAP_STOPPED[@]}"; do
+        [[ -z "$c" ]] && continue
+        if docker inspect "$c" >/dev/null 2>&1; then
+            warn "Exit trap: restarting $c (stopped but state not persisted)"
+            docker start "$c" >/dev/null 2>&1 || warn "  Failed to restart $c"
+        fi
+    done
+}
+trap _rw_trap_restart_stopped EXIT
+
 # ==============================================================================================
 # ━━━ State Helpers ━━━
 # ==============================================================================================
@@ -379,6 +394,7 @@ stop_containers() {
        if timeout "$DOCKER_TIMEOUT" docker stop "$container" >/dev/null 2>&1; then
            warn "Stopped $container (hard pressure)"
            actually_stopped+=("$container")
+            _RW_TRAP_STOPPED+=("$container")
        else
            error "Failed to stop $container"
        fi
@@ -468,7 +484,7 @@ apply_level_3() {
 # ==============================================================================================

 restore_level_3() {
-    log "Restoring from level 3 — starting stopped containers"
+    echo "Restoring from level 3 — starting stopped containers"
    if [[ -n "$STOPPED_LIST" ]]; then
        start_containers "$STOPPED_LIST"
        STOPPED_LIST=""
@@ -478,7 +494,7 @@ restore_level_3() {
 }

 restore_level_2() {
-    log "Restoring from level 2 — unpausing containers"
+    echo "Restoring from level 2 — unpausing containers"
    if [[ -n "$PAUSED_LIST" ]]; then
        unpause_containers "$PAUSED_LIST"
        PAUSED_LIST=""
@@ -486,7 +502,7 @@ restore_level_2() {
 }

 restore_level_1() {
-    log "Restoring from level 1 — removing downloader throttle"
+    echo "Restoring from level 1 — removing downloader throttle"
    sabnzbd_set_speed "0"
    qbit_set_dl_limit 0
 }
@@ -552,9 +568,9 @@ elif [[ "$TARGET_LEVEL" -lt "$CURRENT_LEVEL" ]]; then
 else
    # ── Steady state ────────────────────────────────────────────────────────────────────────
    if [[ "$CURRENT_LEVEL" -gt 0 ]]; then
-        log "Pressure holding at level $CURRENT_LEVEL — waiting for sustained recovery"
+        echo "Pressure holding at level $CURRENT_LEVEL — waiting for sustained recovery"
    else
-        log "System at normal pressure ✅"
+        echo "System at normal pressure ✅"
    fi
    rm_state_set "rm_recover_cycles" 0
 fi
@@ -564,5 +580,6 @@ fi
 # ==============================================================================================
 rm_state_set "rm_paused_containers" "$PAUSED_LIST"
 rm_state_set "rm_stopped_containers" "$STOPPED_LIST"
+trap - EXIT  # state persisted — stopped containers recorded, trap no longer needed
 # Touch state file each run so docker_watchdog stale guard sees fresh mtime
 touch "$RW_STATE_FILE" 2>/dev/null
@@ -501,7 +501,7 @@ echo ""
 if [[ "$DRY_RUN" == true ]]; then
    warn "DRY RUN — no changes made"
 else
-    log "$ICON_DONE Status: done ✅"
+    echo "$ICON_DONE Status: done ✅"
 fi
 echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

@@ -121,6 +121,15 @@ acquire_lock

 detect_hosts

+# ── Exit Trap — restart Docker service if reboot sequence aborts after stopping it ────────────
+_REBOOT_DOCKER_STOPPED=false
+_trap_restart_docker_service() {
+    [[ "$_REBOOT_DOCKER_STOPPED" == true ]] || return
+    warn "Exit trap: restarting Docker service after aborted reboot sequence"
+    /etc/rc.d/rc.docker start >/dev/null 2>&1 || true
+}
+trap _trap_restart_docker_service EXIT
+
 [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made, no reboot will occur"

 # ==============================================================================================
@@ -280,6 +289,7 @@ if [[ "$DRY_RUN" == true ]]; then
    warn "DRY RUN — would stop Docker service"
 else
    if /etc/rc.d/rc.docker stop >/dev/null 2>&1; then
+        _REBOOT_DOCKER_STOPPED=true
        warn "Docker stopped ✅"
    else
        warn "Docker stop returned non-zero — may already be stopped"
@@ -317,5 +327,6 @@ if [[ "$DRY_RUN" == true ]]; then
 else
    warn "$ICON_REBOOT Rebooting $MY_ID now..."
    echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
+    trap - EXIT  # committed to reboot — Docker should stay down
    /sbin/reboot
 fi
@@ -350,6 +350,17 @@ run_strike_check() {
    return 1
 }

+# ── Exit Trap — restart containers stopped before an aborted reboot ───────────────────────────
+_SYS_REBOOT_STOPPED=()
+_trap_sys_reboot_restart() {
+    [[ ${#_SYS_REBOOT_STOPPED[@]} -eq 0 ]] && return
+    warn "Exit trap: restarting containers stopped before aborted reboot"
+    for c in "${_SYS_REBOOT_STOPPED[@]}"; do
+        [[ -z "$c" ]] && continue
+        docker inspect "$c" >/dev/null 2>&1 && docker start "$c" >/dev/null 2>&1 || true
+    done
+}
+
 # ==============================================================================================
 # ── DO REBOOT ─────────────────────────────────────────────────────────────────────────────────
 # ==============================================================================================
@@ -424,6 +435,8 @@ do_reboot() {

    warn "Stopping Docker containers..."
    if command -v docker >/dev/null 2>&1; then
+        mapfile -t _SYS_REBOOT_STOPPED < <(docker ps --format '{{.Names}}' 2>/dev/null)
+        trap _trap_sys_reboot_restart EXIT
        timeout 60 docker ps -q 2>/dev/null | xargs -r docker stop >/dev/null 2>&1
    fi

@@ -433,6 +446,7 @@ do_reboot() {
    warn "Syncing disks..."
    sync

+    trap - EXIT  # committed to reboot — containers should stay down
    sleep 5
    /sbin/reboot
 }
@@ -784,7 +798,7 @@ echo "━━━━━━━━━━━━━━━━━━━━━━━━
        do_reboot "standard" "${TRIGGERS[@]}"
        exit 0
    else
-        log "System healthy ✅ ($(date '+%H:%M:%S'))"
+        echo "System healthy ✅ ($(date '+%H:%M:%S'))"
    fi

    # Keep state file mtime fresh — docker_watchdog stale guard checks this
@@ -172,7 +172,7 @@ FAILED=()
 SKIPPED=()

 if [[ ${#PIDS[@]} -eq 0 ]]; then
-    log "No User Script processes running — nothing to do"
+    echo "No User Script processes running — nothing to do"
 else
    warn "${#PIDS[@]} User Script process(es) found"
    echo ""
@@ -232,7 +232,7 @@ echo "$ICON_TIME   Duration: $(format_duration $(( END - START )))"
 echo ""

 if [[ ${#PIDS[@]} -eq 0 ]]; then
-    log "No processes were running"
+    echo "No processes were running"
 elif [[ "$DRY_RUN" == true ]]; then
    warn "DRY RUN — would have stopped ${#SKIPPED[@]} process(es): ${SKIPPED[*]}"
 else
@@ -248,7 +248,7 @@ elif [[ ${#FAILED[@]} -gt 0 ]]; then
    notify "User Scripts stop failed on $(hostname) ($MY_ID) — unkillable: ${FAILED[*]}" \
        "User Scripts Stop" "warning"
 else
-    log "$ICON_DONE Status: done ✅"
+    echo "$ICON_DONE Status: done ✅"
 fi
 echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"