unRAID_Essentials echo/log audit pass + exit traps
Exit traps: resource_watchdog registers/clears trap for stopped containers; server_reboot registers trap to restart Docker service on abort; system_watchdog records containers before bulk stop and registers trap to restart if reboot aborts. echo/log audit: status conclusions, nothing-to-do, and clean-cycle confirmations → echo across clear_logs, docker_syslog_filter, mover_stop, php_fpm_max_children, rsync_stop, user_scripts_stop. resource_watchdog: disabled flag, restore-level messages, pressure status → echo. system_watchdog: clean-cycle summary → echo. Manual-Unraid_Essentials.md: added Output Tiers section.
This commit is contained in:
@@ -25,6 +25,22 @@ adding or reordering scripts at array start.
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
## Output Tiers
|
||||||
|
|
||||||
|
All scripts use a two-tier output model: `echo` lines are always visible; `log`
|
||||||
|
lines only appear when `--log` is passed.
|
||||||
|
|
||||||
|
**Daemon scripts** (`system_watchdog.sh`, `resource_watchdog.sh`, `webgui_restart.sh`):
|
||||||
|
run on every cycle. Without `--log`, only state transitions, warnings, errors, and
|
||||||
|
the clean-cycle conclusion line are visible. Per-check detail suppressed.
|
||||||
|
|
||||||
|
**One-shot scripts** (`clear_logs.sh`, `docker_syslog_filter.sh`, `inotify_tuning.sh`,
|
||||||
|
`mover_stop.sh`, `php_fpm_max_children.sh`, `rsync_stop.sh`, `server_reboot.sh`,
|
||||||
|
`user_scripts_stop.sh`): without `--log`, section headers, per-step results, and the
|
||||||
|
final summary are visible. Per-item detail suppressed.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
## ARRAY_START_SCRIPTS Order
|
## ARRAY_START_SCRIPTS Order
|
||||||
|
|
||||||
> **The order of scripts in ARRAY_START_SCRIPTS matters for three of these
|
> **The order of scripts in ARRAY_START_SCRIPTS matters for three of these
|
||||||
|
|||||||
@@ -277,12 +277,12 @@ if [[ "$TOTAL_CLEARED" -gt 0 || ${#FAILED[@]} -gt 0 ]]; then
|
|||||||
notify "Log clear failed on $(hostname) ($MY_ID) — ${FAILED[*]}" \
|
notify "Log clear failed on $(hostname) ($MY_ID) — ${FAILED[*]}" \
|
||||||
"Clear Logs" "warning"
|
"Clear Logs" "warning"
|
||||||
else
|
else
|
||||||
log "$ICON_DONE Status: done — ${TOTAL_FREED_H} freed"
|
echo "$ICON_DONE Status: done — ${TOTAL_FREED_H} freed"
|
||||||
fi
|
fi
|
||||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||||
else
|
else
|
||||||
# All logs under threshold — completely silent
|
# All logs under threshold — completely silent
|
||||||
log "All logs under threshold — nothing to clear"
|
echo "All logs under threshold — nothing to clear"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
[[ ${#FAILED[@]} -gt 0 ]] && exit 1
|
[[ ${#FAILED[@]} -gt 0 ]] && exit 1
|
||||||
|
|||||||
@@ -189,12 +189,12 @@ fi
|
|||||||
if [[ "$DRY_RUN" == true ]]; then
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
warn "DRY RUN — would restart rsyslog"
|
warn "DRY RUN — would restart rsyslog"
|
||||||
else
|
else
|
||||||
log "Restarting rsyslog..."
|
echo "Restarting rsyslog..."
|
||||||
if /etc/rc.d/rc.rsyslogd restart >/dev/null 2>&1; then
|
if /etc/rc.d/rc.rsyslogd restart >/dev/null 2>&1; then
|
||||||
sleep 2
|
sleep 2
|
||||||
# Verify rsyslog actually running after restart
|
# Verify rsyslog actually running after restart
|
||||||
if pgrep -x rsyslogd >/dev/null 2>&1; then
|
if pgrep -x rsyslogd >/dev/null 2>&1; then
|
||||||
log "rsyslog restarted and running ✅"
|
echo "rsyslog restarted and running ✅"
|
||||||
else
|
else
|
||||||
error "rsyslog not running after restart"
|
error "rsyslog not running after restart"
|
||||||
notify "rsyslog failed to start after filter update on $(hostname) ($MY_ID)" \
|
notify "rsyslog failed to start after filter update on $(hostname) ($MY_ID)" \
|
||||||
@@ -224,7 +224,7 @@ echo ""
|
|||||||
if [[ "$DRY_RUN" == true ]]; then
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
warn "DRY RUN — no changes made"
|
warn "DRY RUN — no changes made"
|
||||||
else
|
else
|
||||||
log "$ICON_DONE Status: done — Docker veth noise suppressed ✅"
|
echo "$ICON_DONE Status: done — Docker veth noise suppressed ✅"
|
||||||
fi
|
fi
|
||||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||||
|
|
||||||
|
|||||||
@@ -127,7 +127,7 @@ fi
|
|||||||
START=$(date +%s)
|
START=$(date +%s)
|
||||||
|
|
||||||
if ! pgrep -f "emhttp.*Mover" >/dev/null 2>&1; then
|
if ! pgrep -f "emhttp.*Mover" >/dev/null 2>&1; then
|
||||||
log "Mover is not running — nothing to do"
|
echo "Mover is not running — nothing to do"
|
||||||
exit 0
|
exit 0
|
||||||
fi
|
fi
|
||||||
|
|
||||||
@@ -137,7 +137,7 @@ warn "Mover is running (PID $MOVER_PID) — stopping in ${MOVER_STOP_TIMEOUT}s"
|
|||||||
# ── Warn users via wall ───────────────────────────────────────────────────────────────────────
|
# ── Warn users via wall ───────────────────────────────────────────────────────────────────────
|
||||||
if [[ "$DRY_RUN" == false ]]; then
|
if [[ "$DRY_RUN" == false ]]; then
|
||||||
wall "$ICON_WARN $MY_ID ($LOCAL_SERVER_NAME) — unRAID Mover stopping in ${MOVER_STOP_TIMEOUT}s"
|
wall "$ICON_WARN $MY_ID ($LOCAL_SERVER_NAME) — unRAID Mover stopping in ${MOVER_STOP_TIMEOUT}s"
|
||||||
log "Wall message sent — waiting ${MOVER_STOP_TIMEOUT}s..."
|
echo "Wall message sent — waiting ${MOVER_STOP_TIMEOUT}s..."
|
||||||
sleep "$MOVER_STOP_TIMEOUT"
|
sleep "$MOVER_STOP_TIMEOUT"
|
||||||
else
|
else
|
||||||
warn "DRY RUN — would send wall warning and wait ${MOVER_STOP_TIMEOUT}s"
|
warn "DRY RUN — would send wall warning and wait ${MOVER_STOP_TIMEOUT}s"
|
||||||
@@ -187,6 +187,6 @@ echo ""
|
|||||||
if [[ "$DRY_RUN" == true ]]; then
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
warn "DRY RUN — no changes made"
|
warn "DRY RUN — no changes made"
|
||||||
else
|
else
|
||||||
log "$ICON_DONE Status: done — mover stopped ✅"
|
echo "$ICON_DONE Status: done — mover stopped ✅"
|
||||||
fi
|
fi
|
||||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||||
@@ -238,7 +238,7 @@ echo "$ICON_GEAR Config file: $PHP_CONF"
|
|||||||
echo "$ICON_PHP Applied: pm.max_children = $PHP_MAX_CHILDREN"
|
echo "$ICON_PHP Applied: pm.max_children = $PHP_MAX_CHILDREN"
|
||||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||||
echo ""
|
echo ""
|
||||||
log "$ICON_DONE Status: done ✅"
|
echo "$ICON_DONE Status: done ✅"
|
||||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||||
|
|
||||||
exit 0
|
exit 0
|
||||||
@@ -118,7 +118,7 @@ if [[ "$EUID" -ne 0 ]]; then
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
if [[ "${RW_ENABLED:-true}" != "true" ]]; then
|
if [[ "${RW_ENABLED:-true}" != "true" ]]; then
|
||||||
log "Resource Manager disabled (RW_ENABLED=false)"
|
echo "Resource Manager disabled (RW_ENABLED=false)"
|
||||||
exit 0
|
exit 0
|
||||||
fi
|
fi
|
||||||
|
|
||||||
@@ -138,6 +138,21 @@ touch "$RW_STATE_FILE" 2>/dev/null || {
|
|||||||
exit 1
|
exit 1
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# ── Exit Trap — restart containers stopped this run if script crashes ──────────────────────────
|
||||||
|
declare -a _RW_TRAP_STOPPED=()
|
||||||
|
|
||||||
|
_rw_trap_restart_stopped() {
|
||||||
|
[[ ${#_RW_TRAP_STOPPED[@]} -eq 0 ]] && return
|
||||||
|
for c in "${_RW_TRAP_STOPPED[@]}"; do
|
||||||
|
[[ -z "$c" ]] && continue
|
||||||
|
if docker inspect "$c" >/dev/null 2>&1; then
|
||||||
|
warn "Exit trap: restarting $c (stopped but state not persisted)"
|
||||||
|
docker start "$c" >/dev/null 2>&1 || warn " Failed to restart $c"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
}
|
||||||
|
trap _rw_trap_restart_stopped EXIT
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ State Helpers ━━━
|
# ━━━ State Helpers ━━━
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
@@ -379,6 +394,7 @@ stop_containers() {
|
|||||||
if timeout "$DOCKER_TIMEOUT" docker stop "$container" >/dev/null 2>&1; then
|
if timeout "$DOCKER_TIMEOUT" docker stop "$container" >/dev/null 2>&1; then
|
||||||
warn "Stopped $container (hard pressure)"
|
warn "Stopped $container (hard pressure)"
|
||||||
actually_stopped+=("$container")
|
actually_stopped+=("$container")
|
||||||
|
_RW_TRAP_STOPPED+=("$container")
|
||||||
else
|
else
|
||||||
error "Failed to stop $container"
|
error "Failed to stop $container"
|
||||||
fi
|
fi
|
||||||
@@ -468,7 +484,7 @@ apply_level_3() {
|
|||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
|
|
||||||
restore_level_3() {
|
restore_level_3() {
|
||||||
log "Restoring from level 3 — starting stopped containers"
|
echo "Restoring from level 3 — starting stopped containers"
|
||||||
if [[ -n "$STOPPED_LIST" ]]; then
|
if [[ -n "$STOPPED_LIST" ]]; then
|
||||||
start_containers "$STOPPED_LIST"
|
start_containers "$STOPPED_LIST"
|
||||||
STOPPED_LIST=""
|
STOPPED_LIST=""
|
||||||
@@ -478,7 +494,7 @@ restore_level_3() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
restore_level_2() {
|
restore_level_2() {
|
||||||
log "Restoring from level 2 — unpausing containers"
|
echo "Restoring from level 2 — unpausing containers"
|
||||||
if [[ -n "$PAUSED_LIST" ]]; then
|
if [[ -n "$PAUSED_LIST" ]]; then
|
||||||
unpause_containers "$PAUSED_LIST"
|
unpause_containers "$PAUSED_LIST"
|
||||||
PAUSED_LIST=""
|
PAUSED_LIST=""
|
||||||
@@ -486,7 +502,7 @@ restore_level_2() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
restore_level_1() {
|
restore_level_1() {
|
||||||
log "Restoring from level 1 — removing downloader throttle"
|
echo "Restoring from level 1 — removing downloader throttle"
|
||||||
sabnzbd_set_speed "0"
|
sabnzbd_set_speed "0"
|
||||||
qbit_set_dl_limit 0
|
qbit_set_dl_limit 0
|
||||||
}
|
}
|
||||||
@@ -552,9 +568,9 @@ elif [[ "$TARGET_LEVEL" -lt "$CURRENT_LEVEL" ]]; then
|
|||||||
else
|
else
|
||||||
# ── Steady state ────────────────────────────────────────────────────────────────────────
|
# ── Steady state ────────────────────────────────────────────────────────────────────────
|
||||||
if [[ "$CURRENT_LEVEL" -gt 0 ]]; then
|
if [[ "$CURRENT_LEVEL" -gt 0 ]]; then
|
||||||
log "Pressure holding at level $CURRENT_LEVEL — waiting for sustained recovery"
|
echo "Pressure holding at level $CURRENT_LEVEL — waiting for sustained recovery"
|
||||||
else
|
else
|
||||||
log "System at normal pressure ✅"
|
echo "System at normal pressure ✅"
|
||||||
fi
|
fi
|
||||||
rm_state_set "rm_recover_cycles" 0
|
rm_state_set "rm_recover_cycles" 0
|
||||||
fi
|
fi
|
||||||
@@ -564,5 +580,6 @@ fi
|
|||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
rm_state_set "rm_paused_containers" "$PAUSED_LIST"
|
rm_state_set "rm_paused_containers" "$PAUSED_LIST"
|
||||||
rm_state_set "rm_stopped_containers" "$STOPPED_LIST"
|
rm_state_set "rm_stopped_containers" "$STOPPED_LIST"
|
||||||
|
trap - EXIT # state persisted — stopped containers recorded, trap no longer needed
|
||||||
# Touch state file each run so docker_watchdog stale guard sees fresh mtime
|
# Touch state file each run so docker_watchdog stale guard sees fresh mtime
|
||||||
touch "$RW_STATE_FILE" 2>/dev/null
|
touch "$RW_STATE_FILE" 2>/dev/null
|
||||||
|
|||||||
@@ -501,7 +501,7 @@ echo ""
|
|||||||
if [[ "$DRY_RUN" == true ]]; then
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
warn "DRY RUN — no changes made"
|
warn "DRY RUN — no changes made"
|
||||||
else
|
else
|
||||||
log "$ICON_DONE Status: done ✅"
|
echo "$ICON_DONE Status: done ✅"
|
||||||
fi
|
fi
|
||||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||||
|
|
||||||
|
|||||||
@@ -121,6 +121,15 @@ acquire_lock
|
|||||||
|
|
||||||
detect_hosts
|
detect_hosts
|
||||||
|
|
||||||
|
# ── Exit Trap — restart Docker service if reboot sequence aborts after stopping it ────────────
|
||||||
|
_REBOOT_DOCKER_STOPPED=false
|
||||||
|
_trap_restart_docker_service() {
|
||||||
|
[[ "$_REBOOT_DOCKER_STOPPED" == true ]] || return
|
||||||
|
warn "Exit trap: restarting Docker service after aborted reboot sequence"
|
||||||
|
/etc/rc.d/rc.docker start >/dev/null 2>&1 || true
|
||||||
|
}
|
||||||
|
trap _trap_restart_docker_service EXIT
|
||||||
|
|
||||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made, no reboot will occur"
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made, no reboot will occur"
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
@@ -280,6 +289,7 @@ if [[ "$DRY_RUN" == true ]]; then
|
|||||||
warn "DRY RUN — would stop Docker service"
|
warn "DRY RUN — would stop Docker service"
|
||||||
else
|
else
|
||||||
if /etc/rc.d/rc.docker stop >/dev/null 2>&1; then
|
if /etc/rc.d/rc.docker stop >/dev/null 2>&1; then
|
||||||
|
_REBOOT_DOCKER_STOPPED=true
|
||||||
warn "Docker stopped ✅"
|
warn "Docker stopped ✅"
|
||||||
else
|
else
|
||||||
warn "Docker stop returned non-zero — may already be stopped"
|
warn "Docker stop returned non-zero — may already be stopped"
|
||||||
@@ -317,5 +327,6 @@ if [[ "$DRY_RUN" == true ]]; then
|
|||||||
else
|
else
|
||||||
warn "$ICON_REBOOT Rebooting $MY_ID now..."
|
warn "$ICON_REBOOT Rebooting $MY_ID now..."
|
||||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||||
|
trap - EXIT # committed to reboot — Docker should stay down
|
||||||
/sbin/reboot
|
/sbin/reboot
|
||||||
fi
|
fi
|
||||||
@@ -350,6 +350,17 @@ run_strike_check() {
|
|||||||
return 1
|
return 1
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# ── Exit Trap — restart containers stopped before an aborted reboot ───────────────────────────
|
||||||
|
_SYS_REBOOT_STOPPED=()
|
||||||
|
_trap_sys_reboot_restart() {
|
||||||
|
[[ ${#_SYS_REBOOT_STOPPED[@]} -eq 0 ]] && return
|
||||||
|
warn "Exit trap: restarting containers stopped before aborted reboot"
|
||||||
|
for c in "${_SYS_REBOOT_STOPPED[@]}"; do
|
||||||
|
[[ -z "$c" ]] && continue
|
||||||
|
docker inspect "$c" >/dev/null 2>&1 && docker start "$c" >/dev/null 2>&1 || true
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ── DO REBOOT ─────────────────────────────────────────────────────────────────────────────────
|
# ── DO REBOOT ─────────────────────────────────────────────────────────────────────────────────
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
@@ -424,6 +435,8 @@ do_reboot() {
|
|||||||
|
|
||||||
warn "Stopping Docker containers..."
|
warn "Stopping Docker containers..."
|
||||||
if command -v docker >/dev/null 2>&1; then
|
if command -v docker >/dev/null 2>&1; then
|
||||||
|
mapfile -t _SYS_REBOOT_STOPPED < <(docker ps --format '{{.Names}}' 2>/dev/null)
|
||||||
|
trap _trap_sys_reboot_restart EXIT
|
||||||
timeout 60 docker ps -q 2>/dev/null | xargs -r docker stop >/dev/null 2>&1
|
timeout 60 docker ps -q 2>/dev/null | xargs -r docker stop >/dev/null 2>&1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
@@ -433,6 +446,7 @@ do_reboot() {
|
|||||||
warn "Syncing disks..."
|
warn "Syncing disks..."
|
||||||
sync
|
sync
|
||||||
|
|
||||||
|
trap - EXIT # committed to reboot — containers should stay down
|
||||||
sleep 5
|
sleep 5
|
||||||
/sbin/reboot
|
/sbin/reboot
|
||||||
}
|
}
|
||||||
@@ -784,7 +798,7 @@ echo "━━━━━━━━━━━━━━━━━━━━━━━━
|
|||||||
do_reboot "standard" "${TRIGGERS[@]}"
|
do_reboot "standard" "${TRIGGERS[@]}"
|
||||||
exit 0
|
exit 0
|
||||||
else
|
else
|
||||||
log "System healthy ✅ ($(date '+%H:%M:%S'))"
|
echo "System healthy ✅ ($(date '+%H:%M:%S'))"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Keep state file mtime fresh — docker_watchdog stale guard checks this
|
# Keep state file mtime fresh — docker_watchdog stale guard checks this
|
||||||
|
|||||||
@@ -172,7 +172,7 @@ FAILED=()
|
|||||||
SKIPPED=()
|
SKIPPED=()
|
||||||
|
|
||||||
if [[ ${#PIDS[@]} -eq 0 ]]; then
|
if [[ ${#PIDS[@]} -eq 0 ]]; then
|
||||||
log "No User Script processes running — nothing to do"
|
echo "No User Script processes running — nothing to do"
|
||||||
else
|
else
|
||||||
warn "${#PIDS[@]} User Script process(es) found"
|
warn "${#PIDS[@]} User Script process(es) found"
|
||||||
echo ""
|
echo ""
|
||||||
@@ -232,7 +232,7 @@ echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
|||||||
echo ""
|
echo ""
|
||||||
|
|
||||||
if [[ ${#PIDS[@]} -eq 0 ]]; then
|
if [[ ${#PIDS[@]} -eq 0 ]]; then
|
||||||
log "No processes were running"
|
echo "No processes were running"
|
||||||
elif [[ "$DRY_RUN" == true ]]; then
|
elif [[ "$DRY_RUN" == true ]]; then
|
||||||
warn "DRY RUN — would have stopped ${#SKIPPED[@]} process(es): ${SKIPPED[*]}"
|
warn "DRY RUN — would have stopped ${#SKIPPED[@]} process(es): ${SKIPPED[*]}"
|
||||||
else
|
else
|
||||||
@@ -248,7 +248,7 @@ elif [[ ${#FAILED[@]} -gt 0 ]]; then
|
|||||||
notify "User Scripts stop failed on $(hostname) ($MY_ID) — unkillable: ${FAILED[*]}" \
|
notify "User Scripts stop failed on $(hostname) ($MY_ID) — unkillable: ${FAILED[*]}" \
|
||||||
"User Scripts Stop" "warning"
|
"User Scripts Stop" "warning"
|
||||||
else
|
else
|
||||||
log "$ICON_DONE Status: done ✅"
|
echo "$ICON_DONE Status: done ✅"
|
||||||
fi
|
fi
|
||||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user