massive update. Master conf split, now modular with a load sceriprt to drive all configs to scripts. with unraid scpecific safeguard tests , and improved standardized ux. including dynamic host detect, who am i who else it there. EVERY SINGLE SCRIPT UPDATED. DEBATING THAT THIS IS ACUALLY V2
This commit is contained in:
File diff suppressed because it is too large
Load Diff
+222
-102
@@ -1,134 +1,254 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- Clear Logs Script ------------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Clears unRAID system logs and Docker container logs safely.
|
||||
# Log file paths are configured in Master.conf under LOG_FILES.
|
||||
# Supports --dry-run to preview what would be cleared without making changes.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ================================= Clear Logs =================================================
|
||||
# ==============================================================================================
|
||||
# Clears system and Docker container logs to prevent rootfs fill over time.
|
||||
# Runs weekly via WEEKLY_MAINTENANCE_SCRIPTS — Sunday 2:30am.
|
||||
# Uses size thresholds — only clears logs that have grown large enough to matter.
|
||||
#
|
||||
# ── WHAT IT CLEARS ────────────────────────────────────────────────────────────────────────────
|
||||
# System logs — LOG_FILES from master.conf (/var/log/syslog, messages, dmesg)
|
||||
# Cleared if size exceeds LOG_MIN_SIZE_MB
|
||||
# These grow continuously — weekly clearing keeps rootfs healthy
|
||||
#
|
||||
# Docker logs — /var/lib/docker/containers/**/*-json.log
|
||||
# Cleared only if individual container log exceeds LOG_DOCKER_MAX_MB
|
||||
# Active containers (Emby, SABnzbd) grow fastest — 100MB+ easily
|
||||
# Inactive containers not cleared — their logs are typically small
|
||||
#
|
||||
# ── SIZE THRESHOLD APPROACH ───────────────────────────────────────────────────────────────────
|
||||
# Truncating everything blindly destroys useful diagnostic context.
|
||||
# A 2MB log is not worth clearing — it contains useful recent history.
|
||||
# A 500MB log is consuming rootfs and contains mostly noise — clear it.
|
||||
#
|
||||
# LOG_MIN_SIZE_MB — system logs under this size are left alone
|
||||
# LOG_DOCKER_MAX_MB — Docker logs under this size are left alone
|
||||
#
|
||||
# ── WHY NOT LOGROTATE ─────────────────────────────────────────────────────────────────────────
|
||||
# unRAID writes to tmpfs (/var/log) — logrotate's compress + archive approach
|
||||
# would consume even more tmpfs space. Truncation (: > file) keeps the file
|
||||
# descriptor open and valid while emptying content — safe for running services.
|
||||
#
|
||||
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
|
||||
# acquire_lock — prevents concurrent runs corrupting logs
|
||||
# Root check — truncating system logs requires root
|
||||
# Size thresholds — only clears logs that have grown large enough
|
||||
# Byte tracking — reports MB freed for weekly digest
|
||||
# validate_unraid — notify validated before use
|
||||
# Silent on clean — small logs = nothing to clear = no output ✅
|
||||
#
|
||||
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||||
# LOG_FILES — system log paths to check and clear
|
||||
# LOG_MIN_SIZE_MB — minimum system log size before clearing (default 10MB)
|
||||
# LOG_DOCKER_MAX_MB — clear Docker log only if above this size (default 100MB)
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# clear_logs.sh — normal run (threshold-based clearing)
|
||||
# clear_logs.sh --dry-run — show what would be cleared and sizes
|
||||
# clear_logs.sh --status — show current log sizes
|
||||
# clear_logs.sh --log — verbose output per file
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
|
||||
# ROOT CHECK
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
error "Must be run as root — truncating system logs requires root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Running as root"
|
||||
validate_unraid_cmd \
|
||||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||||
"" "" \
|
||||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Status ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
acquire_lock
|
||||
|
||||
detect_hosts
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be cleared"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_HEALTH System Logs: ${LOG_FILES[*]}"
|
||||
echo "$ICON_CONTAINERS Docker Logs: /var/lib/docker/containers"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "━━━━━ $ICON_SUMMARY LOG STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_GEAR Min system log: ${LOG_MIN_SIZE_MB:-10}MB before clearing"
|
||||
echo "$ICON_GEAR Max Docker log: ${LOG_DOCKER_MAX_MB:-100}MB before clearing"
|
||||
echo ""
|
||||
|
||||
echo "━━━ System Logs ━━━"
|
||||
for f in "${LOG_FILES[@]}"; do
|
||||
if [[ -f "$f" ]]; then
|
||||
size=$(du -sh "$f" 2>/dev/null | cut -f1)
|
||||
size_mb=$(du -sm "$f" 2>/dev/null | cut -f1)
|
||||
threshold="${LOG_MIN_SIZE_MB:-10}"
|
||||
if [[ "${size_mb:-0}" -ge "$threshold" ]]; then
|
||||
echo " $ICON_WARN $f — $size (above ${threshold}MB threshold — would clear)"
|
||||
else
|
||||
echo " $ICON_SUCCESS $f — $size (under threshold)"
|
||||
fi
|
||||
else
|
||||
echo " $ICON_SKIP $f — not found"
|
||||
fi
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "━━━ Docker Logs (top 10 by size) ━━━"
|
||||
if [[ -d /var/lib/docker/containers ]]; then
|
||||
find /var/lib/docker/containers/ -name "*-json.log" 2>/dev/null | \
|
||||
while IFS= read -r logfile; do
|
||||
size_mb=$(du -sm "$logfile" 2>/dev/null | cut -f1)
|
||||
container_id=$(basename "$(dirname "$logfile")" | cut -c1-12)
|
||||
container_name=$(docker inspect --format '{{.Name}}' "$container_id" \
|
||||
2>/dev/null | tr -d '/' || echo "$container_id")
|
||||
echo "${size_mb:-0} $container_name $logfile"
|
||||
done | sort -rn | head -10 | \
|
||||
while read -r size_mb name logfile; do
|
||||
threshold="${LOG_DOCKER_MAX_MB:-100}"
|
||||
if [[ "$size_mb" -ge "$threshold" ]]; then
|
||||
echo " $ICON_WARN ${size_mb}MB — $name (above ${threshold}MB — would clear)"
|
||||
else
|
||||
echo " $ICON_SUCCESS ${size_mb}MB — $name"
|
||||
fi
|
||||
done
|
||||
else
|
||||
echo " Docker directory not found"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# FUNCTIONS
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Clear Logs ━━━
|
||||
# ==============================================================================================
|
||||
START=$(date +%s)
|
||||
SYS_CLEARED=0
|
||||
SYS_SKIPPED=0
|
||||
SYS_BYTES=0
|
||||
DOCKER_CLEARED=0
|
||||
DOCKER_SKIPPED=0
|
||||
DOCKER_BYTES=0
|
||||
FAILED=()
|
||||
|
||||
# Clears a single log file if it exists.
|
||||
# Skips with a warning if the file is not found.
|
||||
clear_file() {
|
||||
local file="$1"
|
||||
# ── System Logs ───────────────────────────────────────────────────────────────────────────────
|
||||
for logfile in "${LOG_FILES[@]}"; do
|
||||
if [[ ! -f "$logfile" ]]; then
|
||||
log "$logfile — not found, skipping"
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ ! -f "$file" ]]; then
|
||||
warn "Not found: $file — skipping"
|
||||
return
|
||||
size_bytes=$(stat -c%s "$logfile" 2>/dev/null || echo 0)
|
||||
size_mb=$(( size_bytes / 1048576 ))
|
||||
size_h=$(du -sh "$logfile" 2>/dev/null | cut -f1)
|
||||
threshold="${LOG_MIN_SIZE_MB:-10}"
|
||||
|
||||
if [[ "$size_mb" -lt "$threshold" ]]; then
|
||||
log "$logfile — ${size_h} (under ${threshold}MB — skipping)"
|
||||
(( SYS_SKIPPED++ ))
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would clear: $file"
|
||||
warn "DRY RUN — would clear: $logfile (${size_h})"
|
||||
(( SYS_CLEARED++ ))
|
||||
SYS_BYTES=$(( SYS_BYTES + size_bytes ))
|
||||
continue
|
||||
fi
|
||||
|
||||
if : > "$logfile" 2>/dev/null; then
|
||||
log "Cleared: $logfile (freed ${size_h})"
|
||||
(( SYS_CLEARED++ ))
|
||||
SYS_BYTES=$(( SYS_BYTES + size_bytes ))
|
||||
else
|
||||
: > "$file"
|
||||
success "Cleared: $file"
|
||||
error "Failed to clear: $logfile"
|
||||
FAILED+=("$logfile")
|
||||
fi
|
||||
}
|
||||
|
||||
# Finds and clears all Docker container json log files.
|
||||
# Skips gracefully if Docker directory or log files are not found.
|
||||
clear_docker_logs() {
|
||||
if [[ ! -d /var/lib/docker/containers ]]; then
|
||||
warn "$ICON_CONTAINERS Docker directory not found — skipping"
|
||||
return
|
||||
fi
|
||||
|
||||
local files
|
||||
files=$(find /var/lib/docker/containers/ -name "*-json.log" 2>/dev/null || true)
|
||||
|
||||
if [[ -z "$files" ]]; then
|
||||
warn "$ICON_CONTAINERS No Docker logs found — skipping"
|
||||
return
|
||||
fi
|
||||
|
||||
while IFS= read -r file; do
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would clear Docker log: $file"
|
||||
else
|
||||
: > "$file"
|
||||
success "Cleared Docker log: $(basename "$(dirname "$file")")"
|
||||
fi
|
||||
done <<< "$files"
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_HEALTH Clear Logs ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_HEALTH Clear Logs ━━━"
|
||||
echo "$ICON_HEALTH System Logs: ${LOG_FILES[*]}"
|
||||
echo "$ICON_CONTAINERS Docker Logs: enabled"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo ""
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
|
||||
START=$(date +%s)
|
||||
CLEAR_FAILED=false
|
||||
|
||||
for logfile in "${LOG_FILES[@]}"; do
|
||||
clear_file "$logfile" || CLEAR_FAILED=true
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "━━━ $ICON_CONTAINERS Docker ━━━"
|
||||
clear_docker_logs
|
||||
# ── Docker Logs ───────────────────────────────────────────────────────────────────────────────
|
||||
if [[ ! -d /var/lib/docker/containers ]]; then
|
||||
log "Docker containers directory not found — skipping Docker log clear"
|
||||
else
|
||||
while IFS= read -r logfile; do
|
||||
[[ -z "$logfile" ]] && continue
|
||||
|
||||
size_bytes=$(stat -c%s "$logfile" 2>/dev/null || echo 0)
|
||||
size_mb=$(( size_bytes / 1048576 ))
|
||||
size_h=$(du -sh "$logfile" 2>/dev/null | cut -f1)
|
||||
threshold="${LOG_DOCKER_MAX_MB:-100}"
|
||||
|
||||
# Get container name for display
|
||||
container_id=$(basename "$(dirname "$logfile")" | cut -c1-12)
|
||||
container_name=$(docker inspect --format '{{.Name}}' "$container_id" \
|
||||
2>/dev/null | tr -d '/' || echo "$container_id")
|
||||
|
||||
if [[ "$size_mb" -lt "$threshold" ]]; then
|
||||
log "Docker $container_name — ${size_h} (under ${threshold}MB — skipping)"
|
||||
(( DOCKER_SKIPPED++ ))
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would clear Docker log: $container_name (${size_h})"
|
||||
(( DOCKER_CLEARED++ ))
|
||||
DOCKER_BYTES=$(( DOCKER_BYTES + size_bytes ))
|
||||
continue
|
||||
fi
|
||||
|
||||
if : > "$logfile" 2>/dev/null; then
|
||||
log "Cleared Docker log: $container_name (freed ${size_h})"
|
||||
(( DOCKER_CLEARED++ ))
|
||||
DOCKER_BYTES=$(( DOCKER_BYTES + size_bytes ))
|
||||
else
|
||||
error "Failed to clear Docker log: $container_name"
|
||||
FAILED+=("docker:$container_name")
|
||||
fi
|
||||
done < <(find /var/lib/docker/containers/ -name "*-json.log" 2>/dev/null)
|
||||
fi
|
||||
|
||||
END=$(date +%s)
|
||||
TOTAL_BYTES=$(( SYS_BYTES + DOCKER_BYTES ))
|
||||
TOTAL_FREED_H=$(awk "BEGIN {printf \"%.1fMB\", $TOTAL_BYTES / 1048576}")
|
||||
TOTAL_CLEARED=$(( SYS_CLEARED + DOCKER_CLEARED ))
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Summary ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY LOG CLEANER SUMMARY ━━━━━"
|
||||
echo "$ICON_HEALTH System Logs: ${LOG_FILES[*]}"
|
||||
echo "$ICON_CONTAINERS Docker Logs: $([[ "$DRY_RUN" == true ]] && echo "skipped (dry run)" || echo "cleared")"
|
||||
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "$ICON_WARN Status: DRY RUN — no changes made"
|
||||
elif [[ "$CLEAR_FAILED" == true ]]; then
|
||||
echo "$ICON_ERROR Status: $ICON_ERROR SOME LOGS FAILED TO CLEAR"
|
||||
notify "Log clear completed with errors on $(hostname)" "Clear Logs" "warning"
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$TOTAL_CLEARED" -gt 0 || ${#FAILED[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY LOG CLEANER SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_HEALTH System cleared: $SYS_CLEARED file(s)"
|
||||
echo "$ICON_CONTAINERS Docker cleared: $DOCKER_CLEARED file(s)"
|
||||
echo "$ICON_HEALTH Total freed: $TOTAL_FREED_H"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
[[ "$SYS_SKIPPED" -gt 0 || "$DOCKER_SKIPPED" -gt 0 ]] && \
|
||||
log "Skipped: ${SYS_SKIPPED} system + ${DOCKER_SKIPPED} Docker (under threshold)"
|
||||
echo ""
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no files cleared"
|
||||
elif [[ ${#FAILED[@]} -gt 0 ]]; then
|
||||
echo "$ICON_ERROR Status: SOME FILES FAILED — ${FAILED[*]}"
|
||||
notify "Log clear failed on $(hostname) ($MY_ID) — ${FAILED[*]}" \
|
||||
"Clear Logs" "warning"
|
||||
else
|
||||
log "$ICON_DONE Status: done — ${TOTAL_FREED_H} freed"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
else
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS DONE"
|
||||
notify "Logs cleared successfully on $(hostname)" "Clear Logs" "normal"
|
||||
# All logs under threshold — completely silent
|
||||
log "All logs under threshold — nothing to clear"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
@@ -1,125 +1,204 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- Docker Syslog Filter ---------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Suppresses noisy Docker veth/docker0 syslog messages on unRAID boot.
|
||||
# Creates an rsyslog filter file and restarts the rsyslog service.
|
||||
# Filter file path is configured in Master.conf under FILTER_FILE.
|
||||
# Supports --dry-run to preview what would be done without making changes.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ============================= Docker Syslog Filter ===========================================
|
||||
# ==============================================================================================
|
||||
# Suppresses noisy Docker veth/docker0 interface messages from syslog.
|
||||
# Run once at array start via ARRAY_START_SCRIPTS in master.conf.
|
||||
# Idempotent — completely silent when filter is already correct.
|
||||
#
|
||||
# ── WHY THIS EXISTS ───────────────────────────────────────────────────────────────────────────
|
||||
# Every time Docker creates or destroys a container network interface it logs messages like:
|
||||
# kernel: veth2a3b4c5: renamed from eth0
|
||||
# kernel: docker0: port 1(veth2a3b4c5) entered blocking state
|
||||
# kernel: docker0: port 1(veth2a3b4c5) entered forwarding state
|
||||
#
|
||||
# On a busy server creating and restarting many containers these fill syslog rapidly —
|
||||
# hundreds of entries per minute on container restarts, completely masking real events.
|
||||
# The filter tells rsyslog to drop these before they reach the log file.
|
||||
#
|
||||
# ── HOW IT WORKS ──────────────────────────────────────────────────────────────────────────────
|
||||
# Creates /etc/rsyslog.d/ignore-docker-veth.conf (FILTER_FILE in master.conf).
|
||||
# rsyslog processes .conf files in /etc/rsyslog.d/ automatically on startup.
|
||||
# Filter uses rsyslog's RainerScript to match messages containing "veth" or "docker0"
|
||||
# and calls stop — the message is dropped before reaching any output target.
|
||||
#
|
||||
# ── IDEMPOTENT DESIGN ─────────────────────────────────────────────────────────────────────────
|
||||
# On every array start: checks if filter file already exists with correct content.
|
||||
# If already correct → completely silent — no rsyslog restart, no output.
|
||||
# Only writes + restarts rsyslog if filter is missing or content has changed.
|
||||
# This prevents unnecessary rsyslog restarts on every boot.
|
||||
#
|
||||
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
|
||||
# Root check — writing to /etc/rsyslog.d/ requires root
|
||||
# acquire_lock — prevents concurrent runs at array start
|
||||
# Idempotent check — only restarts rsyslog when filter actually changed
|
||||
# Directory creation — mkdir -p /etc/rsyslog.d/ before writing
|
||||
# rsyslog verify — checks rsyslog running after restart
|
||||
# validate_unraid — notify validated before use
|
||||
# Silent on success — runs every boot, no noise when already correct
|
||||
#
|
||||
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||||
# FILTER_FILE — path for rsyslog drop filter (default /etc/rsyslog.d/ignore-docker-veth.conf)
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# docker_syslog_filter.sh — normal run (idempotent)
|
||||
# docker_syslog_filter.sh --dry-run — show what would change
|
||||
# docker_syslog_filter.sh --status — show filter file state and rsyslog status
|
||||
# docker_syslog_filter.sh --log — verbose output
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
# Expected filter content — used for idempotent check
|
||||
EXPECTED_FILTER='if ($msg contains "veth" or $msg contains "docker0") then {
|
||||
stop
|
||||
}'
|
||||
|
||||
# ROOT CHECK
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
error "Must be run as root — writing to /etc/rsyslog.d/ requires root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Running as root"
|
||||
validate_unraid_cmd \
|
||||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||||
"" "" \
|
||||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Status ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
acquire_lock
|
||||
|
||||
detect_hosts
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_HEALTH Filter File: $FILTER_FILE"
|
||||
echo "$ICON_CONTAINERS Targets: veth, docker0"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "━━━━━ $ICON_SUMMARY SYSLOG FILTER STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_HEALTH Filter file: $FILTER_FILE"
|
||||
echo ""
|
||||
|
||||
if [[ -f "$FILTER_FILE" ]]; then
|
||||
echo " Filter file: EXISTS"
|
||||
echo ""
|
||||
echo " Current content:"
|
||||
while IFS= read -r line; do
|
||||
echo " $line"
|
||||
done < "$FILTER_FILE"
|
||||
echo ""
|
||||
|
||||
if [[ "$(cat "$FILTER_FILE" 2>/dev/null)" == "$EXPECTED_FILTER" ]]; then
|
||||
echo " $ICON_SUCCESS Content: correct ✅"
|
||||
else
|
||||
echo " $ICON_WARN Content: differs from expected — would be rewritten"
|
||||
fi
|
||||
else
|
||||
echo " Filter file: NOT FOUND — would be created"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "━━━ rsyslog Status ━━━"
|
||||
if pgrep -x rsyslogd >/dev/null 2>&1; then
|
||||
echo " rsyslogd: running ✅"
|
||||
else
|
||||
echo " rsyslogd: NOT running"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# FUNCTIONS
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
# Creates the rsyslog filter file that suppresses veth and docker0 noise.
|
||||
# Filter is written to FILTER_FILE defined in Master.conf.
|
||||
create_filter() {
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would create filter file: $FILTER_FILE"
|
||||
return
|
||||
fi
|
||||
|
||||
info "Writing rsyslog filter file: $FILTER_FILE"
|
||||
|
||||
cat <<'EOF' > "$FILTER_FILE"
|
||||
if ($msg contains "veth" or $msg contains "docker0") then {
|
||||
stop
|
||||
}
|
||||
EOF
|
||||
|
||||
success "Filter file written"
|
||||
}
|
||||
|
||||
# Restarts the rsyslog service to apply the new filter.
|
||||
# Uses unRAID's native rc.rsyslogd script.
|
||||
restart_rsyslog() {
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would restart rsyslog service"
|
||||
return 0
|
||||
fi
|
||||
|
||||
info "Restarting rsyslog..."
|
||||
|
||||
if /etc/rc.d/rc.rsyslogd restart; then
|
||||
success "rsyslog restarted"
|
||||
return 0
|
||||
else
|
||||
error "Failed to restart rsyslog"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_HEALTH Syslog Filter ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_HEALTH Syslog Filter ━━━"
|
||||
echo "$ICON_HEALTH Filter File: $FILTER_FILE"
|
||||
echo "$ICON_CONTAINERS Targets: veth / docker0"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo ""
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
# ==============================================================================================
|
||||
# ━━━ Idempotent Check ━━━
|
||||
# ==============================================================================================
|
||||
# Already correct — completely silent
|
||||
if [[ -f "$FILTER_FILE" ]] && \
|
||||
[[ "$(cat "$FILTER_FILE" 2>/dev/null)" == "$EXPECTED_FILTER" ]]; then
|
||||
log "Filter already correct — no changes needed"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Apply Filter ━━━
|
||||
# ==============================================================================================
|
||||
START=$(date +%s)
|
||||
|
||||
create_filter
|
||||
log "Filter file missing or outdated — applying..."
|
||||
|
||||
RSYSLOG_OK=true
|
||||
restart_rsyslog || RSYSLOG_OK=false
|
||||
# Ensure rsyslog.d directory exists
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
mkdir -p "$(dirname "$FILTER_FILE")" || {
|
||||
error "Failed to create directory: $(dirname "$FILTER_FILE")"
|
||||
exit 1
|
||||
}
|
||||
fi
|
||||
|
||||
# Write filter file
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would write filter to: $FILTER_FILE"
|
||||
warn "Content:"
|
||||
echo "$EXPECTED_FILTER" | while IFS= read -r line; do
|
||||
echo " $line"
|
||||
done
|
||||
else
|
||||
echo "$EXPECTED_FILTER" > "$FILTER_FILE" || {
|
||||
error "Failed to write filter file: $FILTER_FILE"
|
||||
notify "Syslog filter write failed on $(hostname) ($MY_ID)" \
|
||||
"Syslog Filter" "warning"
|
||||
exit 1
|
||||
}
|
||||
log "Filter file written: $FILTER_FILE"
|
||||
fi
|
||||
|
||||
# Restart rsyslog to apply
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would restart rsyslog"
|
||||
else
|
||||
log "Restarting rsyslog..."
|
||||
if /etc/rc.d/rc.rsyslogd restart >/dev/null 2>&1; then
|
||||
sleep 2
|
||||
# Verify rsyslog actually running after restart
|
||||
if pgrep -x rsyslogd >/dev/null 2>&1; then
|
||||
log "rsyslog restarted and running ✅"
|
||||
else
|
||||
error "rsyslog not running after restart"
|
||||
notify "rsyslog failed to start after filter update on $(hostname) ($MY_ID)" \
|
||||
"Syslog Filter" "warning"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
error "rsyslog restart command failed"
|
||||
notify "rsyslog restart failed on $(hostname) ($MY_ID) — filter may not be active" \
|
||||
"Syslog Filter" "warning"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Summary ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY SYSLOG FILTER SUMMARY ━━━━━"
|
||||
echo "$ICON_HEALTH Filter File: $FILTER_FILE"
|
||||
echo "$ICON_CONTAINERS Targets: veth / docker0"
|
||||
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_HEALTH Filter file: $FILTER_FILE"
|
||||
echo "$ICON_HEALTH Targets: veth / docker0"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "$ICON_WARN Status: DRY RUN — no changes made"
|
||||
elif [[ "$RSYSLOG_OK" == false ]]; then
|
||||
echo "$ICON_ERROR Status: $ICON_ERROR RSYSLOG RESTART FAILED"
|
||||
notify "rsyslog restart failed on $(hostname) — syslog filter may not be active" "Syslog Filter" "warning"
|
||||
warn "DRY RUN — no changes made"
|
||||
else
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS DONE"
|
||||
notify "Syslog filter applied on $(hostname)" "Syslog Filter" "normal"
|
||||
log "$ICON_DONE Status: done — Docker veth noise suppressed ✅"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
exit 0
|
||||
@@ -1,100 +1,177 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- inotify Tuning --------------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Increases Linux inotify limits to prevent "too many open files" and inotify exhaustion.
|
||||
# Run once at array start via ARRAY_START_SCRIPTS in Master.conf.
|
||||
# ==============================================================================================
|
||||
# ================================= inotify Tuning ============================================
|
||||
# ==============================================================================================
|
||||
# Raises Linux inotify limits at array start to prevent exhaustion across the container stack.
|
||||
# Run once at array start via ARRAY_START_SCRIPTS in master.conf.
|
||||
# Settings are lost on reboot — this script reapplies them on every array start.
|
||||
#
|
||||
# Why this matters:
|
||||
# Each Docker container that watches files (Sonarr, Radarr, Lidarr, NextCloud etc.)
|
||||
# consumes inotify instances and watches. unRAID defaults are very low — with many
|
||||
# containers running you can exhaust the limit silently, causing containers to miss
|
||||
# file events (new downloads not detected, library not updated etc.)
|
||||
# ── THREE INOTIFY LIMITS ──────────────────────────────────────────────────────────────────────
|
||||
# max_user_instances — max number of independent inotify file descriptor objects per user
|
||||
# Each container that calls inotify_init() consumes one instance
|
||||
# Default 128 — exhausted quickly with 20+ active containers
|
||||
#
|
||||
# max_user_instances = max number of inotify instances per user (default: 128)
|
||||
# max_user_watches = max number of files/dirs watched per instance (default: 8192)
|
||||
# max_queued_events = max events queued before dropping (default: 16384)
|
||||
# max_user_watches — SHARED budget across ALL users and containers on the system
|
||||
# Each watched file or directory costs one watch from this pool
|
||||
# Default 8192 — VSCode alone can need 50K-200K for large workspaces
|
||||
#
|
||||
# These settings are lost on reboot — this script reapplies them at every array start.
|
||||
# All values configurable in Master.conf under unRAID Essentials.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# max_queued_events — max events buffered before kernel starts dropping them
|
||||
# Low value = events silently lost during high-activity periods
|
||||
# Default 16384 — sufficient for most setups
|
||||
#
|
||||
# ── WHY VSCODE THROWS "UNABLE TO WATCH FOR FILE CHANGES" ─────────────────────────────────────
|
||||
# VSCode (and Code-Server in Docker) opens one inotify watch per file in the workspace.
|
||||
# A typical project with node_modules can easily have 100K-200K files.
|
||||
# All containers on the host share max_user_watches — the combined usage of:
|
||||
# Sonarr, Radarr, Lidarr, Emby, Nextcloud, Code-Server, AdGuard, all other arrs
|
||||
# easily exceeds 524288 (512K) watches on a busy server.
|
||||
# Raising to 1048576 (1M) gives sufficient headroom — safe on 128GB RAM (~128MB kernel use).
|
||||
#
|
||||
# ── STARTUP ORDER MATTERS ─────────────────────────────────────────────────────────────────────
|
||||
# inotify_tuning.sh must run BEFORE containers that watch files start.
|
||||
# In ARRAY_START_SCRIPTS order: inotify_tuning.sh first, then container-starting scripts.
|
||||
# If Code-Server starts before limits are raised it inherits the old (low) limits.
|
||||
# Code-Server restart fixes this: limits are kernel-wide, not process-bound at start.
|
||||
# So if Code-Server is already running: docker restart Code-Server after this script runs.
|
||||
#
|
||||
# ── CONSUMERS ON THIS STACK ───────────────────────────────────────────────────────────────────
|
||||
# Emby — watches all media library paths (1 watch per folder)
|
||||
# Sonarr — watches TV_Shows folder tree
|
||||
# Radarr — watches Movies folder tree
|
||||
# Lidarr — watches Music folder tree
|
||||
# Nextcloud — watches data directory for changes
|
||||
# Code-Server — watches entire workspace (can be 50K-200K with node_modules)
|
||||
# AdGuard Home — watches config directory
|
||||
# + all other containers using inotify internally
|
||||
#
|
||||
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
|
||||
# acquire_lock — prevents duplicate runs at array start
|
||||
# Root check — sysctl writes require root
|
||||
# validate_unraid — notify validated before use
|
||||
# Silent on success — runs every boot, no noise when already correct
|
||||
# Only warns on changes or failures
|
||||
#
|
||||
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||||
# INOTIFY_MAX_INSTANCES — default 1024
|
||||
# INOTIFY_MAX_WATCHES — default 1048576 (1M)
|
||||
# INOTIFY_MAX_QUEUED_EVENTS — default 32768
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# inotify_tuning.sh — normal run (apply settings)
|
||||
# inotify_tuning.sh --dry-run — show what would change
|
||||
# inotify_tuning.sh --status — show current vs target values and top consumers
|
||||
# inotify_tuning.sh --log — verbose output
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR inotify Tuning ━━━"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
error "Must be run as root — sysctl writes require root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Running as root"
|
||||
validate_unraid_cmd \
|
||||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||||
"" "" \
|
||||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ Current Values ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
CURRENT_INSTANCES=$(sysctl -n fs.inotify.max_user_instances 2>/dev/null || echo "?")
|
||||
CURRENT_WATCHES=$(sysctl -n fs.inotify.max_user_watches 2>/dev/null || echo "?")
|
||||
CURRENT_EVENTS=$(sysctl -n fs.inotify.max_queued_events 2>/dev/null || echo "?")
|
||||
acquire_lock
|
||||
|
||||
info "Current: instances=$CURRENT_INSTANCES watches=$CURRENT_WATCHES queued=$CURRENT_EVENTS"
|
||||
info "Target: instances=${INOTIFY_MAX_INSTANCES} watches=${INOTIFY_MAX_WATCHES} queued=${INOTIFY_MAX_QUEUED_EVENTS}"
|
||||
detect_hosts
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY INOTIFY STATUS ━━━━━"
|
||||
echo " max_user_instances: $CURRENT_INSTANCES (target: ${INOTIFY_MAX_INSTANCES})"
|
||||
echo " max_user_watches: $CURRENT_WATCHES (target: ${INOTIFY_MAX_WATCHES})"
|
||||
echo " max_queued_events: $CURRENT_EVENTS (target: ${INOTIFY_MAX_QUEUED_EVENTS})"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo ""
|
||||
echo " Active inotify instances in use:"
|
||||
find /proc/*/fd -lname "anon_inode:inotify" 2>/dev/null | \
|
||||
awk -F/ '{print $3}' | sort -u | while read -r pid; do
|
||||
cmd=$(cat /proc/$pid/comm 2>/dev/null || echo "?")
|
||||
echo " PID $pid ($cmd)"
|
||||
done | head -20
|
||||
|
||||
echo "━━━ Kernel Limits ━━━"
|
||||
CURRENT_INSTANCES=$(sysctl -n fs.inotify.max_user_instances 2>/dev/null || echo "?")
|
||||
CURRENT_WATCHES=$(sysctl -n fs.inotify.max_user_watches 2>/dev/null || echo "?")
|
||||
CURRENT_EVENTS=$(sysctl -n fs.inotify.max_queued_events 2>/dev/null || echo "?")
|
||||
|
||||
for label in "max_user_instances current=$CURRENT_INSTANCES target=$INOTIFY_MAX_INSTANCES" \
|
||||
"max_user_watches current=$CURRENT_WATCHES target=$INOTIFY_MAX_WATCHES" \
|
||||
"max_queued_events current=$CURRENT_EVENTS target=$INOTIFY_MAX_QUEUED_EVENTS"; do
|
||||
echo " $label"
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "━━━ Active Instances ━━━"
|
||||
USED_INSTANCES=$(find /proc/*/fd -lname 'anon_inode:inotify' 2>/dev/null | wc -l)
|
||||
USED_INSTANCES="${USED_INSTANCES//[^0-9]/}"
|
||||
echo " Instances in use: ${USED_INSTANCES:-0} / $CURRENT_INSTANCES"
|
||||
if [[ "$CURRENT_INSTANCES" -gt 0 ]]; then
|
||||
PCT=$(( ${USED_INSTANCES:-0} * 100 / CURRENT_INSTANCES ))
|
||||
echo " Utilisation: ${PCT}%"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "━━━ Top Consumers ━━━"
|
||||
find /proc/*/fd -lname 'anon_inode:inotify' 2>/dev/null | \
|
||||
awk -F/ '{print $3}' | sort | uniq -c | sort -rn | head -10 | \
|
||||
while read -r count pid; do
|
||||
cmd=$(cat /proc/"$pid"/comm 2>/dev/null || echo "?")
|
||||
cgroup=$(cat /proc/"$pid"/cgroup 2>/dev/null | \
|
||||
grep docker | grep -o '[a-f0-9]\{12\}' | head -1 || echo "")
|
||||
if [[ -n "$cgroup" ]]; then
|
||||
label="[docker:${cgroup}] $cmd"
|
||||
else
|
||||
label="[host] $cmd"
|
||||
fi
|
||||
echo " ${count} instances — $label (PID $pid)"
|
||||
done | head -10
|
||||
|
||||
echo ""
|
||||
echo "━━━ VSCode / Code-Server ━━━"
|
||||
echo " If VSCode shows 'unable to watch for file changes':"
|
||||
echo " 1. Verify max_user_watches target is set high enough"
|
||||
echo " 2. Check total watches used: cat /proc/sys/fs/inotify/max_user_watches"
|
||||
echo " 3. After any limit change: docker restart Code-Server"
|
||||
echo " (running containers inherit limits at start, not dynamically)"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Apply Settings ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
# ==============================================================================================
|
||||
CHANGED=0
|
||||
FAILED=0
|
||||
|
||||
apply_sysctl() {
|
||||
local key="$1"
|
||||
local value="$2"
|
||||
local key="$1" value="$2"
|
||||
local current
|
||||
current=$(sysctl -n "$key" 2>/dev/null || echo 0)
|
||||
|
||||
if [[ "$current" -eq "$value" ]]; then
|
||||
success "$key = $value (already set)"
|
||||
return
|
||||
log "$key = $value (already correct)"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would set $key = $value (currently $current)"
|
||||
return
|
||||
return 0
|
||||
fi
|
||||
|
||||
if sysctl -w "${key}=${value}" >/dev/null 2>&1; then
|
||||
success "$key = $value (was $current)"
|
||||
((CHANGED++))
|
||||
warn "Set $key = $value (was $current)"
|
||||
(( CHANGED++ ))
|
||||
else
|
||||
error "Failed to set $key = $value"
|
||||
((FAILED++))
|
||||
(( FAILED++ ))
|
||||
fi
|
||||
}
|
||||
|
||||
@@ -102,25 +179,35 @@ apply_sysctl "fs.inotify.max_user_instances" "$INOTIFY_MAX_INSTANCES"
|
||||
apply_sysctl "fs.inotify.max_user_watches" "$INOTIFY_MAX_WATCHES"
|
||||
apply_sysctl "fs.inotify.max_queued_events" "$INOTIFY_MAX_QUEUED_EVENTS"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Summary ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY INOTIFY TUNING SUMMARY ━━━━━"
|
||||
echo " max_user_instances: $(sysctl -n fs.inotify.max_user_instances 2>/dev/null)"
|
||||
echo " max_user_watches: $(sysctl -n fs.inotify.max_user_watches 2>/dev/null)"
|
||||
echo " max_queued_events: $(sysctl -n fs.inotify.max_queued_events 2>/dev/null)"
|
||||
echo ""
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "$ICON_WARN Status: DRY RUN — no changes made"
|
||||
elif [[ "$FAILED" -gt 0 ]]; then
|
||||
echo "$ICON_ERROR Status: $FAILED setting(s) failed"
|
||||
notify "inotify tuning failed on $(hostname) — $FAILED setting(s) could not be applied" "inotify Tuning" "warning"
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$FAILED" -gt 0 ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY INOTIFY TUNING SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_ERROR $FAILED setting(s) failed to apply"
|
||||
notify "inotify tuning failed on $(hostname) ($MY_ID) — $FAILED setting(s) could not be applied" \
|
||||
"inotify Tuning" "warning"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 1
|
||||
elif [[ "$CHANGED" -gt 0 ]]; then
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS $CHANGED setting(s) applied"
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY INOTIFY TUNING SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo " max_user_instances: $(sysctl -n fs.inotify.max_user_instances 2>/dev/null)"
|
||||
echo " max_user_watches: $(sysctl -n fs.inotify.max_user_watches 2>/dev/null)"
|
||||
echo " max_queued_events: $(sysctl -n fs.inotify.max_queued_events 2>/dev/null)"
|
||||
echo ""
|
||||
warn "$CHANGED setting(s) updated"
|
||||
if [[ "$CHANGED" -gt 0 ]]; then
|
||||
warn "If Code-Server is running: docker restart Code-Server"
|
||||
warn "Running containers inherit limits at start — restart picks up new values"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
else
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS All settings already correct"
|
||||
# Already correct — completely silent (runs every boot)
|
||||
log "inotify limits already correct — no changes needed"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
exit 0
|
||||
+129
-83
@@ -1,116 +1,162 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- Mover Stop Script ------------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Safely stops the unRAID mover process with a user warning before halting.
|
||||
# Timeout before stopping is configured in Master.conf under MOVER_STOP_TIMEOUT.
|
||||
# Supports --dry-run to preview what would happen without making changes.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ================================= Mover Stop =================================================
|
||||
# ==============================================================================================
|
||||
# Safely stops the unRAID mover process with a warning before halting.
|
||||
# Warns all logged-in users via wall message, waits the configured timeout, then stops.
|
||||
#
|
||||
# ── WHEN TO USE ───────────────────────────────────────────────────────────────────────────────
|
||||
# - Before a planned reboot when mover is running mid-cycle
|
||||
# - Before disk replacement or array operations that need mover stopped
|
||||
# - Before rsync — mover and rsync simultaneously moving the same files causes corruption
|
||||
# - Called automatically by maintenance scripts that need the mover stopped first
|
||||
#
|
||||
# ── STOP SEQUENCE ─────────────────────────────────────────────────────────────────────────────
|
||||
# 1. Check if mover is running — exit cleanly if not
|
||||
# 2. Broadcast wall warning to all logged-in users
|
||||
# 3. Wait MOVER_STOP_TIMEOUT seconds (default 30) — gives active sessions a chance to note it
|
||||
# 4. Send SIGTERM — mover can complete its current file operation before exiting
|
||||
# 5. Wait 5 seconds for graceful exit
|
||||
# 6. Verify stopped — if still running send SIGKILL (force)
|
||||
# 7. Final verify — error if still running after SIGKILL
|
||||
#
|
||||
# ── SIGTERM vs SIGKILL ────────────────────────────────────────────────────────────────────────
|
||||
# SIGTERM first — allows mover to finish the file it is currently moving (no partial files).
|
||||
# SIGKILL only as fallback — forces immediate stop (may leave partial files on cache or array).
|
||||
#
|
||||
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
|
||||
# acquire_lock — prevents concurrent stop attempts racing each other
|
||||
# Root check — pkill on emhttp processes requires root
|
||||
# validate_unraid — notify validated before use
|
||||
# SIGTERM → verify → SIGKILL sequence — graceful then forced
|
||||
# Final verify — confirms mover actually stopped
|
||||
# Silent on clean — mover not running = log() only, no output ✅
|
||||
#
|
||||
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||||
# MOVER_STOP_TIMEOUT — seconds to warn users before stopping (default 30)
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# mover_stop.sh — stop mover with configured timeout
|
||||
# mover_stop.sh --dry-run — show what would happen
|
||||
# mover_stop.sh --status — show mover state
|
||||
# mover_stop.sh --log — verbose output
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
|
||||
# ROOT CHECK
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
error "Must be run as root — pkill on emhttp processes requires root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Running as root"
|
||||
validate_unraid_cmd \
|
||||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||||
"" "" \
|
||||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||
|
||||
# Validate MOVER_STOP_TIMEOUT is a valid integer before using it
|
||||
validate_int MOVER_STOP_TIMEOUT "$MOVER_STOP_TIMEOUT"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Status ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
acquire_lock
|
||||
|
||||
detect_hosts
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_MOVER Timeout: ${MOVER_STOP_TIMEOUT}s"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_MOVER Timeout: ${MOVER_STOP_TIMEOUT}s"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo ""
|
||||
if pgrep -f "emhttp.*Mover" >/dev/null 2>&1; then
|
||||
MOVER_PID=$(pgrep -f "emhttp.*Mover" | head -1)
|
||||
MOVER_START=$(ps -o lstart= -p "$MOVER_PID" 2>/dev/null | xargs)
|
||||
echo " $ICON_MOVER Mover: RUNNING (PID $MOVER_PID)"
|
||||
[[ -n "$MOVER_START" ]] && echo " $ICON_TIME Started: $MOVER_START"
|
||||
else
|
||||
echo " $ICON_MOVER Mover: not running"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# FUNCTIONS
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
# Returns 0 if the unRAID mover process is currently running, 1 if not.
|
||||
check_mover_running() {
|
||||
pgrep -f "emhttp.*Mover" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
# Broadcasts a wall message to all logged in users warning mover is stopping.
|
||||
notify_users() {
|
||||
warn "Notifying users — mover stopping in ${MOVER_STOP_TIMEOUT}s"
|
||||
wall "$ICON_WARN unRAID Mover will stop in ${MOVER_STOP_TIMEOUT} second(s)."
|
||||
}
|
||||
|
||||
# Sends SIGTERM to the mover process via pkill.
|
||||
# Skips if dry run is active.
|
||||
stop_mover() {
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would stop unRAID Mover process"
|
||||
return
|
||||
fi
|
||||
|
||||
info "Stopping unRAID Mover..."
|
||||
|
||||
if pkill -f "emhttp.*Mover"; then
|
||||
success "Mover stopped"
|
||||
else
|
||||
warn "Could not stop mover — may have already stopped"
|
||||
fi
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_MOVER Mover Stop ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_MOVER Mover Stop ━━━"
|
||||
echo "$ICON_MOVER Timeout: ${MOVER_STOP_TIMEOUT}s"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo ""
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Mover Stop ━━━
|
||||
# ==============================================================================================
|
||||
START=$(date +%s)
|
||||
|
||||
if check_mover_running; then
|
||||
info "$ICON_MOVER Mover is running"
|
||||
notify_users
|
||||
info "Waiting ${MOVER_STOP_TIMEOUT}s before stopping..."
|
||||
if ! pgrep -f "emhttp.*Mover" >/dev/null 2>&1; then
|
||||
log "Mover is not running — nothing to do"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
MOVER_PID=$(pgrep -f "emhttp.*Mover" | head -1)
|
||||
warn "Mover is running (PID $MOVER_PID) — stopping in ${MOVER_STOP_TIMEOUT}s"
|
||||
|
||||
# ── Warn users via wall ───────────────────────────────────────────────────────────────────────
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
wall "$ICON_WARN $MY_ID ($LOCAL_SERVER_NAME) — unRAID Mover stopping in ${MOVER_STOP_TIMEOUT}s"
|
||||
log "Wall message sent — waiting ${MOVER_STOP_TIMEOUT}s..."
|
||||
sleep "$MOVER_STOP_TIMEOUT"
|
||||
stop_mover
|
||||
else
|
||||
info "$ICON_MOVER Mover is not running — nothing to do"
|
||||
warn "DRY RUN — would send wall warning and wait ${MOVER_STOP_TIMEOUT}s"
|
||||
fi
|
||||
|
||||
# ── SIGTERM — graceful stop ───────────────────────────────────────────────────────────────────
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would send SIGTERM to mover (PID $MOVER_PID)"
|
||||
else
|
||||
log "Sending SIGTERM to mover (PID $MOVER_PID)..."
|
||||
pkill -TERM -f "emhttp.*Mover" 2>/dev/null || true
|
||||
sleep 5
|
||||
|
||||
# Verify stopped after SIGTERM
|
||||
if ! pgrep -f "emhttp.*Mover" >/dev/null 2>&1; then
|
||||
warn "Mover stopped cleanly (SIGTERM) ✅"
|
||||
else
|
||||
# ── SIGKILL — forced stop ─────────────────────────────────────────────────────────────
|
||||
warn "Mover still running after SIGTERM — sending SIGKILL (may leave partial files)"
|
||||
pkill -KILL -f "emhttp.*Mover" 2>/dev/null || true
|
||||
sleep 2
|
||||
|
||||
# Final verify
|
||||
if pgrep -f "emhttp.*Mover" >/dev/null 2>&1; then
|
||||
error "Mover still running after SIGKILL — manual intervention needed"
|
||||
notify "Mover stop failed on $(hostname) ($MY_ID) — process unkillable" \
|
||||
"Mover Stop" "warning"
|
||||
exit 1
|
||||
else
|
||||
warn "Mover force-stopped (SIGKILL) — check for partial files on cache"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Summary ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY MOVER STOP SUMMARY ━━━━━"
|
||||
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
||||
if check_mover_running; then
|
||||
echo "$ICON_ERROR Status: $ICON_ERROR STILL RUNNING"
|
||||
notify "Mover stop failed — mover still running on $(hostname)" "Mover Stop" "warning"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_MOVER Timeout: ${MOVER_STOP_TIMEOUT}s"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no changes made"
|
||||
else
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS STOPPED / NOT RUNNING"
|
||||
notify "Mover stopped on $(hostname)" "Mover Stop" "normal"
|
||||
log "$ICON_DONE Status: done — mover stopped ✅"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
@@ -1,141 +1,206 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ----------------------------- PHP-FPM Max Children Script ------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Persistently sets PHP-FPM pm.max_children on unRAID.
|
||||
# Config file path and max children value are set in Master.conf.
|
||||
# Supports --dry-run to preview what would be changed without making changes.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ============================= PHP-FPM Max Children ===========================================
|
||||
# ==============================================================================================
|
||||
# Persistently sets PHP-FPM pm.max_children on unRAID to prevent WebGUI slowdowns.
|
||||
# Run once at array start via ARRAY_START_SCRIPTS in master.conf.
|
||||
# Idempotent — completely silent when value is already correct.
|
||||
#
|
||||
# ── WHY THIS EXISTS ───────────────────────────────────────────────────────────────────────────
|
||||
# unRAID's WebGUI runs through PHP-FPM. The default pm.max_children is very low (4-8).
|
||||
# Under load — multiple users, Docker operations, heavy dashboard usage — all PHP workers
|
||||
# saturate and new requests queue. The WebGUI becomes slow or unresponsive.
|
||||
#
|
||||
# pm.max_children controls how many PHP worker processes can run simultaneously.
|
||||
# Raising it allows the WebGUI to handle more concurrent requests without queuing.
|
||||
# Too high: wastes RAM. Too low: WebGUI slowdowns.
|
||||
# PHP_MAX_CHILDREN=250 is appropriate for 128GB — ~2MB per worker = ~500MB total.
|
||||
#
|
||||
# ── WHY IDEMPOTENT ────────────────────────────────────────────────────────────────────────────
|
||||
# This runs at every array start. If the value is already correct there is nothing to do —
|
||||
# no config write, no PHP-FPM restart. Restarting PHP-FPM unnecessarily disrupts active
|
||||
# WebGUI sessions and is annoying on every boot.
|
||||
#
|
||||
# ── APPLY SEQUENCE ────────────────────────────────────────────────────────────────────────────
|
||||
# 1. Read current pm.max_children from PHP_CONF
|
||||
# 2. If already at target → exit silently (idempotent)
|
||||
# 3. Verify sed pattern matches before writing
|
||||
# 4. Apply sed replacement
|
||||
# 5. Restart PHP-FPM via rc.php-fpm
|
||||
# 6. Verify PHP-FPM process running after restart
|
||||
# 7. Verify config file reflects target value
|
||||
#
|
||||
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
|
||||
# Root check — writing to system config requires root
|
||||
# acquire_lock — prevents concurrent runs at array start
|
||||
# Idempotent check — only restarts PHP-FPM when value actually changes
|
||||
# Pattern match check — verifies sed found pm.max_children before writing
|
||||
# Process verify — confirms PHP-FPM running after restart
|
||||
# Config verify — reads back config to confirm value applied
|
||||
# validate_unraid — notify validated before use
|
||||
# Silent on correct — runs every boot, no noise when already set ✅
|
||||
#
|
||||
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||||
# PHP_MAX_CHILDREN — target pm.max_children value (default 250)
|
||||
# PHP_CONF — path to PHP-FPM www.conf (default /etc/php83/php-fpm.d/www.conf)
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# php_fpm_max_children.sh — normal run (idempotent)
|
||||
# php_fpm_max_children.sh --dry-run — show what would change
|
||||
# php_fpm_max_children.sh --status — show current vs target and process state
|
||||
# php_fpm_max_children.sh --log — verbose output
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
|
||||
# ROOT CHECK
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
error "Must be run as root — writing system config requires root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Running as root"
|
||||
validate_unraid_cmd \
|
||||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||||
"" "" \
|
||||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||
|
||||
# VALIDATION
|
||||
validate_int PHP_MAX_CHILDREN "$PHP_MAX_CHILDREN"
|
||||
require_var PHP_CONF
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Status ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
acquire_lock
|
||||
|
||||
detect_hosts
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_GEAR Config File: $PHP_CONF"
|
||||
echo "$ICON_PHP Max Children: $PHP_MAX_CHILDREN"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "━━━━━ $ICON_SUMMARY PHP-FPM STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_GEAR Config file: $PHP_CONF"
|
||||
echo "$ICON_PHP Target: pm.max_children = $PHP_MAX_CHILDREN"
|
||||
echo ""
|
||||
|
||||
if [[ -f "$PHP_CONF" ]]; then
|
||||
CURRENT_VAL=$(grep -E "^pm\.max_children" "$PHP_CONF" 2>/dev/null | \
|
||||
awk '{print $NF}')
|
||||
if [[ "${CURRENT_VAL:-0}" -eq "$PHP_MAX_CHILDREN" ]]; then
|
||||
echo " $ICON_SUCCESS Current: pm.max_children = $CURRENT_VAL (correct ✅)"
|
||||
else
|
||||
echo " $ICON_WARN Current: pm.max_children = ${CURRENT_VAL:-not set} (would update)"
|
||||
fi
|
||||
else
|
||||
echo " $ICON_ERROR Config file not found: $PHP_CONF"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
if pgrep -f "php-fpm" >/dev/null 2>&1; then
|
||||
FPM_COUNT=$(pgrep -fc "php-fpm" 2>/dev/null || echo "?")
|
||||
echo " $ICON_SUCCESS PHP-FPM: running ($FPM_COUNT worker(s))"
|
||||
else
|
||||
echo " $ICON_ERROR PHP-FPM: NOT running"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# FUNCTIONS
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
# Applies pm.max_children to the PHP-FPM config file and restarts the service.
|
||||
# Verifies the value was applied correctly after restart.
|
||||
# Skips all changes if dry run is active.
|
||||
apply_php_max_children() {
|
||||
local target="pm.max_children = $PHP_MAX_CHILDREN"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would set pm.max_children = $PHP_MAX_CHILDREN in $PHP_CONF"
|
||||
warn "DRY RUN — would restart PHP-FPM service"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Verify config file exists before attempting changes
|
||||
if [[ ! -f "$PHP_CONF" ]]; then
|
||||
error "PHP config file not found: $PHP_CONF"
|
||||
return 1
|
||||
fi
|
||||
|
||||
info "Applying pm.max_children = $PHP_MAX_CHILDREN..."
|
||||
|
||||
if ! sed -i "s/^pm\.max_children.*/$target/" "$PHP_CONF"; then
|
||||
error "Failed to update PHP config: $PHP_CONF"
|
||||
return 1
|
||||
fi
|
||||
|
||||
success "Config updated"
|
||||
|
||||
info "Restarting PHP-FPM..."
|
||||
|
||||
if ! /etc/rc.d/rc.php-fpm restart; then
|
||||
error "PHP-FPM restart failed"
|
||||
return 1
|
||||
fi
|
||||
|
||||
success "PHP-FPM restarted"
|
||||
|
||||
# Verify the value was applied correctly
|
||||
local current
|
||||
current=$(grep -E "^pm\.max_children" "$PHP_CONF" || true)
|
||||
|
||||
if [[ -n "$current" ]]; then
|
||||
success "Verified: $current"
|
||||
logger "Userscript: PHP-FPM updated → $current"
|
||||
else
|
||||
warn "Could not verify configuration value — check $PHP_CONF manually"
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_PHP PHP-FPM Config ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_PHP PHP-FPM Config ━━━"
|
||||
echo "$ICON_GEAR Config File: $PHP_CONF"
|
||||
echo "$ICON_PHP Max Children: $PHP_MAX_CHILDREN"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo ""
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ PHP-FPM Config ━━━
|
||||
# ==============================================================================================
|
||||
START=$(date +%s)
|
||||
|
||||
PHP_SUCCESS=false
|
||||
apply_php_max_children && PHP_SUCCESS=true
|
||||
if [[ ! -f "$PHP_CONF" ]]; then
|
||||
error "PHP config file not found: $PHP_CONF"
|
||||
notify "PHP-FPM config not found on $(hostname) ($MY_ID) — $PHP_CONF missing" \
|
||||
"PHP-FPM" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ── Idempotent check ─────────────────────────────────────────────────────────────────────────
|
||||
CURRENT_VAL=$(grep -E "^pm\.max_children" "$PHP_CONF" 2>/dev/null | awk '{print $NF}')
|
||||
if [[ "${CURRENT_VAL:-0}" -eq "$PHP_MAX_CHILDREN" ]]; then
|
||||
log "pm.max_children already $PHP_MAX_CHILDREN — no changes needed"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
warn "pm.max_children: ${CURRENT_VAL:-not set} → $PHP_MAX_CHILDREN"
|
||||
|
||||
# ── Verify pattern exists before writing ─────────────────────────────────────────────────────
|
||||
if ! grep -qE "^pm\.max_children" "$PHP_CONF" 2>/dev/null; then
|
||||
error "pm.max_children not found in $PHP_CONF — cannot apply"
|
||||
error "Add 'pm.max_children = $PHP_MAX_CHILDREN' to $PHP_CONF manually"
|
||||
notify "PHP-FPM pm.max_children not found in config on $(hostname) ($MY_ID)" \
|
||||
"PHP-FPM" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would set pm.max_children = $PHP_MAX_CHILDREN in $PHP_CONF"
|
||||
warn "DRY RUN — would restart PHP-FPM"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ── Apply setting ─────────────────────────────────────────────────────────────────────────────
|
||||
log "Applying pm.max_children = $PHP_MAX_CHILDREN..."
|
||||
if ! sed -i "s/^pm\.max_children.*/pm.max_children = $PHP_MAX_CHILDREN/" "$PHP_CONF"; then
|
||||
error "Failed to update $PHP_CONF"
|
||||
notify "PHP-FPM config update failed on $(hostname) ($MY_ID)" \
|
||||
"PHP-FPM" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log "Config updated"
|
||||
|
||||
# ── Restart PHP-FPM ──────────────────────────────────────────────────────────────────────────
|
||||
log "Restarting PHP-FPM..."
|
||||
if ! /etc/rc.d/rc.php-fpm restart >/dev/null 2>&1; then
|
||||
error "PHP-FPM restart command failed"
|
||||
notify "PHP-FPM restart failed on $(hostname) ($MY_ID)" \
|
||||
"PHP-FPM" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
sleep 3 # Allow PHP-FPM workers to initialise
|
||||
|
||||
# ── Verify process running ────────────────────────────────────────────────────────────────────
|
||||
if ! pgrep -f "php-fpm" >/dev/null 2>&1; then
|
||||
error "PHP-FPM not running after restart — WebGUI may be broken"
|
||||
notify "PHP-FPM failed to start after config update on $(hostname) ($MY_ID)" \
|
||||
"PHP-FPM" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ── Verify config reflects target ────────────────────────────────────────────────────────────
|
||||
APPLIED_VAL=$(grep -E "^pm\.max_children" "$PHP_CONF" 2>/dev/null | awk '{print $NF}')
|
||||
if [[ "${APPLIED_VAL:-0}" -ne "$PHP_MAX_CHILDREN" ]]; then
|
||||
warn "Config reads pm.max_children = ${APPLIED_VAL:-unknown} — expected $PHP_MAX_CHILDREN"
|
||||
warn "Check $PHP_CONF manually"
|
||||
else
|
||||
log "Verified: pm.max_children = $APPLIED_VAL ✅"
|
||||
fi
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Summary ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY PHP-FPM SUMMARY ━━━━━"
|
||||
echo "$ICON_GEAR Config File: $PHP_CONF"
|
||||
echo "$ICON_PHP Max Children: $PHP_MAX_CHILDREN"
|
||||
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "$ICON_WARN Status: DRY RUN — no changes made"
|
||||
elif [[ "$PHP_SUCCESS" == true ]]; then
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS DONE"
|
||||
notify "PHP-FPM max_children set to $PHP_MAX_CHILDREN on $(hostname)" "PHP-FPM" "normal"
|
||||
else
|
||||
echo "$ICON_ERROR Status: $ICON_ERROR FAILED"
|
||||
notify "PHP-FPM config update failed on $(hostname)" "PHP-FPM" "warning"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_GEAR Config file: $PHP_CONF"
|
||||
echo "$ICON_PHP Applied: pm.max_children = $PHP_MAX_CHILDREN"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
log "$ICON_DONE Status: done ✅"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ "$PHP_SUCCESS" == false ]] && exit 1
|
||||
exit 0
|
||||
+260
-199
@@ -1,135 +1,184 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- Rsync Stop Script ------------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Stops rsync intelligently — auto-detects what's running and acts accordingly.
|
||||
# ==============================================================================================
|
||||
# ================================= Rsync Stop =================================================
|
||||
# ==============================================================================================
|
||||
# Stops rsync intelligently on both local and remote servers.
|
||||
# Auto-detects orchestrators and chooses the safest stop strategy automatically.
|
||||
#
|
||||
# Default behavior (just run it):
|
||||
# Detects if an orchestrator (daily/weekly) is running
|
||||
# If yes → kills rsync subprocess only
|
||||
# orchestrator sees rsync died → moves to next share or exits cleanly
|
||||
# If no → kills rsync processes directly (solo rsync.sh run)
|
||||
# Cleans stale lock files
|
||||
# Recovers any containers left stopped by interrupted rsync
|
||||
# ── TWO MODES ─────────────────────────────────────────────────────────────────────────────────
|
||||
# Default (smart):
|
||||
# Detects if an orchestrator (daily/weekly/critical sync) is running
|
||||
# If orchestrator found → kills rsync subprocess only
|
||||
# Orchestrator sees rsync died → moves to next share or exits cleanly
|
||||
# If no orchestrator → kills rsync directly (standalone rsync.sh run)
|
||||
# Cleans stale lock files after kill
|
||||
# Recovers containers left stopped by interrupted rsync (local only)
|
||||
#
|
||||
# --full-stop flag (nuclear):
|
||||
# Kills orchestrator first → then rsync
|
||||
# Use when: you want everything dead immediately
|
||||
# daily/weekly loop will NOT continue to next share
|
||||
# --full-stop (nuclear):
|
||||
# Kills orchestrator first → then kills rsync
|
||||
# Orchestrator will NOT continue to next share
|
||||
# Use when: you need everything dead immediately
|
||||
#
|
||||
# Both local and remote are handled in one run.
|
||||
# Remote containers left as-is — docker_watchdog.sh handles remote recovery.
|
||||
# ── REMOTE HANDLING ───────────────────────────────────────────────────────────────────────────
|
||||
# Both local and remote handled in one run via SSH.
|
||||
# Remote containers left as-is — docker_watchdog.sh handles remote container recovery.
|
||||
# If remote unreachable → skips remote cleanly, logs warning.
|
||||
#
|
||||
# Flags:
|
||||
# (none) ← smart mode — auto-detects, rsync-only if orchestrator running
|
||||
# --full-stop ← nuclear — kill orchestrator + rsync
|
||||
# --dry-run ← preview without changes
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ── ORCHESTRATOR DETECTION ────────────────────────────────────────────────────────────────────
|
||||
# detect_rsync_parent() scans all lock files to find which running process
|
||||
# has rsync as a descendant. No hardcoded list — works for any orchestrator.
|
||||
# Returns: "script_name:parent_pid" if found, empty if rsync running standalone.
|
||||
#
|
||||
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
|
||||
# Root check — pkill and docker require root
|
||||
# acquire_lock — prevents concurrent stop attempts racing
|
||||
# DOCKER_TIMEOUT — all docker calls protected against hung daemon
|
||||
# SSH_TIMEOUT — all remote SSH calls timeout-protected
|
||||
# SIGTERM → SIGKILL — graceful then forced for orchestrators
|
||||
# Container recovery — restarts local containers left stopped by killed rsync
|
||||
# validate_unraid_cmd — notify validated before use
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# rsync_stop.sh — smart stop (auto-detect)
|
||||
# rsync_stop.sh --full-stop — kill orchestrator + rsync
|
||||
# rsync_stop.sh --rsync-only — skip container recovery (called by other scripts)
|
||||
# rsync_stop.sh --dry-run — preview without changes
|
||||
# rsync_stop.sh --status — show what's currently running
|
||||
# rsync_stop.sh --full-stop --dry-run — preview full stop
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
# Check for --full-stop before parse_args
|
||||
DOCKER_TIMEOUT=15
|
||||
SSH_TIMEOUT=15
|
||||
|
||||
# ── Parse special flags before parse_args ─────────────────────────────────────────────────────
|
||||
FULL_STOP=false
|
||||
RSYNC_ONLY_MODE=false
|
||||
FILTERED_ARGS=()
|
||||
for arg in "$@"; do
|
||||
if [[ "$arg" == "--full-stop" ]]; then
|
||||
FULL_STOP=true
|
||||
else
|
||||
FILTERED_ARGS+=("$arg")
|
||||
fi
|
||||
case "$arg" in
|
||||
--full-stop) FULL_STOP=true ;;
|
||||
--rsync-only) RSYNC_ONLY_MODE=true ;;
|
||||
*) FILTERED_ARGS+=("$arg") ;;
|
||||
esac
|
||||
done
|
||||
|
||||
parse_args "${FILTERED_ARGS[@]}"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root — pkill and docker require root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
validate_unraid_cmd \
|
||||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||||
"" "" \
|
||||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||
|
||||
acquire_lock
|
||||
|
||||
detect_hosts
|
||||
resolve_remote_ip
|
||||
|
||||
REMOTE_REACHABLE=true
|
||||
if ! ping -c1 -W3 "$REMOTE_SERVER" &>/dev/null; then
|
||||
warn "$ICON_PING Remote $REMOTE_SERVER_NAME unreachable — will skip remote"
|
||||
REMOTE_REACHABLE=false
|
||||
# Remote reachability
|
||||
REMOTE_REACHABLE=false
|
||||
if timeout "$SSH_TIMEOUT" ping -c1 -W3 "$REMOTE_SERVER" &>/dev/null; then
|
||||
REMOTE_REACHABLE=true
|
||||
log "$REMOTE_SERVER_NAME reachable ✅"
|
||||
else
|
||||
info "$ICON_PING $REMOTE_SERVER_NAME reachable"
|
||||
warn "$REMOTE_SERVER_NAME unreachable — remote operations will be skipped"
|
||||
fi
|
||||
|
||||
[[ "$SHOW_STATUS" == true ]] && show_status && exit 0
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
[[ "$FULL_STOP" == true ]] && warn "FULL STOP mode — orchestrator + rsync will be killed"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY RSYNC STOP STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_NET Remote: $REMOTE_ID ($REMOTE_SERVER_NAME)"
|
||||
echo ""
|
||||
|
||||
LOCAL_PIDS=$(pgrep -x rsync 2>/dev/null | tr '\n' ' ')
|
||||
echo " $ICON_SYNC Local rsync PIDs: ${LOCAL_PIDS:-none}"
|
||||
|
||||
for lockfile in "$LOCK_DIR"/*.lock; do
|
||||
[[ -f "$lockfile" ]] || continue
|
||||
content=$(cat "$lockfile" 2>/dev/null)
|
||||
pid="${content%%:*}"
|
||||
name="${content##*:}"
|
||||
[[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null && \
|
||||
echo " $ICON_RUNNING Lock: $name (PID $pid)"
|
||||
done
|
||||
|
||||
if [[ "$REMOTE_REACHABLE" == true ]]; then
|
||||
REMOTE_PIDS=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" \
|
||||
root@"$REMOTE_SERVER" "pgrep -x rsync || true" 2>/dev/null | tr '\n' ' ')
|
||||
echo " $ICON_SYNC Remote rsync PIDs: ${REMOTE_PIDS:-none}"
|
||||
else
|
||||
echo " $ICON_WARN Remote: unreachable"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ── ORCHESTRATOR DETECTION ────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Scans lock files to find which running process has rsync as a descendant.
|
||||
# No hardcoded script names — detects any orchestrator automatically.
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ Auto-detect orchestrators ━━━
|
||||
# Check if daily or weekly is running on local and remote
|
||||
# This determines default behavior
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# detect_rsync_parent — scans all lock files, finds which running process has rsync as a child
|
||||
# No hardcoded list — works for any orchestrator automatically
|
||||
#
|
||||
# Returns: "script_name:parent_pid" if found, empty if rsync running standalone
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
detect_rsync_parent() {
|
||||
local found=""
|
||||
|
||||
# Get all rsync PIDs running locally
|
||||
local rsync_pids
|
||||
rsync_pids=$(pgrep -x rsync 2>/dev/null || true)
|
||||
[[ -z "$rsync_pids" ]] && echo "" && return
|
||||
|
||||
# Scan all lock files in LOCK_DIR
|
||||
for lockfile in "$LOCK_DIR"/*.lock; do
|
||||
[[ -f "$lockfile" ]] || continue
|
||||
|
||||
local content pid locked_name
|
||||
content=$(cat "$lockfile" 2>/dev/null)
|
||||
pid="${content%%:*}"
|
||||
locked_name="${content##*:}"
|
||||
|
||||
# Skip if PID dead or is itself a rsync lock
|
||||
[[ -z "$pid" ]] && continue
|
||||
! kill -0 "$pid" 2>/dev/null && continue
|
||||
[[ "$locked_name" == rsync_* ]] && continue
|
||||
|
||||
# Check if any rsync PID is a child of this lock's PID
|
||||
local children
|
||||
children=$(cat /proc/"$pid"/task/"$pid"/children 2>/dev/null || \
|
||||
tr ' ' '\n' < /proc/"$pid"/children 2>/dev/null || true)
|
||||
|
||||
# Walk the child tree — rsync may be a grandchild (bash → rsync.sh → rsync)
|
||||
local all_descendants
|
||||
all_descendants=$(pgrep -P "$pid" 2>/dev/null || true)
|
||||
|
||||
# Check if any rsync PID is in the descendants
|
||||
while IFS= read -r rsync_pid; do
|
||||
[[ -z "$rsync_pid" ]] && continue
|
||||
local ppid
|
||||
ppid=$(awk '/^PPid:/{print $2}' /proc/"$rsync_pid"/status 2>/dev/null || echo "")
|
||||
if echo "$all_descendants" | grep -qw "$rsync_pid" 2>/dev/null || \
|
||||
[[ "$(cat /proc/"$rsync_pid"/status 2>/dev/null | awk '/^PPid:/{print $2}')" == "$pid" ]]; then
|
||||
found="$locked_name:$pid"
|
||||
break 2
|
||||
[[ "$ppid" == "$pid" ]]; then
|
||||
echo "${locked_name}:${pid}"
|
||||
return
|
||||
fi
|
||||
done <<< "$rsync_pids"
|
||||
done
|
||||
|
||||
echo "$found"
|
||||
echo ""
|
||||
}
|
||||
|
||||
detect_rsync_parent_remote() {
|
||||
[[ "$REMOTE_REACHABLE" != true ]] && echo "" && return
|
||||
|
||||
# Run the same logic on remote via SSH
|
||||
local found
|
||||
found=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" bash << 'REMOTE_SCRIPT'
|
||||
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" \
|
||||
root@"$REMOTE_SERVER" bash << 'REMOTE_SCRIPT' 2>/dev/null
|
||||
LOCK_DIR="/tmp/unraid_locks"
|
||||
rsync_pids=$(pgrep -x rsync 2>/dev/null || true)
|
||||
[[ -z "$rsync_pids" ]] && exit 0
|
||||
|
||||
for lockfile in "$LOCK_DIR"/*.lock; do
|
||||
[[ -f "$lockfile" ]] || continue
|
||||
content=$(cat "$lockfile" 2>/dev/null)
|
||||
@@ -138,21 +187,18 @@ for lockfile in "$LOCK_DIR"/*.lock; do
|
||||
[[ -z "$pid" ]] && continue
|
||||
! kill -0 "$pid" 2>/dev/null && continue
|
||||
[[ "$locked_name" == rsync_* ]] && continue
|
||||
|
||||
all_descendants=$(pgrep -P "$pid" 2>/dev/null || true)
|
||||
while IFS= read -r rsync_pid; do
|
||||
[[ -z "$rsync_pid" ]] && continue
|
||||
ppid=$(awk '/^PPid:/{print $2}' /proc/"$rsync_pid"/status 2>/dev/null)
|
||||
ppid=$(awk '/^PPid:/{print $2}' /proc/"$rsync_pid"/status 2>/dev/null || echo "")
|
||||
if echo "$all_descendants" | grep -qw "$rsync_pid" 2>/dev/null || \
|
||||
[[ "$ppid" == "$pid" ]]; then
|
||||
echo "$locked_name:$pid"
|
||||
echo "${locked_name}:${pid}"
|
||||
exit 0
|
||||
fi
|
||||
done <<< "$rsync_pids"
|
||||
done
|
||||
REMOTE_SCRIPT
|
||||
2>/dev/null)
|
||||
echo "$found"
|
||||
}
|
||||
|
||||
LOCAL_ORCH=$(detect_rsync_parent)
|
||||
@@ -162,27 +208,26 @@ REMOTE_ORCH=""
|
||||
# Determine mode
|
||||
if [[ "$FULL_STOP" == true ]]; then
|
||||
MODE="full-stop"
|
||||
info "Mode: FULL STOP — orchestrator + rsync will be killed"
|
||||
elif [[ -n "$LOCAL_ORCH" ]] || [[ -n "$REMOTE_ORCH" ]]; then
|
||||
MODE="rsync-only"
|
||||
[[ -n "$LOCAL_ORCH" ]] && info "Detected local orchestrator: ${LOCAL_ORCH%%:*} — rsync-only mode"
|
||||
[[ -n "$REMOTE_ORCH" ]] && info "Detected remote orchestrator: ${REMOTE_ORCH%%:*} — rsync-only mode"
|
||||
info "Orchestrator will continue after rsync is killed"
|
||||
info "Use --full-stop to also kill the orchestrator"
|
||||
[[ -n "$LOCAL_ORCH" ]] && \
|
||||
warn "Local orchestrator detected: ${LOCAL_ORCH%%:*} — rsync-only mode"
|
||||
[[ -n "$REMOTE_ORCH" ]] && \
|
||||
warn "Remote orchestrator detected: ${REMOTE_ORCH%%:*} — rsync-only mode"
|
||||
warn "Use --full-stop to also kill the orchestrator"
|
||||
else
|
||||
MODE="rsync-only"
|
||||
info "No orchestrator detected — killing rsync directly"
|
||||
log "No orchestrator detected — killing rsync directly"
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_STOP Kill Orchestrators (full-stop only) ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ── Kill Orchestrators (full-stop only) ───────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
ORCHESTRATORS_KILLED=()
|
||||
REMOTE_ORCHESTRATORS_KILLED=()
|
||||
|
||||
kill_orchestrator() {
|
||||
local script_name="$1"
|
||||
local pid="$2"
|
||||
local script_name="$1" pid="$2"
|
||||
local lockfile="$LOCK_DIR/${script_name}.lock"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
@@ -196,7 +241,7 @@ kill_orchestrator() {
|
||||
sleep 1
|
||||
|
||||
if ! kill -0 "$pid" 2>/dev/null; then
|
||||
success "$script_name stopped ✅"
|
||||
warn "$script_name stopped (PID $pid) ✅"
|
||||
rm -f "$lockfile"
|
||||
return 0
|
||||
else
|
||||
@@ -207,70 +252,62 @@ kill_orchestrator() {
|
||||
|
||||
if [[ "$MODE" == "full-stop" ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_STOP Orchestrators ━━━"
|
||||
echo "━━━ $ICON_STOP Kill Orchestrators ━━━"
|
||||
|
||||
# Local
|
||||
if [[ -n "$LOCAL_ORCH" ]]; then
|
||||
name="${LOCAL_ORCH%%:*}"
|
||||
pid="${LOCAL_ORCH##*:}"
|
||||
info "Killing local: $name (PID $pid)"
|
||||
if kill_orchestrator "$name" "$pid"; then
|
||||
ORCHESTRATORS_KILLED+=("$name")
|
||||
fi
|
||||
local_name="${LOCAL_ORCH%%:*}"
|
||||
local_pid="${LOCAL_ORCH##*:}"
|
||||
warn "Killing local: $local_name (PID $local_pid)"
|
||||
kill_orchestrator "$local_name" "$local_pid" && \
|
||||
ORCHESTRATORS_KILLED+=("$local_name")
|
||||
else
|
||||
info "No local orchestrator running"
|
||||
log "No local orchestrator running"
|
||||
fi
|
||||
|
||||
# Remote
|
||||
if [[ "$REMOTE_REACHABLE" == true ]] && [[ -n "$REMOTE_ORCH" ]]; then
|
||||
name="${REMOTE_ORCH%%:*}"
|
||||
pid="${REMOTE_ORCH##*:}"
|
||||
lockfile="$LOCK_DIR/${name}.lock"
|
||||
info "Killing remote: $name (PID $pid)"
|
||||
remote_name="${REMOTE_ORCH%%:*}"
|
||||
remote_pid="${REMOTE_ORCH##*:}"
|
||||
remote_lock="$LOCK_DIR/${remote_name}.lock"
|
||||
warn "Killing remote: $remote_name (PID $remote_pid)"
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||||
"kill -TERM '$pid' 2>/dev/null; sleep 2; \
|
||||
kill -0 '$pid' 2>/dev/null && kill -KILL '$pid' 2>/dev/null; \
|
||||
rm -f '$lockfile'" 2>/dev/null
|
||||
success "Remote $name stopped ✅"
|
||||
REMOTE_ORCHESTRATORS_KILLED+=("$name")
|
||||
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" \
|
||||
root@"$REMOTE_SERVER" \
|
||||
"kill -TERM '$remote_pid' 2>/dev/null; sleep 2; \
|
||||
kill -0 '$remote_pid' 2>/dev/null && kill -KILL '$remote_pid' 2>/dev/null; \
|
||||
rm -f '$remote_lock'" 2>/dev/null
|
||||
warn "Remote $remote_name stopped ✅"
|
||||
REMOTE_ORCHESTRATORS_KILLED+=("$remote_name")
|
||||
else
|
||||
warn "DRY RUN — would kill remote $name (PID $pid)"
|
||||
warn "DRY RUN — would kill remote $remote_name (PID $remote_pid)"
|
||||
fi
|
||||
elif [[ "$REMOTE_REACHABLE" == true ]]; then
|
||||
info "No remote orchestrator running"
|
||||
log "No remote orchestrator running"
|
||||
fi
|
||||
|
||||
# Wait for subprocesses to settle
|
||||
if [[ ${#ORCHESTRATORS_KILLED[@]} -gt 0 ]] || \
|
||||
[[ ${#REMOTE_ORCHESTRATORS_KILLED[@]} -gt 0 ]]; then
|
||||
info "Waiting 3s for subprocesses to settle..."
|
||||
sleep 3
|
||||
fi
|
||||
[[ ${#ORCHESTRATORS_KILLED[@]} -gt 0 || \
|
||||
${#REMOTE_ORCHESTRATORS_KILLED[@]} -gt 0 ]] && sleep 3
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_STOP Local Rsync ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Local Rsync ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_STOP Local Rsync ━━━"
|
||||
|
||||
LOCAL_KILLED=false
|
||||
LOCAL_PIDS=$(pgrep -x rsync || true)
|
||||
LOCAL_PIDS=$(pgrep -x rsync 2>/dev/null || true)
|
||||
|
||||
if [[ -z "$LOCAL_PIDS" ]]; then
|
||||
info "No rsync processes running locally"
|
||||
log "No rsync processes running locally"
|
||||
else
|
||||
info "Found PIDs: $(echo "$LOCAL_PIDS" | tr '\n' ' ')"
|
||||
warn "Found local rsync PIDs: $(echo "$LOCAL_PIDS" | tr '\n' ' ')"
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would kill local rsync"
|
||||
else
|
||||
if pkill -x rsync; then
|
||||
success "Local rsync killed ✅"
|
||||
LOCAL_KILLED=true
|
||||
else
|
||||
warn "pkill non-zero — may have already exited"
|
||||
fi
|
||||
pkill -x rsync 2>/dev/null && LOCAL_KILLED=true || \
|
||||
warn "pkill returned non-zero — rsync may have already exited"
|
||||
[[ "$LOCAL_KILLED" == true ]] && warn "Local rsync killed ✅"
|
||||
fi
|
||||
fi
|
||||
|
||||
@@ -280,61 +317,65 @@ for lockfile in "$LOCK_DIR"/rsync_*.lock; do
|
||||
content=$(cat "$lockfile" 2>/dev/null)
|
||||
pid="${content%%:*}"
|
||||
if [[ -n "$pid" ]] && ! kill -0 "$pid" 2>/dev/null; then
|
||||
info "Cleaning stale lock: $(basename "$lockfile")"
|
||||
log "Cleaning stale lock: $(basename "$lockfile")"
|
||||
[[ "$DRY_RUN" == false ]] && rm -f "$lockfile"
|
||||
fi
|
||||
done
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_STOP Remote Rsync ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Remote Rsync ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_STOP Remote Rsync ($REMOTE_SERVER_NAME) ━━━"
|
||||
echo "━━━ $ICON_STOP Remote Rsync — $REMOTE_SERVER_NAME ━━━"
|
||||
|
||||
REMOTE_KILLED=false
|
||||
|
||||
if [[ "$REMOTE_REACHABLE" == false ]]; then
|
||||
warn "Skipping — $REMOTE_SERVER_NAME unreachable"
|
||||
else
|
||||
REMOTE_PIDS=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||||
"pgrep -x rsync || true" 2>/dev/null || true)
|
||||
REMOTE_PIDS=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" \
|
||||
root@"$REMOTE_SERVER" "pgrep -x rsync || true" 2>/dev/null || true)
|
||||
|
||||
if [[ -z "$REMOTE_PIDS" ]]; then
|
||||
info "No rsync running on $REMOTE_SERVER_NAME"
|
||||
log "No rsync running on $REMOTE_SERVER_NAME"
|
||||
else
|
||||
info "Found PIDs on $REMOTE_SERVER_NAME: $(echo "$REMOTE_PIDS" | tr '\n' ' ')"
|
||||
warn "Found remote rsync PIDs: $(echo "$REMOTE_PIDS" | tr '\n' ' ')"
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would kill remote rsync"
|
||||
else
|
||||
if ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" "pkill -x rsync || true" 2>/dev/null; then
|
||||
success "Remote rsync killed ✅"
|
||||
REMOTE_KILLED=true
|
||||
else
|
||||
warn "Remote pkill non-zero — may have already exited"
|
||||
fi
|
||||
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" \
|
||||
root@"$REMOTE_SERVER" "pkill -x rsync || true" 2>/dev/null && \
|
||||
REMOTE_KILLED=true || \
|
||||
warn "Remote pkill returned non-zero — rsync may have already exited"
|
||||
[[ "$REMOTE_KILLED" == true ]] && warn "Remote rsync killed ✅"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_START $ICON_CONTAINERS Container Recovery ━━━
|
||||
# Restart containers left stopped by interrupted rsync
|
||||
# Only runs if something was actually killed locally
|
||||
# Remote containers left as-is — docker_watchdog.sh handles remote
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Container Recovery ━━━
|
||||
# ==============================================================================================
|
||||
# Restart local containers left stopped by interrupted rsync.
|
||||
# Remote containers left for docker_watchdog.sh to recover.
|
||||
# Skipped with --rsync-only flag (called by other scripts that handle recovery themselves).
|
||||
CONTAINERS_RESTARTED=()
|
||||
CONTAINERS_FAILED=()
|
||||
|
||||
if [[ "$RSYNC_ONLY_MODE" == false ]] && \
|
||||
{ [[ "$LOCAL_KILLED" == true ]] || [[ ${#ORCHESTRATORS_KILLED[@]} -gt 0 ]]; }; then
|
||||
|
||||
if [[ "$LOCAL_KILLED" == true ]] || [[ ${#ORCHESTRATORS_KILLED[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_START $ICON_CONTAINERS Container Recovery ━━━"
|
||||
info "Checking all profile containers..."
|
||||
echo "━━━ $ICON_START Container Recovery ━━━"
|
||||
log "Checking profile containers for recovery..."
|
||||
|
||||
declare -A SEEN
|
||||
ALL_CONTAINERS=()
|
||||
|
||||
for profile_containers in "${PROFILE_CRITICAL_CONTAINER_NAMES[@]}"; do
|
||||
for profile_containers in "${PROFILE_CRITICAL_CONTAINER_NAMES[@]:-}"; do
|
||||
read -r -a container_list <<< "$profile_containers"
|
||||
for c in "${container_list[@]}"; do
|
||||
for c in "${container_list[@]:-}"; do
|
||||
[[ -z "$c" ]] && continue
|
||||
if [[ -z "${SEEN[$c]:-}" ]]; then
|
||||
SEEN[$c]=1
|
||||
@@ -344,69 +385,89 @@ if [[ "$LOCAL_KILLED" == true ]] || [[ ${#ORCHESTRATORS_KILLED[@]} -gt 0 ]]; the
|
||||
done
|
||||
|
||||
if [[ ${#ALL_CONTAINERS[@]} -eq 0 ]]; then
|
||||
info "No containers defined — skipping recovery"
|
||||
log "No profile containers defined — skipping recovery"
|
||||
else
|
||||
for c in "${ALL_CONTAINERS[@]}"; do
|
||||
STATUS=$(docker inspect -f '{{.State.Running}}' "$c" 2>/dev/null || echo "unknown")
|
||||
if [[ "$STATUS" == "true" ]]; then
|
||||
info "$ICON_RUNNING $c — running ✅"
|
||||
elif [[ "$STATUS" == "false" ]]; then
|
||||
warn "$ICON_NOT_RUNNING $c — stopped, restarting..."
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would restart $c"
|
||||
else
|
||||
if docker start "$c" >/dev/null 2>&1; then
|
||||
success "$c restarted ✅"
|
||||
CONTAINERS_RESTARTED+=("$c")
|
||||
STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
|
||||
'{{.State.Running}}' "$c" 2>/dev/null || echo "unknown")
|
||||
case "$STATUS" in
|
||||
true)
|
||||
log "$c — running ✅"
|
||||
;;
|
||||
false)
|
||||
warn "$c — stopped — restarting..."
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would restart $c"
|
||||
else
|
||||
error "Failed to restart $c"
|
||||
if timeout "$DOCKER_TIMEOUT" docker start "$c" >/dev/null 2>&1; then
|
||||
warn "$c restarted ✅"
|
||||
CONTAINERS_RESTARTED+=("$c")
|
||||
else
|
||||
error "Failed to restart $c"
|
||||
CONTAINERS_FAILED+=("$c")
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
else
|
||||
info "$c not found on this host — skipping"
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
log "$c not found locally — skipping"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
fi
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Summary ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY RSYNC STOP SUMMARY ━━━━━"
|
||||
echo " Mode: $MODE"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_GEAR Mode: $MODE"
|
||||
echo ""
|
||||
|
||||
echo "$ICON_HOST Local ($LOCAL_SERVER_NAME):"
|
||||
echo "$ICON_HOST Local ($MY_ID):"
|
||||
[[ ${#ORCHESTRATORS_KILLED[@]} -gt 0 ]] && \
|
||||
echo " $ICON_STOPPED Orchestrators killed: ${ORCHESTRATORS_KILLED[*]}"
|
||||
[[ "$LOCAL_KILLED" == true ]] && \
|
||||
echo " $ICON_STOPPED Rsync killed" || \
|
||||
echo " $ICON_SUCCESS No rsync was running"
|
||||
warn " Orchestrators killed: ${ORCHESTRATORS_KILLED[*]}"
|
||||
if [[ "$LOCAL_KILLED" == true ]]; then
|
||||
warn " Rsync killed ✅"
|
||||
else
|
||||
log " No rsync was running"
|
||||
fi
|
||||
|
||||
echo "$ICON_NET Remote ($REMOTE_SERVER_NAME):"
|
||||
echo "$ICON_NET Remote ($REMOTE_ID — $REMOTE_SERVER_NAME):"
|
||||
if [[ "$REMOTE_REACHABLE" == false ]]; then
|
||||
echo " $ICON_WARN Unreachable — skipped"
|
||||
warn " Unreachable — skipped"
|
||||
else
|
||||
[[ ${#REMOTE_ORCHESTRATORS_KILLED[@]} -gt 0 ]] && \
|
||||
echo " $ICON_STOPPED Orchestrators killed: ${REMOTE_ORCHESTRATORS_KILLED[*]}"
|
||||
[[ "$REMOTE_KILLED" == true ]] && \
|
||||
echo " $ICON_STOPPED Rsync killed" || \
|
||||
echo " $ICON_SUCCESS No rsync was running"
|
||||
warn " Orchestrators killed: ${REMOTE_ORCHESTRATORS_KILLED[*]}"
|
||||
if [[ "$REMOTE_KILLED" == true ]]; then
|
||||
warn " Rsync killed ✅"
|
||||
else
|
||||
log " No rsync was running"
|
||||
fi
|
||||
fi
|
||||
|
||||
[[ ${#CONTAINERS_RESTARTED[@]} -gt 0 ]] && \
|
||||
echo "$ICON_CONTAINERS Containers recovered: ${CONTAINERS_RESTARTED[*]}"
|
||||
warn "$ICON_CONTAINERS Containers recovered: ${CONTAINERS_RESTARTED[*]}"
|
||||
[[ ${#CONTAINERS_FAILED[@]} -gt 0 ]] && \
|
||||
echo "$ICON_ERROR Containers failed to restart: ${CONTAINERS_FAILED[*]}"
|
||||
|
||||
echo ""
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "$ICON_WARN Status: DRY RUN — no changes made"
|
||||
warn "DRY RUN — no changes made"
|
||||
else
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS DONE"
|
||||
log "$ICON_DONE Status: done ✅"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
if [[ "$LOCAL_KILLED" == true ]] || [[ "$REMOTE_KILLED" == true ]] || \
|
||||
[[ ${#ORCHESTRATORS_KILLED[@]} -gt 0 ]]; then
|
||||
notify "Rsync stopped on $(hostname) — mode: $MODE — ${#CONTAINERS_RESTARTED[@]} containers recovered" \
|
||||
"Rsync Stop" "warning"
|
||||
# Notify if anything was actually killed or failed
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
if [[ ${#CONTAINERS_FAILED[@]} -gt 0 ]]; then
|
||||
notify "Rsync stop on $(hostname) ($MY_ID) — containers failed to restart: ${CONTAINERS_FAILED[*]}" \
|
||||
"Rsync Stop" "warning"
|
||||
elif [[ "$LOCAL_KILLED" == true || "$REMOTE_KILLED" == true || \
|
||||
${#ORCHESTRATORS_KILLED[@]} -gt 0 ]]; then
|
||||
notify "Rsync stopped on $(hostname) ($MY_ID) — mode: $MODE${CONTAINERS_RESTARTED:+ — recovered: ${CONTAINERS_RESTARTED[*]}}" \
|
||||
"Rsync Stop" "warning"
|
||||
fi
|
||||
fi
|
||||
+253
-124
@@ -1,159 +1,288 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- Server Reboot Script ---------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Gracefully reboots the unRAID server with a configurable user warning delay.
|
||||
# Stops Docker and VM Manager cleanly before issuing reboot.
|
||||
# Reboot delay is configured in Master.conf under REBOOT_SLEEP.
|
||||
# Supports --dry-run to walk through the sequence without actually rebooting.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ================================= Server Reboot ==============================================
|
||||
# ==============================================================================================
|
||||
# Gracefully reboots the unRAID server with full pre-flight checks and clean shutdown sequence.
|
||||
# Warns all users, checks for active processes, stops services, syncs disks, then reboots.
|
||||
#
|
||||
# ── SHUTDOWN SEQUENCE ─────────────────────────────────────────────────────────────────────────
|
||||
# 1. Pre-flight warnings — rsync, mover, active Emby sessions (warn not block)
|
||||
# 2. Wall message to all logged-in terminal users
|
||||
# 3. unRAID notification to dashboard
|
||||
# 4. Wait REBOOT_SLEEP seconds (default 30) — gives users time to save work
|
||||
# 5. Gracefully shutdown VMs (virsh shutdown each, then wait)
|
||||
# 6. Stop libvirt (VM Manager)
|
||||
# 7. Stop Docker service
|
||||
# 8. Sync filesystem buffers to disk
|
||||
# 9. Reboot
|
||||
#
|
||||
# ── PRE-FLIGHT WARNINGS ───────────────────────────────────────────────────────────────────────
|
||||
# The following are warnings only — they do not block the reboot. You called this script,
|
||||
# so you know what you're doing. The warnings give you context before the countdown starts.
|
||||
# - rsync running → partial files possible if mid-transfer
|
||||
# - mover running → files may be left on cache or array mid-move
|
||||
# - Emby sessions → active streams/transcodes will be interrupted
|
||||
#
|
||||
# ── VM GRACEFUL SHUTDOWN ──────────────────────────────────────────────────────────────────────
|
||||
# virsh shutdown sends ACPI power button signal to each VM — same as pressing power button.
|
||||
# VM gets a chance to flush its own buffers and shutdown cleanly.
|
||||
# Waits REBOOT_VM_WAIT seconds (default 30) for VMs to shut down before stopping libvirt.
|
||||
# If VMs don't shut down in time libvirt stops anyway — system reboot takes priority.
|
||||
#
|
||||
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
|
||||
# detect_hosts() sets MY_ID — used in wall message, notification, and summary.
|
||||
# Critical on a two-server setup — wall and notifications show WHICH server is rebooting.
|
||||
#
|
||||
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
|
||||
# Root check — reboot requires root
|
||||
# acquire_lock — prevents concurrent reboot calls
|
||||
# detect_hosts() — MY_ID in all user-facing messages
|
||||
# validate_unraid_cmd — notify validated before use
|
||||
# Graceful VM shutdown — VMs get clean ACPI signal before libvirt stops
|
||||
# sync before reboot — filesystem buffers flushed to disk
|
||||
#
|
||||
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||||
# REBOOT_SLEEP — seconds to warn users before starting shutdown sequence (default 30)
|
||||
# REBOOT_VM_WAIT — seconds to wait for VMs to shut down gracefully (default 30)
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# server_reboot.sh — reboot with 30s warning
|
||||
# server_reboot.sh --dry-run — walk through sequence without rebooting
|
||||
# server_reboot.sh --status — show running processes that would be affected
|
||||
# server_reboot.sh --reason="maintenance" — log reason for reboot
|
||||
# server_reboot.sh --log — verbose output
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
# ── Parse --reason flag before parse_args ─────────────────────────────────────────────────────
|
||||
REBOOT_REASON="manual"
|
||||
FILTERED_ARGS=()
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--reason=*) REBOOT_REASON="${arg#--reason=}" ;;
|
||||
*) FILTERED_ARGS+=("$arg") ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
parse_args "${FILTERED_ARGS[@]}"
|
||||
|
||||
# ROOT CHECK
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
error "Must be run as root — reboot requires root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Running as root"
|
||||
validate_unraid_cmd \
|
||||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||||
"" "" \
|
||||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||
|
||||
# VALIDATION
|
||||
validate_int REBOOT_SLEEP "$REBOOT_SLEEP"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Status ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
acquire_lock
|
||||
|
||||
detect_hosts
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made, no reboot will occur"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_REBOOT Delay: ${REBOOT_SLEEP}s"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "━━━━━ $ICON_SUMMARY REBOOT STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_REBOOT Delay: ${REBOOT_SLEEP}s"
|
||||
echo "$ICON_GEAR VM wait: ${REBOOT_VM_WAIT:-30}s"
|
||||
echo "$ICON_GEAR Reason: $REBOOT_REASON"
|
||||
echo ""
|
||||
echo "━━━ Active Processes ━━━"
|
||||
|
||||
pgrep -x rsync >/dev/null 2>&1 && \
|
||||
warn " rsync: RUNNING — partial files if rebooted now" || \
|
||||
log " rsync: not running"
|
||||
|
||||
pgrep -f "emhttp.*Mover" >/dev/null 2>&1 && \
|
||||
warn " mover: RUNNING — files may be left mid-move" || \
|
||||
log " mover: not running"
|
||||
|
||||
if command -v virsh >/dev/null 2>&1; then
|
||||
VM_COUNT=$(virsh list --name 2>/dev/null | grep -c "." || echo 0)
|
||||
[[ "$VM_COUNT" -gt 0 ]] && \
|
||||
warn " VMs: $VM_COUNT running — will be gracefully shut down" || \
|
||||
log " VMs: none running"
|
||||
fi
|
||||
|
||||
if command -v docker >/dev/null 2>&1; then
|
||||
CONTAINER_COUNT=$(docker ps -q 2>/dev/null | wc -l || echo 0)
|
||||
log " Docker: $CONTAINER_COUNT container(s) running"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# FUNCTIONS
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
# Broadcasts a wall message warning all logged in users of the upcoming reboot.
|
||||
notify_users() {
|
||||
warn "Notifying users — reboot in ${REBOOT_SLEEP}s"
|
||||
wall "$ICON_WARN unRAID server will reboot in ${REBOOT_SLEEP} second(s). Save your work."
|
||||
}
|
||||
|
||||
# Stops the Docker service cleanly.
|
||||
# Warns but continues if Docker is already stopped or fails — shutdown must proceed.
|
||||
stop_docker() {
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would stop Docker service"
|
||||
return
|
||||
fi
|
||||
|
||||
info "Stopping Docker service..."
|
||||
|
||||
if /etc/rc.d/rc.docker stop; then
|
||||
success "Docker stopped"
|
||||
else
|
||||
warn "Docker stop failed or already stopped — continuing"
|
||||
fi
|
||||
}
|
||||
|
||||
# Stops the VM Manager (libvirt) cleanly.
|
||||
# Warns but continues if libvirt is already stopped or fails — shutdown must proceed.
|
||||
stop_vm_manager() {
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would stop VM Manager (libvirt)"
|
||||
return
|
||||
fi
|
||||
|
||||
info "Stopping VM Manager..."
|
||||
|
||||
if /etc/rc.d/rc.libvirt stop; then
|
||||
success "VM Manager stopped"
|
||||
else
|
||||
warn "VM Manager stop failed or already stopped — continuing"
|
||||
fi
|
||||
}
|
||||
|
||||
# Flushes filesystem buffers to disk before reboot.
|
||||
sync_disks() {
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would sync filesystem buffers"
|
||||
return
|
||||
fi
|
||||
|
||||
info "Syncing disks..."
|
||||
|
||||
if sync; then
|
||||
success "Disk sync complete"
|
||||
else
|
||||
warn "Sync returned an error — continuing"
|
||||
fi
|
||||
}
|
||||
|
||||
# Issues the system reboot command.
|
||||
# System will not return from this call unless dry-run is active.
|
||||
reboot_system() {
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would reboot system now"
|
||||
return
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "$ICON_REBOOT Rebooting system NOW..."
|
||||
/sbin/reboot
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_REBOOT Reboot Sequence ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_REBOOT Reboot Sequence ━━━"
|
||||
echo "$ICON_REBOOT Delay: ${REBOOT_SLEEP}s"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
# ==============================================================================================
|
||||
# ━━━ Pre-flight Warnings ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_SHIELD Pre-flight ━━━"
|
||||
WARNINGS=()
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
# rsync check — partial files if killed mid-transfer
|
||||
if pgrep -x rsync >/dev/null 2>&1; then
|
||||
RSYNC_PIDS=$(pgrep -x rsync | tr '\n' ' ')
|
||||
warn "rsync is running (PIDs: $RSYNC_PIDS) — partial files possible"
|
||||
warn "Consider: rsync_stop.sh before rebooting"
|
||||
WARNINGS+=("rsync running")
|
||||
fi
|
||||
|
||||
# mover check — files may be left mid-move
|
||||
if pgrep -f "emhttp.*Mover" >/dev/null 2>&1; then
|
||||
warn "Mover is running — files may be left mid-move on cache or array"
|
||||
warn "Consider: mover_stop.sh before rebooting"
|
||||
WARNINGS+=("mover running")
|
||||
fi
|
||||
|
||||
# Emby sessions check — active streams interrupted
|
||||
if [[ -n "${EMBY_URL:-}" ]] && [[ -n "${EMBY_API_KEY:-}" ]]; then
|
||||
ACTIVE_STREAMS=$(curl -sf --max-time 5 \
|
||||
-H "X-Emby-Token: $EMBY_API_KEY" \
|
||||
"${EMBY_URL}/Sessions" 2>/dev/null | \
|
||||
grep -c "NowPlayingItem" 2>/dev/null || echo 0)
|
||||
ACTIVE_STREAMS="${ACTIVE_STREAMS//[^0-9]/}"
|
||||
if [[ "${ACTIVE_STREAMS:-0}" -gt 0 ]]; then
|
||||
warn "$ACTIVE_STREAMS active Emby stream(s) — will be interrupted"
|
||||
WARNINGS+=("${ACTIVE_STREAMS} Emby sessions")
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ ${#WARNINGS[@]} -eq 0 ]]; then
|
||||
log "Pre-flight clean — no active processes to warn about"
|
||||
else
|
||||
warn "Proceeding with reboot despite warnings — ${WARNINGS[*]}"
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Notify and Wait ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_REBOOT Reboot Sequence — $MY_ID ━━━"
|
||||
echo " Reason: $REBOOT_REASON"
|
||||
echo " Delay: ${REBOOT_SLEEP}s"
|
||||
echo " Dry Run: $DRY_RUN"
|
||||
echo ""
|
||||
|
||||
START=$(date +%s)
|
||||
|
||||
if [[ "$REBOOT_SLEEP" -gt 0 ]]; then
|
||||
notify_users
|
||||
info "Waiting ${REBOOT_SLEEP}s before shutdown sequence..."
|
||||
sleep "$REBOOT_SLEEP"
|
||||
# Wall message — terminal users
|
||||
wall "$ICON_WARN $MY_ID ($LOCAL_SERVER_NAME) rebooting in ${REBOOT_SLEEP}s — reason: $REBOOT_REASON. Save your work now."
|
||||
|
||||
# unRAID notification — dashboard
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
notify "$MY_ID ($LOCAL_SERVER_NAME) rebooting in ${REBOOT_SLEEP}s — reason: $REBOOT_REASON${WARNINGS:+ — warnings: ${WARNINGS[*]}}" \
|
||||
"Server Reboot" "warning"
|
||||
fi
|
||||
|
||||
warn "Waiting ${REBOOT_SLEEP}s before shutdown sequence..."
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
sleep "$REBOOT_SLEEP"
|
||||
else
|
||||
warn "DRY RUN — skipping sleep"
|
||||
fi
|
||||
fi
|
||||
|
||||
stop_docker
|
||||
stop_vm_manager
|
||||
sync_disks
|
||||
reboot_system
|
||||
# ==============================================================================================
|
||||
# ━━━ Graceful VM Shutdown ━━━
|
||||
# ==============================================================================================
|
||||
if command -v virsh >/dev/null 2>&1; then
|
||||
VM_LIST=$(virsh list --name 2>/dev/null | grep -v "^$" || true)
|
||||
if [[ -n "$VM_LIST" ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Graceful VM Shutdown ━━━"
|
||||
while IFS= read -r vm; do
|
||||
[[ -z "$vm" ]] && continue
|
||||
warn "Sending ACPI shutdown to VM: $vm"
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
virsh shutdown "$vm" >/dev/null 2>&1 || true
|
||||
else
|
||||
warn "DRY RUN — would virsh shutdown $vm"
|
||||
fi
|
||||
done <<< "$VM_LIST"
|
||||
|
||||
# NOTE: system will not reach here unless --dry-run is active
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
VM_WAIT="${REBOOT_VM_WAIT:-30}"
|
||||
log "Waiting ${VM_WAIT}s for VMs to shut down..."
|
||||
sleep "$VM_WAIT"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Stop VM Manager ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Stop VM Manager ━━━"
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would stop VM Manager (libvirt)"
|
||||
else
|
||||
if /etc/rc.d/rc.libvirt stop >/dev/null 2>&1; then
|
||||
warn "VM Manager stopped ✅"
|
||||
else
|
||||
warn "VM Manager stop returned non-zero — may already be stopped"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Stop Docker ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_CONTAINERS Stop Docker ━━━"
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would stop Docker service"
|
||||
else
|
||||
if /etc/rc.d/rc.docker stop >/dev/null 2>&1; then
|
||||
warn "Docker stopped ✅"
|
||||
else
|
||||
warn "Docker stop returned non-zero — may already be stopped"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Sync Disks ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_DISK Sync Disks ━━━"
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would sync filesystem buffers"
|
||||
else
|
||||
sync
|
||||
log "Filesystem buffers flushed ✅"
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Reboot ━━━
|
||||
# ==============================================================================================
|
||||
END=$(date +%s)
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Summary ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY REBOOT SUMMARY ━━━━━"
|
||||
echo "$ICON_REBOOT Delay: ${REBOOT_SLEEP}s"
|
||||
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_REBOOT Reason: $REBOOT_REASON"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
[[ ${#WARNINGS[@]} -gt 0 ]] && warn "Warnings: ${WARNINGS[*]}"
|
||||
echo ""
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "$ICON_WARN Status: DRY RUN — no reboot executed"
|
||||
warn "DRY RUN — sequence complete, no reboot executed"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
else
|
||||
echo "$ICON_REBOOT Status: $ICON_WARN SYSTEM SHOULD BE REBOOTING"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
warn "$ICON_REBOOT Rebooting $MY_ID now..."
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
/sbin/reboot
|
||||
fi
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,126 +1,231 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- User Script Stop -------------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ============================= User Scripts Stop ==============================================
|
||||
# ==============================================================================================
|
||||
# Stops all running User Script processes spawned by the unRAID User Scripts plugin.
|
||||
# Identifies processes by their /tmp/user.scripts path signature.
|
||||
# Supports --dry-run to preview what would be killed without making changes.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Shows script names not just PIDs — you know what's being stopped.
|
||||
#
|
||||
# ── WHEN TO USE ───────────────────────────────────────────────────────────────────────────────
|
||||
# - Before a planned reboot when scripts are running mid-cycle
|
||||
# - When a script is stuck and won't respond to the Abort button in the UI
|
||||
# - Called automatically by server_reboot.sh as part of shutdown sequence
|
||||
# - Emergency stop of all background ecosystem scripts
|
||||
#
|
||||
# ── HOW IT IDENTIFIES PROCESSES ───────────────────────────────────────────────────────────────
|
||||
# Scans /proc/*/cmdline for processes whose command line contains "/tmp/user.scripts".
|
||||
# The unRAID User Scripts plugin stages all scripts in /tmp/user.scripts/ before execution.
|
||||
# This is more reliable than process name matching which can vary.
|
||||
#
|
||||
# ── STOP SEQUENCE PER PROCESS ─────────────────────────────────────────────────────────────────
|
||||
# 1. Send SIGTERM — allows script to trap and clean up gracefully
|
||||
# 2. Wait 5 seconds
|
||||
# 3. Check if still running → SIGKILL (force) if SIGTERM ignored
|
||||
# 4. Verify dead after SIGKILL
|
||||
#
|
||||
# ── SELF-EXCLUSION ────────────────────────────────────────────────────────────────────────────
|
||||
# If this script itself is run via the User Scripts plugin it would find its own PID.
|
||||
# Self-exclusion prevents this script from killing itself mid-execution.
|
||||
#
|
||||
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
|
||||
# Root check — kill requires root for other users' processes
|
||||
# acquire_lock — prevents concurrent stop attempts
|
||||
# Self-exclusion — never kills its own process tree
|
||||
# SIGTERM → SIGKILL — graceful then forced
|
||||
# Verify after kill — confirms processes are actually dead
|
||||
# validate_unraid_cmd — notify validated before use
|
||||
# Silent when clean — no processes running = log() only ✅
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# user_scripts_stop.sh — stop all user scripts
|
||||
# user_scripts_stop.sh --dry-run — show what would be stopped
|
||||
# user_scripts_stop.sh --status — show currently running user scripts
|
||||
# user_scripts_stop.sh --log — verbose output
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
MY_PID=$$
|
||||
MY_PPID=$PPID
|
||||
|
||||
# ROOT CHECK
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
error "Must be run as root — kill requires root for other users' processes"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Running as root"
|
||||
validate_unraid_cmd \
|
||||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||||
"" "" \
|
||||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Status ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
acquire_lock
|
||||
|
||||
detect_hosts
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no processes will be killed"
|
||||
|
||||
# ==============================================================================================
|
||||
# ── HELPER FUNCTIONS ──────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# Get script name from PID — extracts meaningful name from /tmp/user.scripts path
|
||||
get_script_name() {
|
||||
local pid="$1"
|
||||
local cmdline
|
||||
cmdline=$(tr '\0' ' ' < /proc/"$pid"/cmdline 2>/dev/null || echo "")
|
||||
# Extract the script filename from the /tmp/user.scripts/... path
|
||||
echo "$cmdline" | grep -o '/tmp/user\.scripts[^ ]*' | \
|
||||
awk -F/ '{print $NF}' | head -1 || echo "pid-$pid"
|
||||
}
|
||||
|
||||
# Get all user script PIDs — excludes self and own parent process tree
|
||||
get_user_script_pids() {
|
||||
local -a pids=()
|
||||
while IFS= read -r pid; do
|
||||
[[ -z "$pid" ]] && continue
|
||||
# Self-exclusion — don't kill our own process or parent
|
||||
[[ "$pid" == "$MY_PID" ]] && continue
|
||||
[[ "$pid" == "$MY_PPID" ]] && continue
|
||||
pids+=("$pid")
|
||||
done < <(
|
||||
for dir in /proc/[0-9]*/cmdline; do
|
||||
pid="${dir%/cmdline}"
|
||||
pid="${pid#/proc/}"
|
||||
if grep -ql '/tmp/user\.scripts' "$dir" 2>/dev/null; then
|
||||
echo "$pid"
|
||||
fi
|
||||
done
|
||||
)
|
||||
printf '%s\n' "${pids[@]}"
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_PLUGIN Target: /tmp/user.scripts processes"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "━━━━━ $ICON_SUMMARY USER SCRIPTS STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo ""
|
||||
|
||||
mapfile -t PIDS < <(get_user_script_pids)
|
||||
|
||||
if [[ ${#PIDS[@]} -eq 0 ]]; then
|
||||
log "No User Script processes running"
|
||||
else
|
||||
echo " ${#PIDS[@]} User Script process(es) running:"
|
||||
for pid in "${PIDS[@]}"; do
|
||||
name=$(get_script_name "$pid")
|
||||
elapsed=$(ps -o etimes= -p "$pid" 2>/dev/null | tr -d ' ')
|
||||
runtime=$(format_duration "${elapsed:-0}")
|
||||
echo " $ICON_RUNNING PID $pid — $name (${runtime})"
|
||||
done
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# FUNCTIONS
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
# Returns PIDs of all processes running under /tmp/user.scripts
|
||||
# These are processes spawned by the unRAID User Scripts plugin.
|
||||
get_user_script_pids() {
|
||||
/usr/bin/ps -eo pid,cmd | grep "/tmp/user.scripts" | grep -v grep | awk '{print $1}'
|
||||
}
|
||||
|
||||
# Kills all running User Script processes one by one.
|
||||
# Reports each PID killed or skipped in dry run mode.
|
||||
stop_user_scripts() {
|
||||
local pids
|
||||
pids=$(get_user_script_pids)
|
||||
|
||||
if [[ -z "$pids" ]]; then
|
||||
info "$ICON_PLUGIN No running User Script processes found — nothing to do"
|
||||
return
|
||||
fi
|
||||
|
||||
local count=0
|
||||
|
||||
for pid in $pids; do
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would kill User Script PID $pid"
|
||||
else
|
||||
info "Killing User Script PID $pid..."
|
||||
|
||||
if kill "$pid" 2>/dev/null; then
|
||||
success "Killed PID $pid"
|
||||
else
|
||||
warn "Could not kill PID $pid — may have already exited"
|
||||
fi
|
||||
fi
|
||||
|
||||
count=$((count + 1))
|
||||
done
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would have targeted $count process(es)"
|
||||
else
|
||||
info "$count process(es) targeted"
|
||||
fi
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_PLUGIN User Script Stop ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ User Scripts Stop ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_PLUGIN User Script Stop ━━━"
|
||||
echo "$ICON_PLUGIN Target: User Scripts Plugin processes"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo ""
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
echo "━━━ $ICON_PLUGIN User Scripts Stop — $MY_ID ━━━"
|
||||
|
||||
START=$(date +%s)
|
||||
|
||||
stop_user_scripts
|
||||
mapfile -t PIDS < <(get_user_script_pids)
|
||||
|
||||
KILLED=()
|
||||
FAILED=()
|
||||
SKIPPED=()
|
||||
|
||||
if [[ ${#PIDS[@]} -eq 0 ]]; then
|
||||
log "No User Script processes running — nothing to do"
|
||||
else
|
||||
warn "${#PIDS[@]} User Script process(es) found"
|
||||
echo ""
|
||||
|
||||
for pid in "${PIDS[@]}"; do
|
||||
name=$(get_script_name "$pid")
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would stop: $name (PID $pid)"
|
||||
SKIPPED+=("$name")
|
||||
continue
|
||||
fi
|
||||
|
||||
# Verify still running before trying to kill
|
||||
if ! kill -0 "$pid" 2>/dev/null; then
|
||||
log "$name (PID $pid) — already exited"
|
||||
continue
|
||||
fi
|
||||
|
||||
# SIGTERM — graceful stop
|
||||
log "Sending SIGTERM to $name (PID $pid)..."
|
||||
kill -TERM "$pid" 2>/dev/null || true
|
||||
sleep 5
|
||||
|
||||
# Check if stopped after SIGTERM
|
||||
if ! kill -0 "$pid" 2>/dev/null; then
|
||||
warn "Stopped: $name (PID $pid) ✅"
|
||||
KILLED+=("$name")
|
||||
continue
|
||||
fi
|
||||
|
||||
# SIGKILL — forced stop
|
||||
warn "$name still running after SIGTERM — sending SIGKILL"
|
||||
kill -KILL "$pid" 2>/dev/null || true
|
||||
sleep 2
|
||||
|
||||
# Final verify
|
||||
if ! kill -0 "$pid" 2>/dev/null; then
|
||||
warn "Force-stopped: $name (PID $pid) ✅"
|
||||
KILLED+=("$name")
|
||||
else
|
||||
error "Failed to kill: $name (PID $pid)"
|
||||
FAILED+=("$name")
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Summary ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY USER SCRIPTS STOP SUMMARY ━━━━━"
|
||||
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "$ICON_WARN Status: DRY RUN — no processes killed"
|
||||
if [[ ${#PIDS[@]} -eq 0 ]]; then
|
||||
log "No processes were running"
|
||||
elif [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would have stopped ${#SKIPPED[@]} process(es): ${SKIPPED[*]}"
|
||||
else
|
||||
REMAINING=$(get_user_script_pids)
|
||||
if [[ -z "$REMAINING" ]]; then
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS ALL PROCESSES STOPPED"
|
||||
notify "User Scripts stopped on $(hostname)" "User Script Stop" "warning"
|
||||
else
|
||||
echo "$ICON_WARN Status: $ICON_WARN SOME PROCESSES MAY STILL BE RUNNING"
|
||||
notify "User Script stop completed but some processes may still be running on $(hostname)" "User Script Stop" "warning"
|
||||
fi
|
||||
[[ ${#KILLED[@]} -gt 0 ]] && warn "Stopped (${#KILLED[@]}): ${KILLED[*]}"
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed (${#FAILED[@]}): ${FAILED[*]}"
|
||||
fi
|
||||
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo ""
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no changes made"
|
||||
elif [[ ${#FAILED[@]} -gt 0 ]]; then
|
||||
echo "$ICON_ERROR Status: SOME PROCESSES COULD NOT BE KILLED"
|
||||
notify "User Scripts stop failed on $(hostname) ($MY_ID) — unkillable: ${FAILED[*]}" \
|
||||
"User Scripts Stop" "warning"
|
||||
else
|
||||
log "$ICON_DONE Status: done ✅"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
+207
-138
@@ -1,198 +1,267 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- WebGUI Watchdog --------------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Monitors unRAID's WebGUI and restarts it if unresponsive.
|
||||
# Uses an escalating restart strategy — tries nginx first, then emhttp if needed.
|
||||
# emhttp is the unRAID management daemon — restarting it is more disruptive than nginx
|
||||
# but recovers cleanly. Notification sent on any restart so you know what happened.
|
||||
# ==============================================================================================
|
||||
# ================================= WebGUI Watchdog ============================================
|
||||
# ==============================================================================================
|
||||
# Monitors the unRAID WebGUI and restarts services if unresponsive.
|
||||
# Uses a three-step escalating strategy — lightest fix first, heaviest last.
|
||||
# Run every 5-10 minutes via User Scripts plugin.
|
||||
# Silent when healthy — only produces output when something needs fixing.
|
||||
#
|
||||
# Escalation path:
|
||||
# Check WebGUI → unresponsive → restart nginx → recheck
|
||||
# Still unresponsive → restart emhttp → recheck
|
||||
# Still unresponsive → notify warning, manual intervention needed
|
||||
# ── ESCALATION PATH ───────────────────────────────────────────────────────────────────────────
|
||||
# Check WebGUI → responding → log() + exit 0 (completely silent ✅)
|
||||
#
|
||||
# Run every 5-10 minutes via cron/User Scripts plugin.
|
||||
# All configuration in Master.conf under WebGUI Watchdog section.
|
||||
# Supports --dry-run to show what would be restarted without acting.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Not responding:
|
||||
# Step 1 — Restart nginx
|
||||
# Lightest fix — handles most transient WebGUI failures
|
||||
# nginx crash, worker stuck, connection timeout
|
||||
# Wait WEBGUI_NGINX_WAIT seconds → recheck
|
||||
#
|
||||
# Step 2 — Restart php-fpm
|
||||
# WebGUI runs through PHP-FPM — worker exhaustion causes silent failure
|
||||
# php-fpm workers saturated → new requests queue → WebGUI appears frozen
|
||||
# system_tuning_monitor.sh tracks usage — this recovers it
|
||||
# Wait WEBGUI_PHP_WAIT seconds → recheck
|
||||
#
|
||||
# Step 3 — Restart emhttp
|
||||
# Heaviest fix — emhttp is the unRAID management daemon
|
||||
# Array, Docker, shares stay running — only WebGUI management restarts
|
||||
# Takes longer to recover — WEBGUI_EMHTTP_WAIT gives it time
|
||||
# Wait WEBGUI_EMHTTP_WAIT seconds → recheck
|
||||
#
|
||||
# All three failed → notify warning, manual intervention needed → exit 1
|
||||
#
|
||||
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
|
||||
# detect_hosts() sets MY_ID — used in all notifications and summary.
|
||||
# Critical on two-server setup — which server's WebGUI failed?
|
||||
#
|
||||
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
|
||||
# acquire_lock — prevents concurrent runs double-restarting services
|
||||
# detect_hosts() — MY_ID in all notifications
|
||||
# Process verify — pgrep check after each service restart
|
||||
# Silent healthy — completely silent on healthy cycle ✅
|
||||
# validate_unraid_cmd — notify validated before use
|
||||
#
|
||||
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||||
# WEBGUI_URL — URL to check (default http://localhost)
|
||||
# WEBGUI_TIMEOUT — curl timeout in seconds (default 5)
|
||||
# WEBGUI_NGINX_WAIT — seconds after nginx restart before rechecking (default 15)
|
||||
# WEBGUI_PHP_WAIT — seconds after php-fpm restart before rechecking (default 10)
|
||||
# WEBGUI_EMHTTP_WAIT — seconds after emhttp restart before rechecking (default 30)
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# webgui_restart.sh — check and recover if needed
|
||||
# webgui_restart.sh --dry-run — show what would be restarted
|
||||
# webgui_restart.sh --status — show current WebGUI and service states
|
||||
# webgui_restart.sh --log — verbose output
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Running as root"
|
||||
validate_unraid_cmd \
|
||||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||||
"" "" \
|
||||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Status ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
acquire_lock
|
||||
|
||||
detect_hosts
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no services will be restarted"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_WEBGUI URL: $WEBGUI_URL"
|
||||
echo "$ICON_WEBGUI Curl timeout: ${WEBGUI_TIMEOUT}s"
|
||||
echo "$ICON_WEBGUI Nginx wait: ${WEBGUI_NGINX_WAIT}s"
|
||||
echo "$ICON_WEBGUI emhttp wait: ${WEBGUI_EMHTTP_WAIT}s"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "━━━━━ $ICON_SUMMARY WEBGUI WATCHDOG STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_WEBGUI URL: $WEBGUI_URL"
|
||||
echo "$ICON_WEBGUI Timeouts: curl=${WEBGUI_TIMEOUT}s nginx=${WEBGUI_NGINX_WAIT}s php=${WEBGUI_PHP_WAIT:-10}s emhttp=${WEBGUI_EMHTTP_WAIT}s"
|
||||
echo ""
|
||||
|
||||
# Show current state
|
||||
if curl -sf --max-time "$WEBGUI_TIMEOUT" "$WEBGUI_URL" >/dev/null 2>&1; then
|
||||
echo "$ICON_WEBGUI WebGUI: $ICON_RUNNING responding"
|
||||
echo " $ICON_SUCCESS WebGUI: responding ✅"
|
||||
else
|
||||
echo "$ICON_WEBGUI WebGUI: $ICON_NOT_RUNNING not responding"
|
||||
echo " $ICON_ERROR WebGUI: NOT responding"
|
||||
fi
|
||||
|
||||
pgrep -x nginx >/dev/null 2>&1 && \
|
||||
echo " $ICON_SUCCESS nginx: running ✅" || \
|
||||
echo " $ICON_ERROR nginx: NOT running"
|
||||
|
||||
pgrep -f "php-fpm" >/dev/null 2>&1 && \
|
||||
FPM_COUNT=$(pgrep -fc "php-fpm" 2>/dev/null || echo "?") && \
|
||||
echo " $ICON_SUCCESS php-fpm: running ($FPM_COUNT workers) ✅" || \
|
||||
echo " $ICON_ERROR php-fpm: NOT running"
|
||||
|
||||
pgrep -x emhttp >/dev/null 2>&1 && \
|
||||
echo " $ICON_SUCCESS emhttp: running ✅" || \
|
||||
echo " $ICON_ERROR emhttp: NOT running"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no services will be restarted"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# FUNCTIONS
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
# Check if WebGUI is responding
|
||||
# ==============================================================================================
|
||||
# ── CHECK AND ESCALATE ────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
check_webgui() {
|
||||
curl -sf --max-time "$WEBGUI_TIMEOUT" "$WEBGUI_URL" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
# Restart nginx — lightweight fix, try first
|
||||
restart_nginx() {
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would restart nginx"
|
||||
return 0
|
||||
fi
|
||||
|
||||
info "$ICON_WEBGUI Restarting nginx..."
|
||||
if /etc/rc.d/rc.nginx restart >/dev/null 2>&1; then
|
||||
success "nginx restarted"
|
||||
return 0
|
||||
else
|
||||
error "nginx restart failed"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Restart emhttp — heavier fix, escalate if nginx didn't help
|
||||
# emhttp drives the array, Docker management, shares — recovers cleanly but takes longer
|
||||
restart_emhttp() {
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would restart emhttp"
|
||||
return 0
|
||||
fi
|
||||
|
||||
info "$ICON_WEBGUI Restarting emhttp..."
|
||||
if /etc/rc.d/rc.emhttp restart >/dev/null 2>&1; then
|
||||
success "emhttp restarted"
|
||||
return 0
|
||||
else
|
||||
error "emhttp restart failed"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_WEBGUI WebGUI Watchdog ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_WEBGUI WebGUI Watchdog — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo "$ICON_WEBGUI URL: $WEBGUI_URL"
|
||||
echo ""
|
||||
|
||||
START=$(date +%s)
|
||||
RECOVERY_ACTION=""
|
||||
RECOVERY_OK=false
|
||||
|
||||
# Initial check
|
||||
info "Checking WebGUI..."
|
||||
log "WebGUI check — $WEBGUI_URL"
|
||||
|
||||
# ── Healthy — completely silent ───────────────────────────────────────────────────────────────
|
||||
if check_webgui; then
|
||||
success "$ICON_WEBGUI WebGUI is responding — nothing to do"
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY WEBGUI WATCHDOG SUMMARY ━━━━━"
|
||||
echo "$ICON_WEBGUI Status: $ICON_RUNNING HEALTHY"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(($(date +%s) - START)))"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
log "WebGUI responding — healthy ✅"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# WebGUI not responding — begin escalation
|
||||
warn "$ICON_WEBGUI WebGUI is not responding at $WEBGUI_URL"
|
||||
|
||||
# ── Step 1: Restart nginx ──
|
||||
# ── Not responding — begin escalation ────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_WEBGUI Step 1 — Nginx Restart ━━━"
|
||||
echo "━━━ $ICON_WEBGUI WebGUI Watchdog — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo ""
|
||||
warn "WebGUI not responding at $WEBGUI_URL — beginning escalation"
|
||||
|
||||
restart_nginx
|
||||
# ── Step 1 — nginx restart ────────────────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ Step 1 — nginx Restart ━━━"
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
info "Waiting ${WEBGUI_NGINX_WAIT}s for nginx to recover..."
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would restart nginx"
|
||||
else
|
||||
warn "Restarting nginx..."
|
||||
if /etc/rc.d/rc.nginx restart >/dev/null 2>&1; then
|
||||
# Verify nginx actually running
|
||||
sleep 2
|
||||
if pgrep -x nginx >/dev/null 2>&1; then
|
||||
warn "nginx restarted ✅"
|
||||
else
|
||||
error "nginx not running after restart command"
|
||||
fi
|
||||
else
|
||||
error "nginx restart command failed"
|
||||
fi
|
||||
|
||||
log "Waiting ${WEBGUI_NGINX_WAIT}s for nginx to recover..."
|
||||
sleep "$WEBGUI_NGINX_WAIT"
|
||||
|
||||
if check_webgui; then
|
||||
success "$ICON_WEBGUI WebGUI recovered after nginx restart"
|
||||
notify "WebGUI recovered on $(hostname) after nginx restart" "WebGUI Watchdog" "warning"
|
||||
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY WEBGUI WATCHDOG SUMMARY ━━━━━"
|
||||
echo "$ICON_WEBGUI Status: $ICON_SUCCESS RECOVERED via nginx restart"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(($(date +%s) - START)))"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
RECOVERY_ACTION="nginx restart"
|
||||
RECOVERY_OK=true
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Step 2 — php-fpm restart ──────────────────────────────────────────────────────────────────
|
||||
if [[ "$RECOVERY_OK" == false ]]; then
|
||||
echo ""
|
||||
echo "━━━ Step 2 — php-fpm Restart ━━━"
|
||||
warn "WebGUI still not responding — restarting php-fpm"
|
||||
warn "WebGUI may be frozen due to worker exhaustion (check system_tuning_monitor.sh)"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would restart php-fpm"
|
||||
else
|
||||
if /etc/rc.d/rc.php-fpm restart >/dev/null 2>&1; then
|
||||
sleep 2
|
||||
if pgrep -f "php-fpm" >/dev/null 2>&1; then
|
||||
warn "php-fpm restarted ✅"
|
||||
else
|
||||
error "php-fpm not running after restart command"
|
||||
fi
|
||||
else
|
||||
error "php-fpm restart command failed"
|
||||
fi
|
||||
|
||||
warn "WebGUI still not responding after nginx restart — escalating to emhttp"
|
||||
log "Waiting ${WEBGUI_PHP_WAIT:-10}s for php-fpm to recover..."
|
||||
sleep "${WEBGUI_PHP_WAIT:-10}"
|
||||
|
||||
if check_webgui; then
|
||||
RECOVERY_ACTION="php-fpm restart"
|
||||
RECOVERY_OK=true
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Step 2: Restart emhttp ──
|
||||
echo ""
|
||||
echo "━━━ $ICON_WEBGUI Step 2 — emhttp Restart ━━━"
|
||||
warn "Restarting emhttp — this is the unRAID management daemon"
|
||||
warn "Array, Docker management and shares remain running but WebGUI will be briefly unavailable"
|
||||
|
||||
restart_emhttp
|
||||
# ── Step 3 — emhttp restart ───────────────────────────────────────────────────────────────────
|
||||
if [[ "$RECOVERY_OK" == false ]]; then
|
||||
echo ""
|
||||
echo "━━━ Step 3 — emhttp Restart ━━━"
|
||||
warn "WebGUI still not responding — restarting emhttp (unRAID management daemon)"
|
||||
warn "Array, Docker, and shares remain running — WebGUI management will briefly restart"
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
info "Waiting ${WEBGUI_EMHTTP_WAIT}s for emhttp to recover..."
|
||||
sleep "$WEBGUI_EMHTTP_WAIT"
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would restart emhttp"
|
||||
else
|
||||
if /etc/rc.d/rc.emhttp restart >/dev/null 2>&1; then
|
||||
sleep 2
|
||||
if pgrep -x emhttp >/dev/null 2>&1; then
|
||||
warn "emhttp restarted ✅"
|
||||
else
|
||||
error "emhttp not running after restart command"
|
||||
fi
|
||||
else
|
||||
error "emhttp restart command failed"
|
||||
fi
|
||||
|
||||
if check_webgui; then
|
||||
success "$ICON_WEBGUI WebGUI recovered after emhttp restart"
|
||||
notify "WebGUI recovered on $(hostname) after emhttp restart — check system health" "WebGUI Watchdog" "warning"
|
||||
log "Waiting ${WEBGUI_EMHTTP_WAIT}s for emhttp to recover..."
|
||||
sleep "$WEBGUI_EMHTTP_WAIT"
|
||||
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY WEBGUI WATCHDOG SUMMARY ━━━━━"
|
||||
echo "$ICON_WEBGUI Status: $ICON_SUCCESS RECOVERED via emhttp restart"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(($(date +%s) - START)))"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
if check_webgui; then
|
||||
RECOVERY_ACTION="emhttp restart"
|
||||
RECOVERY_OK=true
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Both restarts failed ──
|
||||
END=$(date +%s)
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY WEBGUI WATCHDOG SUMMARY ━━━━━"
|
||||
echo "$ICON_WEBGUI Status: $ICON_ERROR UNRECOVERED — manual intervention needed"
|
||||
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_WEBGUI URL: $WEBGUI_URL"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
notify "WebGUI unrecovered on $(hostname) after nginx and emhttp restart — manual intervention needed" "WebGUI Watchdog" "warning"
|
||||
exit 1
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no services restarted"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ "$RECOVERY_OK" == true ]]; then
|
||||
warn "$ICON_SUCCESS WebGUI recovered via: $RECOVERY_ACTION"
|
||||
notify "WebGUI recovered on $(hostname) ($MY_ID) via $RECOVERY_ACTION — monitor for recurrence" \
|
||||
"WebGUI Watchdog" "warning"
|
||||
else
|
||||
echo "$ICON_ERROR Status: UNRECOVERED — all three restart steps failed"
|
||||
echo "$ICON_ERROR Manual intervention needed:"
|
||||
echo " 1. Check: pgrep -x nginx emhttp"
|
||||
echo " 2. Check: journalctl -u nginx --since '10 minutes ago'"
|
||||
echo " 3. Try: server_reboot.sh if nothing else works"
|
||||
notify "WebGUI UNRECOVERED on $(hostname) ($MY_ID) — nginx + php-fpm + emhttp restart all failed — manual intervention needed" \
|
||||
"WebGUI Watchdog" "warning"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ "$RECOVERY_OK" == false && "$DRY_RUN" == false ]] && exit 1
|
||||
exit 0
|
||||
Reference in New Issue
Block a user