Gmer4Lfe
c377ddfcca
Finishes the pass: every script now documents its safeguards, and the deliberate absences in the sourced libraries are recorded so they are not "corrected" later.
648 lines
28 KiB
Bash
Executable File
648 lines
28 KiB
Bash
Executable File
#!/bin/bash
|
||
# ==============================================================================================
|
||
# ================================= Resource Manager ===========================================
|
||
# ==============================================================================================
|
||
#
|
||
# PURPOSE
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# Pressure reduction layer — detects rising system load and reduces it before
|
||
# things break. Called by watchdog_orchestrator.sh every 15 minutes as a
|
||
# single-pass run. The middle layer between docker_watchdog.sh (fixes broken
|
||
# containers) and stability_watchdog.sh (reboots). Does neither of those things.
|
||
#
|
||
# ==============================================================================================
|
||
# OPERATIONAL MODEL
|
||
# ==============================================================================================
|
||
#
|
||
# Three-Level Pressure Response
|
||
#
|
||
# Level 1 — SOFT (RAM < RW_RAM_SOFT_GB OR load > RW_LOAD_SOFT_MULTIPLIER × cores):
|
||
# Throttle SABnzbd download speed to RW_SABNZBD_SPEED_SOFT.
|
||
# Throttle qBittorrent download to RW_QBIT_DL_SOFT KB/s.
|
||
#
|
||
# Level 2 — MEDIUM (RAM < RW_RAM_MEDIUM_GB OR load > RW_LOAD_MEDIUM_MULTIPLIER × cores):
|
||
# Further throttle SABnzbd + qBittorrent to medium limits.
|
||
# docker pause RW_PAUSE_CONTAINERS — suspend without losing state, instantly reversible.
|
||
#
|
||
# Level 3 — HARD (RAM < RW_RAM_HARD_GB):
|
||
# docker stop RW_STOP_CONTAINERS — optional/heavy services (games, LocalAI, etc.).
|
||
# Write mem_shutdown_active=true → signals docker_watchdog to defer container restarts.
|
||
#
|
||
# Recovery
|
||
# Pressure must stay below current threshold for RW_RECOVER_CYCLES consecutive
|
||
# runs before restoring. De-escalates one level at a time — prevents re-triggering
|
||
# immediately after recovery. Level 3 additionally requires RAM >= RW_RAM_RECOVER_GB
|
||
# before containers are un-stopped.
|
||
#
|
||
# Coordination with docker_watchdog.sh
|
||
# At level 3, writes mem_shutdown_active=true to RW_STATE_FILE.
|
||
# docker_watchdog.sh reads this and defers all container restart logic.
|
||
# Without this, docker_watchdog would immediately restart containers that were
|
||
# just stopped to free RAM — defeating the purpose of level 3.
|
||
# Cleared when pressure resolves and containers are restarted.
|
||
#
|
||
# ==============================================================================================
|
||
# DESIGN PRINCIPLES
|
||
# ==============================================================================================
|
||
#
|
||
# Graduated Response
|
||
# Pressure is answered with the smallest effective action first — throttle,
|
||
# then pause, then stop. Each level is only reached because the level below it
|
||
# failed to relieve pressure. Nothing jumps straight to stopping containers.
|
||
#
|
||
# Reversibility First
|
||
# docker pause suspends a container without losing its state and is instantly
|
||
# reversible, so it is preferred at level 2. docker stop, which discards
|
||
# in-memory state, is held back to level 3 and applied only to services
|
||
# explicitly listed as expendable in RW_STOP_CONTAINERS.
|
||
#
|
||
# Hysteresis on Recovery
|
||
# Restoring requires RW_RECOVER_CYCLES consecutive clear cycles and
|
||
# de-escalates one level at a time. Recovering instantly on a single good
|
||
# reading would flap — restore, re-trigger, restore — under sustained load.
|
||
#
|
||
# Cross-Watchdog Coordination
|
||
# Level 3 publishes mem_shutdown_active=true so docker_watchdog.sh defers its
|
||
# restart logic. Two watchdogs acting on the same containers with opposite
|
||
# intent would otherwise fight: one stopping to free RAM, the other restarting
|
||
# to restore health.
|
||
#
|
||
# ==============================================================================================
|
||
# OPERATIONAL SAFEGUARDS
|
||
# ==============================================================================================
|
||
#
|
||
# Root Required
|
||
# docker pause/stop require root.
|
||
#
|
||
# Single Instance Lock
|
||
# acquire_lock prevents concurrent runs from racing on state file writes.
|
||
#
|
||
# Docker Presence Check
|
||
# Verifies the docker binary exists before any pressure response — every
|
||
# level-2 and level-3 action depends on it.
|
||
#
|
||
# Host Detection
|
||
# detect_hosts() aliases HOST*_RW_PAUSE_CONTAINERS, HOST*_RW_STOP_CONTAINERS
|
||
# and the downloader credentials to the correct host's values.
|
||
#
|
||
# State File Verification
|
||
# Exits if RW_STATE_FILE cannot be created. Without durable state the script
|
||
# cannot track recovery cycles or know which containers it paused, and would
|
||
# never restore them.
|
||
#
|
||
# RW_CRITICAL_CONTAINERS
|
||
# Containers listed here are never paused or stopped regardless of pressure
|
||
# level. Enforced by is_critical(), which gates both the pause and the stop
|
||
# path — not just the configuration lists.
|
||
#
|
||
# RW_ENABLED Flag
|
||
# Set RW_ENABLED=false to disable the entire script without removing it from
|
||
# the orchestrator schedule.
|
||
#
|
||
# Timeout Protection
|
||
# All docker commands wrapped in a 15 second timeout. Pressure response runs
|
||
# during a degraded system, which is exactly when the daemon is most likely
|
||
# to be slow — a hang here would stall the whole watchdog chain for that cycle.
|
||
#
|
||
# Downloader Availability Guards
|
||
# SABnzbd and qBittorrent throttling no-ops when the service is disabled or
|
||
# its URL/credentials are unset. A missing downloader never blocks the
|
||
# container-level pressure response.
|
||
#
|
||
# Recovery Hysteresis
|
||
# Restoration requires RW_RECOVER_CYCLES consecutive clear cycles and
|
||
# de-escalates one level per cycle, preventing flapping under sustained load.
|
||
#
|
||
# Dry Run Support
|
||
# --dry-run reports every throttle, pause and stop without performing any.
|
||
#
|
||
# ==============================================================================================
|
||
# CONFIGURATION
|
||
# ==============================================================================================
|
||
#
|
||
# master.conf
|
||
# RW_ENABLED, RW_STATE_FILE
|
||
# RW_RAM_SOFT_GB, RW_RAM_MEDIUM_GB, RW_RAM_HARD_GB, RW_RAM_RECOVER_GB
|
||
# RW_LOAD_SOFT_MULTIPLIER, RW_LOAD_MEDIUM_MULTIPLIER
|
||
# RW_RECOVER_CYCLES
|
||
# RW_SABNZBD_ENABLED, RW_SABNZBD_SPEED_SOFT, RW_SABNZBD_SPEED_MEDIUM
|
||
# RW_QBIT_ENABLED, RW_QBIT_DL_SOFT, RW_QBIT_DL_MEDIUM
|
||
# RW_CRITICAL_CONTAINERS — never paused or stopped regardless of pressure
|
||
#
|
||
# host*.conf (aliased by detect_hosts())
|
||
# HOST*_RW_PAUSE_CONTAINERS — docker pause at medium pressure
|
||
# HOST*_RW_STOP_CONTAINERS — docker stop at hard pressure
|
||
# HOST*_SABNZBD_URL, HOST*_SABNZBD_API_KEY
|
||
# HOST*_QBIT_URL, HOST*_QBIT_USERNAME, HOST*_QBIT_PASSWORD
|
||
#
|
||
# ==============================================================================================
|
||
# STATE FILES
|
||
# ==============================================================================================
|
||
#
|
||
# RW_STATE_FILE
|
||
# Pressure level, recovery cycle count, stopped container list, and the
|
||
# mem_shutdown_active coordination flag read by docker_watchdog.sh.
|
||
#
|
||
# ==============================================================================================
|
||
# RUNTIME MODES
|
||
# ==============================================================================================
|
||
#
|
||
# resource_watchdog.sh
|
||
# Single-pass pressure check. Apply actions if threshold crossed. Silent if below.
|
||
#
|
||
# resource_watchdog.sh --dry-run
|
||
# Show current pressure level and what would be throttled/paused/stopped. No changes.
|
||
#
|
||
# resource_watchdog.sh --status
|
||
# Show current pressure level, active actions, recovery cycle count, stopped containers.
|
||
#
|
||
# resource_watchdog.sh --log
|
||
# Verbose per-check output — show RAM, load, each threshold comparison, each action.
|
||
#
|
||
# ==============================================================================================
|
||
|
||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||
|
||
source "$SCRIPT_DIR/../load_config.sh"
|
||
|
||
parse_args "$@"
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Setup ━━━
|
||
# ==============================================================================================
|
||
if [[ "$EUID" -ne 0 ]]; then
|
||
error "Must be run as root"
|
||
exit 1
|
||
fi
|
||
|
||
if [[ "${RW_ENABLED:-true}" != "true" ]]; then
|
||
echo "Resource Manager disabled (RW_ENABLED=false)"
|
||
exit 0
|
||
fi
|
||
|
||
acquire_lock
|
||
|
||
if ! command -v docker &>/dev/null; then
|
||
error "Docker command not found"
|
||
exit 1
|
||
fi
|
||
|
||
detect_hosts
|
||
|
||
DOCKER_TIMEOUT=15
|
||
|
||
TOTAL_CORES=$(nproc)
|
||
RW_LOAD_SOFT_THRESH=$(awk "BEGIN{printf \"%.0f\", $TOTAL_CORES * ${RW_LOAD_SOFT_MULTIPLIER:-2.0}}")
|
||
RW_LOAD_MEDIUM_THRESH=$(awk "BEGIN{printf \"%.0f\", $TOTAL_CORES * ${RW_LOAD_MEDIUM_MULTIPLIER:-3.0}}")
|
||
|
||
log "$ICON_GEAR Config: soft=RAM<${RW_RAM_SOFT_GB}GB/load≥${RW_LOAD_SOFT_THRESH} medium=RAM<${RW_RAM_MEDIUM_GB}GB/load≥${RW_LOAD_MEDIUM_THRESH} hard=RAM<${RW_RAM_HARD_GB}GB recover=RAM≥${RW_RAM_RECOVER_GB}GB cycles=${RW_RECOVER_CYCLES}"
|
||
log "$ICON_CONTAINERS Pause at medium: ${RW_PAUSE_CONTAINERS[*]:-none} Stop at hard: ${RW_STOP_CONTAINERS[*]:-none}"
|
||
|
||
touch "$RW_STATE_FILE" 2>/dev/null || {
|
||
error "Cannot create state file: $RW_STATE_FILE"
|
||
exit 1
|
||
}
|
||
|
||
# ── Exit Trap — restart containers stopped this run if script crashes ──────────────────────────
|
||
declare -a _RW_TRAP_STOPPED=()
|
||
|
||
_rw_trap_restart_stopped() {
|
||
[[ ${#_RW_TRAP_STOPPED[@]} -eq 0 ]] && return
|
||
for c in "${_RW_TRAP_STOPPED[@]}"; do
|
||
[[ -z "$c" ]] && continue
|
||
if docker inspect "$c" >/dev/null 2>&1; then
|
||
warn "Exit trap: restarting $c (stopped but state not persisted)"
|
||
docker start "$c" >/dev/null 2>&1 || warn " Failed to restart $c"
|
||
fi
|
||
done
|
||
}
|
||
trap "_release_all_locks; _rw_trap_restart_stopped" EXIT
|
||
|
||
# ==============================================================================================
|
||
# ━━━ State Helpers ━━━
|
||
# ==============================================================================================
|
||
# rm_state_get/set use : separator for RM-internal state
|
||
# rm_state_get_eq/set_eq use = separator for docker_watchdog coordination flags
|
||
# Both wrap common.sh's wd_state_get()/wd_state_set() — file is implicit here (this
|
||
# script's own state file), unlike the generic helper which always takes it as an argument.
|
||
|
||
rm_state_get() {
|
||
wd_state_get "$1" "$RW_STATE_FILE"
|
||
}
|
||
|
||
rm_state_set() {
|
||
local key="$1" val="$2"
|
||
wd_state_set "$key" "$val" "$RW_STATE_FILE"
|
||
}
|
||
|
||
rm_state_get_eq() {
|
||
wd_state_get "$1" "$RW_STATE_FILE" "="
|
||
}
|
||
|
||
rm_state_set_eq() {
|
||
local key="$1" val="$2"
|
||
wd_state_set "$key" "$val" "$RW_STATE_FILE" "="
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Load State ━━━
|
||
# ==============================================================================================
|
||
CURRENT_LEVEL=$(rm_state_get "rm_action_level"); CURRENT_LEVEL=${CURRENT_LEVEL:-0}
|
||
RECOVER_CYCLES=$(rm_state_get "rm_recover_cycles"); RECOVER_CYCLES=${RECOVER_CYCLES:-0}
|
||
PAUSED_LIST=$(rm_state_get "rm_paused_containers"); PAUSED_LIST=${PAUSED_LIST:-""}
|
||
STOPPED_LIST=$(rm_state_get "rm_stopped_containers"); STOPPED_LIST=${STOPPED_LIST:-""}
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Pressure Calculation ━━━
|
||
# ==============================================================================================
|
||
MEM_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo)
|
||
MEM_GB=$(( MEM_KB / 1024 / 1024 ))
|
||
LOAD=$(awk '{print $1}' /proc/loadavg)
|
||
LOAD_INT=$(printf "%.0f" "$LOAD")
|
||
|
||
TARGET_LEVEL=0
|
||
TARGET_REASON=""
|
||
if [[ "$MEM_GB" -lt "${RW_RAM_HARD_GB:-6}" ]]; then
|
||
TARGET_LEVEL=3
|
||
TARGET_REASON="RAM ${MEM_GB}GB < hard threshold ${RW_RAM_HARD_GB}GB"
|
||
elif [[ "$MEM_GB" -lt "${RW_RAM_MEDIUM_GB:-8}" ]] || [[ "$LOAD_INT" -ge "$RW_LOAD_MEDIUM_THRESH" ]]; then
|
||
TARGET_LEVEL=2
|
||
[[ "$MEM_GB" -lt "${RW_RAM_MEDIUM_GB:-8}" ]] && TARGET_REASON="RAM ${MEM_GB}GB < medium threshold ${RW_RAM_MEDIUM_GB}GB"
|
||
[[ "$LOAD_INT" -ge "$RW_LOAD_MEDIUM_THRESH" ]] && TARGET_REASON="${TARGET_REASON:+$TARGET_REASON, }load ${LOAD} >= medium threshold ${RW_LOAD_MEDIUM_THRESH}"
|
||
elif [[ "$MEM_GB" -lt "${RW_RAM_SOFT_GB:-12}" ]] || [[ "$LOAD_INT" -ge "$RW_LOAD_SOFT_THRESH" ]]; then
|
||
TARGET_LEVEL=1
|
||
[[ "$MEM_GB" -lt "${RW_RAM_SOFT_GB:-12}" ]] && TARGET_REASON="RAM ${MEM_GB}GB < soft threshold ${RW_RAM_SOFT_GB}GB"
|
||
[[ "$LOAD_INT" -ge "$RW_LOAD_SOFT_THRESH" ]] && TARGET_REASON="${TARGET_REASON:+$TARGET_REASON, }load ${LOAD} >= soft threshold ${RW_LOAD_SOFT_THRESH}"
|
||
fi
|
||
|
||
LEVEL_NAMES=("normal" "soft" "medium" "hard")
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Status ━━━
|
||
# ==============================================================================================
|
||
if [[ "$SHOW_STATUS" == true ]]; then
|
||
echo ""
|
||
echo "━━━━━ $ICON_SUMMARY RESOURCE MANAGER STATUS ━━━━━"
|
||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||
echo ""
|
||
echo "── Current State ──"
|
||
echo " Action level: $CURRENT_LEVEL (${LEVEL_NAMES[$CURRENT_LEVEL]:-unknown})"
|
||
echo " Target level: $TARGET_LEVEL (${LEVEL_NAMES[$TARGET_LEVEL]:-unknown})"
|
||
echo " Recover cycles: $RECOVER_CYCLES / ${RW_RECOVER_CYCLES:-3}"
|
||
[[ -n "$PAUSED_LIST" ]] && echo " Paused: $PAUSED_LIST"
|
||
[[ -n "$STOPPED_LIST" ]] && echo " Stopped: $STOPPED_LIST"
|
||
MEM_SHUTDOWN_ACTIVE=$(rm_state_get_eq "mem_shutdown_active")
|
||
[[ "$MEM_SHUTDOWN_ACTIVE" == "true" ]] && warn " docker_watchdog DEFERRED (mem_shutdown_active=true)"
|
||
echo ""
|
||
echo "── System Pressure ──"
|
||
echo " RAM free: ${MEM_GB}GB (soft:<${RW_RAM_SOFT_GB} medium:<${RW_RAM_MEDIUM_GB} hard:<${RW_RAM_HARD_GB} recover:>=${RW_RAM_RECOVER_GB})"
|
||
echo " Load avg: ${LOAD} (soft:>=${RW_LOAD_SOFT_THRESH} medium:>=${RW_LOAD_MEDIUM_THRESH} cores:${TOTAL_CORES})"
|
||
echo ""
|
||
echo "── Configuration ──"
|
||
echo " SABnzbd throttle: ${RW_SABNZBD_ENABLED:-true} soft=${RW_SABNZBD_SPEED_SOFT} medium=${RW_SABNZBD_SPEED_MEDIUM}"
|
||
echo " qBit throttle: ${RW_QBIT_ENABLED:-true} soft=${RW_QBIT_DL_SOFT}KB/s medium=${RW_QBIT_DL_MEDIUM}KB/s"
|
||
echo ""
|
||
echo "── Container Lists (this host) ──"
|
||
echo " Pause at medium: ${RW_PAUSE_CONTAINERS[*]:-none configured}"
|
||
echo " Stop at hard: ${RW_STOP_CONTAINERS[*]:-none configured}"
|
||
echo " Critical (never touched): ${RW_CRITICAL_CONTAINERS[*]:-none}"
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||
exit 0
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Critical Container Guard ━━━
|
||
# ==============================================================================================
|
||
# Returns 0 if container is safe to pause/stop, 1 if it is critical
|
||
is_critical() {
|
||
local container="$1"
|
||
for c in "${RW_CRITICAL_CONTAINERS[@]:-}"; do
|
||
[[ "$c" == "$container" ]] && return 1
|
||
done
|
||
return 0
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ━━━ SABnzbd API ━━━
|
||
# ==============================================================================================
|
||
sabnzbd_set_speed() {
|
||
local speed="$1"
|
||
[[ "${RW_SABNZBD_ENABLED:-true}" != "true" ]] && return 0
|
||
[[ -z "$SABNZBD_URL" || -z "$SABNZBD_API_KEY" ]] && return 0
|
||
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would set SABnzbd speed to $speed"
|
||
return 0
|
||
fi
|
||
|
||
curl -sf --max-time 10 \
|
||
"${SABNZBD_URL}/api?mode=config&name=speedlimit&value=${speed}&apikey=${SABNZBD_API_KEY}" \
|
||
>/dev/null 2>&1 && log "SABnzbd speed → $speed" || warn "SABnzbd API call failed"
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ━━━ qBittorrent API ━━━
|
||
# ==============================================================================================
|
||
QBIT_COOKIE="/tmp/rm_qbit_cookie.txt"
|
||
|
||
qbit_login() {
|
||
[[ "${RW_QBIT_ENABLED:-true}" != "true" ]] && return 0
|
||
[[ -z "$QBIT_URL" || -z "$QBIT_USERNAME" || -z "$QBIT_PASSWORD" ]] && return 0
|
||
|
||
curl -sf --max-time 10 -c "$QBIT_COOKIE" \
|
||
-X POST "${QBIT_URL}/api/v2/auth/login" \
|
||
-d "username=${QBIT_USERNAME}&password=${QBIT_PASSWORD}" >/dev/null 2>&1
|
||
}
|
||
|
||
qbit_set_dl_limit() {
|
||
local kbps="$1" # KB/s — 0 = unlimited
|
||
[[ "${RW_QBIT_ENABLED:-true}" != "true" ]] && return 0
|
||
[[ -z "$QBIT_URL" ]] && return 0
|
||
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would set qBit download limit to ${kbps}KB/s"
|
||
return 0
|
||
fi
|
||
|
||
local bps=$(( kbps * 1024 ))
|
||
qbit_login
|
||
curl -sf --max-time 10 -b "$QBIT_COOKIE" \
|
||
-X POST "${QBIT_URL}/api/v2/transfer/setDownloadLimit" \
|
||
-d "limit=${bps}" >/dev/null 2>&1 && log "qBit download limit → ${kbps}KB/s" || warn "qBit API call failed"
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Container Actions ━━━
|
||
# ==============================================================================================
|
||
|
||
# Pause a list of containers — returns newline-separated list of actually-paused containers
|
||
pause_containers() {
|
||
local actually_paused=()
|
||
for container in "$@"; do
|
||
[[ -z "$container" ]] && continue
|
||
is_critical "$container" || { log "$container — critical, skipping pause"; continue; }
|
||
|
||
local status
|
||
status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Status}}' "$container" 2>/dev/null)
|
||
if [[ "$status" != "running" ]]; then
|
||
log "$container — not running (status: ${status:-unknown}), skipping pause"
|
||
continue
|
||
fi
|
||
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would docker pause $container"
|
||
actually_paused+=("$container")
|
||
continue
|
||
fi
|
||
|
||
if timeout "$DOCKER_TIMEOUT" docker pause "$container" >/dev/null 2>&1; then
|
||
warn "Paused $container (medium pressure)"
|
||
actually_paused+=("$container")
|
||
else
|
||
error "Failed to pause $container"
|
||
fi
|
||
done
|
||
printf '%s,' "${actually_paused[@]}" | sed 's/,$//'
|
||
}
|
||
|
||
# Unpause a comma-separated list of containers
|
||
unpause_containers() {
|
||
local IFS=','
|
||
for container in $1; do
|
||
[[ -z "$container" ]] && continue
|
||
|
||
local status
|
||
status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Status}}' "$container" 2>/dev/null)
|
||
if [[ "$status" != "paused" ]]; then
|
||
log "$container — not paused (status: ${status:-unknown}), skipping unpause"
|
||
continue
|
||
fi
|
||
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would docker unpause $container"
|
||
continue
|
||
fi
|
||
|
||
if timeout "$DOCKER_TIMEOUT" docker unpause "$container" >/dev/null 2>&1; then
|
||
warn "Unpaused $container (pressure reduced)"
|
||
else
|
||
error "Failed to unpause $container"
|
||
fi
|
||
done
|
||
}
|
||
|
||
# Stop a list of containers — returns comma-separated list of actually-stopped containers
|
||
# Named rw_ to avoid colliding with common.sh's stop_containers() (different signature/semantics)
|
||
rw_stop_containers() {
|
||
local actually_stopped=()
|
||
for container in "$@"; do
|
||
[[ -z "$container" ]] && continue
|
||
is_critical "$container" || { log "$container — critical, skipping stop"; continue; }
|
||
|
||
local status
|
||
status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Status}}' "$container" 2>/dev/null)
|
||
if [[ "$status" != "running" && "$status" != "paused" ]]; then
|
||
log "$container — not running (status: ${status:-unknown}), skipping stop"
|
||
continue
|
||
fi
|
||
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would docker stop $container"
|
||
actually_stopped+=("$container")
|
||
continue
|
||
fi
|
||
|
||
if timeout "$DOCKER_TIMEOUT" docker stop "$container" >/dev/null 2>&1; then
|
||
warn "Stopped $container (hard pressure)"
|
||
actually_stopped+=("$container")
|
||
_RW_TRAP_STOPPED+=("$container")
|
||
else
|
||
error "Failed to stop $container"
|
||
fi
|
||
done
|
||
printf '%s,' "${actually_stopped[@]}" | sed 's/,$//'
|
||
}
|
||
|
||
# Start a comma-separated list of containers (only those RM stopped)
|
||
# Named rw_ to avoid colliding with common.sh's start_containers() (different signature/semantics)
|
||
rw_start_containers() {
|
||
local IFS=','
|
||
for container in $1; do
|
||
[[ -z "$container" ]] && continue
|
||
|
||
local status
|
||
status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Status}}' "$container" 2>/dev/null)
|
||
if [[ "$status" == "running" ]]; then
|
||
log "$container — already running"
|
||
continue
|
||
fi
|
||
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would docker start $container"
|
||
continue
|
||
fi
|
||
|
||
if timeout "$DOCKER_TIMEOUT" docker start "$container" >/dev/null 2>&1; then
|
||
warn "Started $container (pressure cleared)"
|
||
else
|
||
error "Failed to start $container"
|
||
fi
|
||
done
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Apply Level Actions ━━━
|
||
# ==============================================================================================
|
||
|
||
apply_level_1() {
|
||
log "Applying level 1 (soft) — throttling downloaders"
|
||
sabnzbd_set_speed "${RW_SABNZBD_SPEED_SOFT:-50M}"
|
||
qbit_set_dl_limit "${RW_QBIT_DL_SOFT:-51200}"
|
||
}
|
||
|
||
apply_level_2() {
|
||
log "Applying level 2 (medium) — throttling + pausing background containers"
|
||
sabnzbd_set_speed "${RW_SABNZBD_SPEED_MEDIUM:-10M}"
|
||
qbit_set_dl_limit "${RW_QBIT_DL_MEDIUM:-10240}"
|
||
|
||
if [[ ${#RW_PAUSE_CONTAINERS[@]} -gt 0 ]]; then
|
||
local newly_paused
|
||
newly_paused=$(pause_containers "${RW_PAUSE_CONTAINERS[@]}")
|
||
# Merge with existing paused list (avoid duplicates on re-escalation)
|
||
if [[ -n "$newly_paused" ]]; then
|
||
if [[ -n "$PAUSED_LIST" ]]; then
|
||
PAUSED_LIST="${PAUSED_LIST},${newly_paused}"
|
||
else
|
||
PAUSED_LIST="$newly_paused"
|
||
fi
|
||
fi
|
||
fi
|
||
}
|
||
|
||
apply_level_3() {
|
||
log "Applying level 3 (hard) — stopping optional containers"
|
||
sabnzbd_set_speed "${RW_SABNZBD_SPEED_MEDIUM:-10M}" # already at medium from level 2
|
||
qbit_set_dl_limit "${RW_QBIT_DL_MEDIUM:-10240}"
|
||
|
||
if [[ ${#RW_STOP_CONTAINERS[@]} -gt 0 ]]; then
|
||
local newly_stopped
|
||
newly_stopped=$(rw_stop_containers "${RW_STOP_CONTAINERS[@]}")
|
||
if [[ -n "$newly_stopped" ]]; then
|
||
if [[ -n "$STOPPED_LIST" ]]; then
|
||
STOPPED_LIST="${STOPPED_LIST},${newly_stopped}"
|
||
else
|
||
STOPPED_LIST="$newly_stopped"
|
||
fi
|
||
fi
|
||
fi
|
||
|
||
# Signal docker_watchdog to defer container restarts
|
||
rm_state_set_eq "mem_shutdown_active" "true"
|
||
warn "mem_shutdown_active=true — docker_watchdog will defer restarts"
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Restore Level Actions ━━━
|
||
# ==============================================================================================
|
||
|
||
restore_level_3() {
|
||
echo "Restoring from level 3 — starting stopped containers"
|
||
if [[ -n "$STOPPED_LIST" ]]; then
|
||
rw_start_containers "$STOPPED_LIST"
|
||
STOPPED_LIST=""
|
||
fi
|
||
rm_state_set_eq "mem_shutdown_active" "false"
|
||
warn "mem_shutdown_active=false — docker_watchdog restoring normal operation"
|
||
}
|
||
|
||
restore_level_2() {
|
||
echo "Restoring from level 2 — unpausing containers"
|
||
if [[ -n "$PAUSED_LIST" ]]; then
|
||
unpause_containers "$PAUSED_LIST"
|
||
PAUSED_LIST=""
|
||
fi
|
||
}
|
||
|
||
restore_level_1() {
|
||
echo "Restoring from level 1 — removing downloader throttle"
|
||
sabnzbd_set_speed "0"
|
||
qbit_set_dl_limit 0
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Pressure Decision ━━━
|
||
# ==============================================================================================
|
||
echo ""
|
||
echo "━━━ $ICON_GEAR Resource Manager — $MY_ID — $(date '+%H:%M:%S') ━━━"
|
||
echo " RAM: ${MEM_GB}GB free Load: ${LOAD} Action: ${CURRENT_LEVEL} → ${TARGET_LEVEL} (${LEVEL_NAMES[$TARGET_LEVEL]:-unknown})"
|
||
|
||
if [[ "$TARGET_LEVEL" -gt "$CURRENT_LEVEL" ]]; then
|
||
# ── Escalate ────────────────────────────────────────────────────────────────────────────
|
||
warn "Pressure escalating to level $TARGET_LEVEL — $TARGET_REASON"
|
||
notify "Resource Manager: pressure level $TARGET_LEVEL on $(hostname) ($MY_ID) — $TARGET_REASON" \
|
||
"Resource Manager" "warning"
|
||
|
||
for (( lvl = CURRENT_LEVEL + 1; lvl <= TARGET_LEVEL; lvl++ )); do
|
||
case "$lvl" in
|
||
1) apply_level_1 ;;
|
||
2) apply_level_2 ;;
|
||
3) apply_level_3 ;;
|
||
esac
|
||
done
|
||
|
||
rm_state_set "rm_action_level" "$TARGET_LEVEL"
|
||
rm_state_set "rm_recover_cycles" 0
|
||
|
||
elif [[ "$TARGET_LEVEL" -lt "$CURRENT_LEVEL" ]]; then
|
||
# ── Tracking recovery ───────────────────────────────────────────────────────────────────
|
||
RECOVER_CYCLES=$(( RECOVER_CYCLES + 1 ))
|
||
rm_state_set "rm_recover_cycles" "$RECOVER_CYCLES"
|
||
log "Pressure at level $TARGET_LEVEL — recovery cycle $RECOVER_CYCLES/${RW_RECOVER_CYCLES:-3} before restoring level $CURRENT_LEVEL actions"
|
||
|
||
if [[ "$RECOVER_CYCLES" -ge "${RW_RECOVER_CYCLES:-3}" ]]; then
|
||
# Level 3 de-escalation requires RAM above recover threshold
|
||
if [[ "$CURRENT_LEVEL" -ge 3 && "$MEM_GB" -lt "${RW_RAM_RECOVER_GB:-20}" ]]; then
|
||
warn "Level 3 restore blocked — RAM ${MEM_GB}GB still below recover threshold ${RW_RAM_RECOVER_GB}GB"
|
||
else
|
||
warn "Pressure sustained below level $CURRENT_LEVEL — restoring"
|
||
case "$CURRENT_LEVEL" in
|
||
3) restore_level_3 ;;
|
||
2) restore_level_2 ;;
|
||
1) restore_level_1 ;;
|
||
esac
|
||
|
||
NEW_LEVEL=$(( CURRENT_LEVEL - 1 ))
|
||
rm_state_set "rm_action_level" "$NEW_LEVEL"
|
||
rm_state_set "rm_recover_cycles" 0
|
||
rm_state_set "rm_paused_containers" "$PAUSED_LIST"
|
||
rm_state_set "rm_stopped_containers" "$STOPPED_LIST"
|
||
|
||
if [[ "$NEW_LEVEL" -gt 0 ]]; then
|
||
warn "De-escalated to level $NEW_LEVEL (${LEVEL_NAMES[$NEW_LEVEL]}) — ${RW_RECOVER_CYCLES:-3} more cycles to fully clear"
|
||
else
|
||
echo "All pressure cleared — system at normal operation ✅"
|
||
notify "Resource Manager: pressure resolved on $(hostname) ($MY_ID) — system back to normal" \
|
||
"Resource Manager" "normal"
|
||
fi
|
||
fi
|
||
fi
|
||
|
||
else
|
||
# ── Steady state ────────────────────────────────────────────────────────────────────────
|
||
if [[ "$CURRENT_LEVEL" -gt 0 ]]; then
|
||
echo "Pressure holding at level $CURRENT_LEVEL — waiting for sustained recovery"
|
||
else
|
||
log "System normal — RAM ${MEM_GB}GB free | load ${LOAD} | ${TOTAL_CORES} cores | SABnzbd=${RW_SABNZBD_ENABLED:-true} qBit=${RW_QBIT_ENABLED:-true} ✅"
|
||
fi
|
||
rm_state_set "rm_recover_cycles" 0
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Persist State ━━━
|
||
# ==============================================================================================
|
||
rm_state_set "rm_paused_containers" "$PAUSED_LIST"
|
||
rm_state_set "rm_stopped_containers" "$STOPPED_LIST"
|
||
trap "_release_all_locks" EXIT # state persisted — disable restart trap, keep lock cleanup
|
||
# Touch state file each run so docker_watchdog stale guard sees fresh mtime
|
||
touch "$RW_STATE_FILE" 2>/dev/null
|