refactor: rename resource_manager → resource_watchdog, RM_ → RW_
Consistent naming with the watchdog family (system_watchdog, docker_watchdog). File renamed, all RM_ config variables and internal references updated to RW_ across master.conf, master_host1.conf, common.sh, docker_watchdog.sh, system_watchdog.sh, watchdog_orchestrator.sh, and sunday_morning_coffee_report.sh. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 4.6
parent
309546e615
commit
9a5f6f88f2
@@ -13,25 +13,25 @@
|
||||
#
|
||||
# ── THREE-LEVEL PRESSURE RESPONSE ─────────────────────────────────────────────────────────────
|
||||
#
|
||||
# Level 1 — SOFT (RAM < RM_RAM_SOFT_GB OR load > RM_LOAD_SOFT_MULTIPLIER × cores):
|
||||
# Throttle SABnzbd download speed to RM_SABNZBD_SPEED_SOFT
|
||||
# Throttle qBittorrent download to RM_QBIT_DL_SOFT KB/s
|
||||
# Level 1 — SOFT (RAM < RW_RAM_SOFT_GB OR load > RW_LOAD_SOFT_MULTIPLIER × cores):
|
||||
# Throttle SABnzbd download speed to RW_SABNZBD_SPEED_SOFT
|
||||
# Throttle qBittorrent download to RW_QBIT_DL_SOFT KB/s
|
||||
#
|
||||
# Level 2 — MEDIUM (RAM < RM_RAM_MEDIUM_GB OR load > RM_LOAD_MEDIUM_MULTIPLIER × cores):
|
||||
# Level 2 — MEDIUM (RAM < RW_RAM_MEDIUM_GB OR load > RW_LOAD_MEDIUM_MULTIPLIER × cores):
|
||||
# Further throttle SABnzbd + qBittorrent to medium limits
|
||||
# docker pause RM_PAUSE_CONTAINERS — suspend without losing state, instant reversible
|
||||
# docker pause RW_PAUSE_CONTAINERS — suspend without losing state, instant reversible
|
||||
#
|
||||
# Level 3 — HARD (RAM < RM_RAM_HARD_GB):
|
||||
# docker stop RM_STOP_CONTAINERS — optional/heavy services (games, LocalAI, etc.)
|
||||
# Level 3 — HARD (RAM < RW_RAM_HARD_GB):
|
||||
# docker stop RW_STOP_CONTAINERS — optional/heavy services (games, LocalAI, etc.)
|
||||
# Write mem_shutdown_active=true — signals docker_watchdog to defer container restarts
|
||||
#
|
||||
# ── RECOVERY ──────────────────────────────────────────────────────────────────────────────────
|
||||
# Pressure must stay below current action threshold for RM_RECOVER_CYCLES consecutive runs
|
||||
# Pressure must stay below current action threshold for RW_RECOVER_CYCLES consecutive runs
|
||||
# before restoring. De-escalates one level at a time to avoid re-triggering immediately.
|
||||
# Level 3 de-escalation additionally requires RAM >= RM_RAM_RECOVER_GB before un-stopping.
|
||||
# Level 3 de-escalation additionally requires RAM >= RW_RAM_RECOVER_GB before un-stopping.
|
||||
#
|
||||
# ── COORDINATION WITH DOCKER WATCHDOG ─────────────────────────────────────────────────────────
|
||||
# At level 3: writes mem_shutdown_active=true to RM_STATE_FILE.
|
||||
# At level 3: writes mem_shutdown_active=true to RW_STATE_FILE.
|
||||
# docker_watchdog.sh reads this and defers all container restart logic.
|
||||
# Cleared when pressure fully resolves and containers are restarted.
|
||||
# This prevents docker_watchdog from restarting containers that RM just stopped to free RAM.
|
||||
@@ -39,28 +39,28 @@
|
||||
# ── NOT RESPONSIBLE FOR ───────────────────────────────────────────────────────────────────────
|
||||
# Restarting broken containers — docker_watchdog.sh
|
||||
# Rebooting the system — system_watchdog.sh
|
||||
# Reacting to single data points — RM_RECOVER_CYCLES prevents flip-flopping
|
||||
# Reacting to single data points — RW_RECOVER_CYCLES prevents flip-flopping
|
||||
#
|
||||
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||||
# RM_ENABLED, RM_STATE_FILE
|
||||
# RM_RAM_SOFT_GB, RM_RAM_MEDIUM_GB, RM_RAM_HARD_GB, RM_RAM_RECOVER_GB
|
||||
# RM_LOAD_SOFT_MULTIPLIER, RM_LOAD_MEDIUM_MULTIPLIER
|
||||
# RM_RECOVER_CYCLES
|
||||
# RM_SABNZBD_ENABLED, RM_SABNZBD_SPEED_SOFT, RM_SABNZBD_SPEED_MEDIUM
|
||||
# RM_QBIT_ENABLED, RM_QBIT_DL_SOFT, RM_QBIT_DL_MEDIUM
|
||||
# RM_CRITICAL_CONTAINERS — never paused or stopped regardless of pressure
|
||||
# RW_ENABLED, RW_STATE_FILE
|
||||
# RW_RAM_SOFT_GB, RW_RAM_MEDIUM_GB, RW_RAM_HARD_GB, RW_RAM_RECOVER_GB
|
||||
# RW_LOAD_SOFT_MULTIPLIER, RW_LOAD_MEDIUM_MULTIPLIER
|
||||
# RW_RECOVER_CYCLES
|
||||
# RW_SABNZBD_ENABLED, RW_SABNZBD_SPEED_SOFT, RW_SABNZBD_SPEED_MEDIUM
|
||||
# RW_QBIT_ENABLED, RW_QBIT_DL_SOFT, RW_QBIT_DL_MEDIUM
|
||||
# RW_CRITICAL_CONTAINERS — never paused or stopped regardless of pressure
|
||||
#
|
||||
# ── CONFIGURATION (master_host*.conf) ─────────────────────────────────────────────────────────
|
||||
# HOST*_RM_PAUSE_CONTAINERS — docker pause at medium pressure (aliased by detect_hosts)
|
||||
# HOST*_RM_STOP_CONTAINERS — docker stop at hard pressure (aliased by detect_hosts)
|
||||
# HOST*_RW_PAUSE_CONTAINERS — docker pause at medium pressure (aliased by detect_hosts)
|
||||
# HOST*_RW_STOP_CONTAINERS — docker stop at hard pressure (aliased by detect_hosts)
|
||||
# HOST*_SABNZBD_URL, HOST*_SABNZBD_API_KEY
|
||||
# HOST*_QBIT_URL, HOST*_QBIT_USERNAME, HOST*_QBIT_PASSWORD
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# resource_manager.sh — normal run (via watchdog_orchestrator.sh)
|
||||
# resource_manager.sh --dry-run — show what would happen without acting
|
||||
# resource_manager.sh --status — current pressure level and active actions
|
||||
# resource_manager.sh --log — verbose per-check output
|
||||
# resource_watchdog.sh — normal run (via watchdog_orchestrator.sh)
|
||||
# resource_watchdog.sh --dry-run — show what would happen without acting
|
||||
# resource_watchdog.sh --status — current pressure level and active actions
|
||||
# resource_watchdog.sh --log — verbose per-check output
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
@@ -77,8 +77,8 @@ if [[ "$EUID" -ne 0 ]]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ "${RM_ENABLED:-true}" != "true" ]]; then
|
||||
log "Resource Manager disabled (RM_ENABLED=false)"
|
||||
if [[ "${RW_ENABLED:-true}" != "true" ]]; then
|
||||
log "Resource Manager disabled (RW_ENABLED=false)"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
@@ -88,8 +88,8 @@ detect_hosts
|
||||
|
||||
DOCKER_TIMEOUT=15
|
||||
|
||||
touch "$RM_STATE_FILE" 2>/dev/null || {
|
||||
error "Cannot create state file: $RM_STATE_FILE"
|
||||
touch "$RW_STATE_FILE" 2>/dev/null || {
|
||||
error "Cannot create state file: $RW_STATE_FILE"
|
||||
exit 1
|
||||
}
|
||||
|
||||
@@ -100,25 +100,25 @@ touch "$RM_STATE_FILE" 2>/dev/null || {
|
||||
# rm_state_get_eq/set_eq use = separator for docker_watchdog coordination flags
|
||||
|
||||
rm_state_get() {
|
||||
grep -E "^${1}:" "$RM_STATE_FILE" 2>/dev/null | cut -d: -f2-
|
||||
grep -E "^${1}:" "$RW_STATE_FILE" 2>/dev/null | cut -d: -f2-
|
||||
}
|
||||
|
||||
rm_state_set() {
|
||||
local key="$1" val="$2"
|
||||
grep -vE "^${key}:" "$RM_STATE_FILE" 2>/dev/null > "${RM_STATE_FILE}.tmp"
|
||||
echo "${key}:${val}" >> "${RM_STATE_FILE}.tmp"
|
||||
mv "${RM_STATE_FILE}.tmp" "$RM_STATE_FILE"
|
||||
grep -vE "^${key}:" "$RW_STATE_FILE" 2>/dev/null > "${RW_STATE_FILE}.tmp"
|
||||
echo "${key}:${val}" >> "${RW_STATE_FILE}.tmp"
|
||||
mv "${RW_STATE_FILE}.tmp" "$RW_STATE_FILE"
|
||||
}
|
||||
|
||||
rm_state_get_eq() {
|
||||
grep -E "^${1}=" "$RM_STATE_FILE" 2>/dev/null | cut -d= -f2-
|
||||
grep -E "^${1}=" "$RW_STATE_FILE" 2>/dev/null | cut -d= -f2-
|
||||
}
|
||||
|
||||
rm_state_set_eq() {
|
||||
local key="$1" val="$2"
|
||||
grep -vE "^${key}=" "$RM_STATE_FILE" 2>/dev/null > "${RM_STATE_FILE}.tmp"
|
||||
echo "${key}=${val}" >> "${RM_STATE_FILE}.tmp"
|
||||
mv "${RM_STATE_FILE}.tmp" "$RM_STATE_FILE"
|
||||
grep -vE "^${key}=" "$RW_STATE_FILE" 2>/dev/null > "${RW_STATE_FILE}.tmp"
|
||||
echo "${key}=${val}" >> "${RW_STATE_FILE}.tmp"
|
||||
mv "${RW_STATE_FILE}.tmp" "$RW_STATE_FILE"
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
@@ -137,22 +137,22 @@ MEM_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo)
|
||||
MEM_GB=$(( MEM_KB / 1024 / 1024 ))
|
||||
LOAD=$(awk '{print $1}' /proc/loadavg)
|
||||
LOAD_INT=$(printf "%.0f" "$LOAD")
|
||||
RM_LOAD_SOFT_THRESH=$(awk "BEGIN{printf \"%.0f\", $TOTAL_CORES * ${RM_LOAD_SOFT_MULTIPLIER:-2.0}}")
|
||||
RM_LOAD_MEDIUM_THRESH=$(awk "BEGIN{printf \"%.0f\", $TOTAL_CORES * ${RM_LOAD_MEDIUM_MULTIPLIER:-3.0}}")
|
||||
RW_LOAD_SOFT_THRESH=$(awk "BEGIN{printf \"%.0f\", $TOTAL_CORES * ${RW_LOAD_SOFT_MULTIPLIER:-2.0}}")
|
||||
RW_LOAD_MEDIUM_THRESH=$(awk "BEGIN{printf \"%.0f\", $TOTAL_CORES * ${RW_LOAD_MEDIUM_MULTIPLIER:-3.0}}")
|
||||
|
||||
TARGET_LEVEL=0
|
||||
TARGET_REASON=""
|
||||
if [[ "$MEM_GB" -lt "${RM_RAM_HARD_GB:-6}" ]]; then
|
||||
if [[ "$MEM_GB" -lt "${RW_RAM_HARD_GB:-6}" ]]; then
|
||||
TARGET_LEVEL=3
|
||||
TARGET_REASON="RAM ${MEM_GB}GB < hard threshold ${RM_RAM_HARD_GB}GB"
|
||||
elif [[ "$MEM_GB" -lt "${RM_RAM_MEDIUM_GB:-8}" ]] || [[ "$LOAD_INT" -ge "$RM_LOAD_MEDIUM_THRESH" ]]; then
|
||||
TARGET_REASON="RAM ${MEM_GB}GB < hard threshold ${RW_RAM_HARD_GB}GB"
|
||||
elif [[ "$MEM_GB" -lt "${RW_RAM_MEDIUM_GB:-8}" ]] || [[ "$LOAD_INT" -ge "$RW_LOAD_MEDIUM_THRESH" ]]; then
|
||||
TARGET_LEVEL=2
|
||||
[[ "$MEM_GB" -lt "${RM_RAM_MEDIUM_GB:-8}" ]] && TARGET_REASON="RAM ${MEM_GB}GB < medium threshold ${RM_RAM_MEDIUM_GB}GB"
|
||||
[[ "$LOAD_INT" -ge "$RM_LOAD_MEDIUM_THRESH" ]] && TARGET_REASON="${TARGET_REASON:+$TARGET_REASON, }load ${LOAD} >= medium threshold ${RM_LOAD_MEDIUM_THRESH}"
|
||||
elif [[ "$MEM_GB" -lt "${RM_RAM_SOFT_GB:-12}" ]] || [[ "$LOAD_INT" -ge "$RM_LOAD_SOFT_THRESH" ]]; then
|
||||
[[ "$MEM_GB" -lt "${RW_RAM_MEDIUM_GB:-8}" ]] && TARGET_REASON="RAM ${MEM_GB}GB < medium threshold ${RW_RAM_MEDIUM_GB}GB"
|
||||
[[ "$LOAD_INT" -ge "$RW_LOAD_MEDIUM_THRESH" ]] && TARGET_REASON="${TARGET_REASON:+$TARGET_REASON, }load ${LOAD} >= medium threshold ${RW_LOAD_MEDIUM_THRESH}"
|
||||
elif [[ "$MEM_GB" -lt "${RW_RAM_SOFT_GB:-12}" ]] || [[ "$LOAD_INT" -ge "$RW_LOAD_SOFT_THRESH" ]]; then
|
||||
TARGET_LEVEL=1
|
||||
[[ "$MEM_GB" -lt "${RM_RAM_SOFT_GB:-12}" ]] && TARGET_REASON="RAM ${MEM_GB}GB < soft threshold ${RM_RAM_SOFT_GB}GB"
|
||||
[[ "$LOAD_INT" -ge "$RM_LOAD_SOFT_THRESH" ]] && TARGET_REASON="${TARGET_REASON:+$TARGET_REASON, }load ${LOAD} >= soft threshold ${RM_LOAD_SOFT_THRESH}"
|
||||
[[ "$MEM_GB" -lt "${RW_RAM_SOFT_GB:-12}" ]] && TARGET_REASON="RAM ${MEM_GB}GB < soft threshold ${RW_RAM_SOFT_GB}GB"
|
||||
[[ "$LOAD_INT" -ge "$RW_LOAD_SOFT_THRESH" ]] && TARGET_REASON="${TARGET_REASON:+$TARGET_REASON, }load ${LOAD} >= soft threshold ${RW_LOAD_SOFT_THRESH}"
|
||||
fi
|
||||
|
||||
LEVEL_NAMES=("normal" "soft" "medium" "hard")
|
||||
@@ -168,24 +168,24 @@ if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo "── Current State ──"
|
||||
echo " Action level: $CURRENT_LEVEL (${LEVEL_NAMES[$CURRENT_LEVEL]:-unknown})"
|
||||
echo " Target level: $TARGET_LEVEL (${LEVEL_NAMES[$TARGET_LEVEL]:-unknown})"
|
||||
echo " Recover cycles: $RECOVER_CYCLES / ${RM_RECOVER_CYCLES:-3}"
|
||||
echo " Recover cycles: $RECOVER_CYCLES / ${RW_RECOVER_CYCLES:-3}"
|
||||
[[ -n "$PAUSED_LIST" ]] && echo " Paused: $PAUSED_LIST"
|
||||
[[ -n "$STOPPED_LIST" ]] && echo " Stopped: $STOPPED_LIST"
|
||||
MEM_SHUTDOWN_ACTIVE=$(rm_state_get_eq "mem_shutdown_active")
|
||||
[[ "$MEM_SHUTDOWN_ACTIVE" == "true" ]] && warn " docker_watchdog DEFERRED (mem_shutdown_active=true)"
|
||||
echo ""
|
||||
echo "── System Pressure ──"
|
||||
echo " RAM free: ${MEM_GB}GB (soft:<${RM_RAM_SOFT_GB} medium:<${RM_RAM_MEDIUM_GB} hard:<${RM_RAM_HARD_GB} recover:>=${RM_RAM_RECOVER_GB})"
|
||||
echo " Load avg: ${LOAD} (soft:>=${RM_LOAD_SOFT_THRESH} medium:>=${RM_LOAD_MEDIUM_THRESH} cores:${TOTAL_CORES})"
|
||||
echo " RAM free: ${MEM_GB}GB (soft:<${RW_RAM_SOFT_GB} medium:<${RW_RAM_MEDIUM_GB} hard:<${RW_RAM_HARD_GB} recover:>=${RW_RAM_RECOVER_GB})"
|
||||
echo " Load avg: ${LOAD} (soft:>=${RW_LOAD_SOFT_THRESH} medium:>=${RW_LOAD_MEDIUM_THRESH} cores:${TOTAL_CORES})"
|
||||
echo ""
|
||||
echo "── Configuration ──"
|
||||
echo " SABnzbd throttle: ${RM_SABNZBD_ENABLED:-true} soft=${RM_SABNZBD_SPEED_SOFT} medium=${RM_SABNZBD_SPEED_MEDIUM}"
|
||||
echo " qBit throttle: ${RM_QBIT_ENABLED:-true} soft=${RM_QBIT_DL_SOFT}KB/s medium=${RM_QBIT_DL_MEDIUM}KB/s"
|
||||
echo " SABnzbd throttle: ${RW_SABNZBD_ENABLED:-true} soft=${RW_SABNZBD_SPEED_SOFT} medium=${RW_SABNZBD_SPEED_MEDIUM}"
|
||||
echo " qBit throttle: ${RW_QBIT_ENABLED:-true} soft=${RW_QBIT_DL_SOFT}KB/s medium=${RW_QBIT_DL_MEDIUM}KB/s"
|
||||
echo ""
|
||||
echo "── Container Lists (this host) ──"
|
||||
echo " Pause at medium: ${RM_PAUSE_CONTAINERS[*]:-none configured}"
|
||||
echo " Stop at hard: ${RM_STOP_CONTAINERS[*]:-none configured}"
|
||||
echo " Critical (never touched): ${RM_CRITICAL_CONTAINERS[*]:-none}"
|
||||
echo " Pause at medium: ${RW_PAUSE_CONTAINERS[*]:-none configured}"
|
||||
echo " Stop at hard: ${RW_STOP_CONTAINERS[*]:-none configured}"
|
||||
echo " Critical (never touched): ${RW_CRITICAL_CONTAINERS[*]:-none}"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
@@ -196,7 +196,7 @@ fi
|
||||
# Returns 0 if container is safe to pause/stop, 1 if it is critical
|
||||
is_critical() {
|
||||
local container="$1"
|
||||
for c in "${RM_CRITICAL_CONTAINERS[@]:-}"; do
|
||||
for c in "${RW_CRITICAL_CONTAINERS[@]:-}"; do
|
||||
[[ "$c" == "$container" ]] && return 1
|
||||
done
|
||||
return 0
|
||||
@@ -207,7 +207,7 @@ is_critical() {
|
||||
# ==============================================================================================
|
||||
sabnzbd_set_speed() {
|
||||
local speed="$1"
|
||||
[[ "${RM_SABNZBD_ENABLED:-true}" != "true" ]] && return 0
|
||||
[[ "${RW_SABNZBD_ENABLED:-true}" != "true" ]] && return 0
|
||||
[[ -z "$SABNZBD_URL" || -z "$SABNZBD_API_KEY" ]] && return 0
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
@@ -226,7 +226,7 @@ sabnzbd_set_speed() {
|
||||
QBIT_COOKIE="/tmp/rm_qbit_cookie.txt"
|
||||
|
||||
qbit_login() {
|
||||
[[ "${RM_QBIT_ENABLED:-true}" != "true" ]] && return 0
|
||||
[[ "${RW_QBIT_ENABLED:-true}" != "true" ]] && return 0
|
||||
[[ -z "$QBIT_URL" || -z "$QBIT_USERNAME" || -z "$QBIT_PASSWORD" ]] && return 0
|
||||
|
||||
curl -sf --max-time 10 -c "$QBIT_COOKIE" \
|
||||
@@ -236,7 +236,7 @@ qbit_login() {
|
||||
|
||||
qbit_set_dl_limit() {
|
||||
local kbps="$1" # KB/s — 0 = unlimited
|
||||
[[ "${RM_QBIT_ENABLED:-true}" != "true" ]] && return 0
|
||||
[[ "${RW_QBIT_ENABLED:-true}" != "true" ]] && return 0
|
||||
[[ -z "$QBIT_URL" ]] && return 0
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
@@ -373,18 +373,18 @@ start_containers() {
|
||||
|
||||
apply_level_1() {
|
||||
log "Applying level 1 (soft) — throttling downloaders"
|
||||
sabnzbd_set_speed "${RM_SABNZBD_SPEED_SOFT:-50M}"
|
||||
qbit_set_dl_limit "${RM_QBIT_DL_SOFT:-51200}"
|
||||
sabnzbd_set_speed "${RW_SABNZBD_SPEED_SOFT:-50M}"
|
||||
qbit_set_dl_limit "${RW_QBIT_DL_SOFT:-51200}"
|
||||
}
|
||||
|
||||
apply_level_2() {
|
||||
log "Applying level 2 (medium) — throttling + pausing background containers"
|
||||
sabnzbd_set_speed "${RM_SABNZBD_SPEED_MEDIUM:-10M}"
|
||||
qbit_set_dl_limit "${RM_QBIT_DL_MEDIUM:-10240}"
|
||||
sabnzbd_set_speed "${RW_SABNZBD_SPEED_MEDIUM:-10M}"
|
||||
qbit_set_dl_limit "${RW_QBIT_DL_MEDIUM:-10240}"
|
||||
|
||||
if [[ ${#RM_PAUSE_CONTAINERS[@]} -gt 0 ]]; then
|
||||
if [[ ${#RW_PAUSE_CONTAINERS[@]} -gt 0 ]]; then
|
||||
local newly_paused
|
||||
newly_paused=$(pause_containers "${RM_PAUSE_CONTAINERS[@]}")
|
||||
newly_paused=$(pause_containers "${RW_PAUSE_CONTAINERS[@]}")
|
||||
# Merge with existing paused list (avoid duplicates on re-escalation)
|
||||
if [[ -n "$newly_paused" ]]; then
|
||||
if [[ -n "$PAUSED_LIST" ]]; then
|
||||
@@ -398,12 +398,12 @@ apply_level_2() {
|
||||
|
||||
apply_level_3() {
|
||||
log "Applying level 3 (hard) — stopping optional containers"
|
||||
sabnzbd_set_speed "${RM_SABNZBD_SPEED_MEDIUM:-10M}" # already at medium from level 2
|
||||
qbit_set_dl_limit "${RM_QBIT_DL_MEDIUM:-10240}"
|
||||
sabnzbd_set_speed "${RW_SABNZBD_SPEED_MEDIUM:-10M}" # already at medium from level 2
|
||||
qbit_set_dl_limit "${RW_QBIT_DL_MEDIUM:-10240}"
|
||||
|
||||
if [[ ${#RM_STOP_CONTAINERS[@]} -gt 0 ]]; then
|
||||
if [[ ${#RW_STOP_CONTAINERS[@]} -gt 0 ]]; then
|
||||
local newly_stopped
|
||||
newly_stopped=$(stop_containers "${RM_STOP_CONTAINERS[@]}")
|
||||
newly_stopped=$(stop_containers "${RW_STOP_CONTAINERS[@]}")
|
||||
if [[ -n "$newly_stopped" ]]; then
|
||||
if [[ -n "$STOPPED_LIST" ]]; then
|
||||
STOPPED_LIST="${STOPPED_LIST},${newly_stopped}"
|
||||
@@ -474,12 +474,12 @@ elif [[ "$TARGET_LEVEL" -lt "$CURRENT_LEVEL" ]]; then
|
||||
# ── Tracking recovery ───────────────────────────────────────────────────────────────────
|
||||
RECOVER_CYCLES=$(( RECOVER_CYCLES + 1 ))
|
||||
rm_state_set "rm_recover_cycles" "$RECOVER_CYCLES"
|
||||
log "Pressure at level $TARGET_LEVEL — recovery cycle $RECOVER_CYCLES/${RM_RECOVER_CYCLES:-3} before restoring level $CURRENT_LEVEL actions"
|
||||
log "Pressure at level $TARGET_LEVEL — recovery cycle $RECOVER_CYCLES/${RW_RECOVER_CYCLES:-3} before restoring level $CURRENT_LEVEL actions"
|
||||
|
||||
if [[ "$RECOVER_CYCLES" -ge "${RM_RECOVER_CYCLES:-3}" ]]; then
|
||||
if [[ "$RECOVER_CYCLES" -ge "${RW_RECOVER_CYCLES:-3}" ]]; then
|
||||
# Level 3 de-escalation requires RAM above recover threshold
|
||||
if [[ "$CURRENT_LEVEL" -ge 3 && "$MEM_GB" -lt "${RM_RAM_RECOVER_GB:-20}" ]]; then
|
||||
warn "Level 3 restore blocked — RAM ${MEM_GB}GB still below recover threshold ${RM_RAM_RECOVER_GB}GB"
|
||||
if [[ "$CURRENT_LEVEL" -ge 3 && "$MEM_GB" -lt "${RW_RAM_RECOVER_GB:-20}" ]]; then
|
||||
warn "Level 3 restore blocked — RAM ${MEM_GB}GB still below recover threshold ${RW_RAM_RECOVER_GB}GB"
|
||||
else
|
||||
warn "Pressure sustained below level $CURRENT_LEVEL — restoring"
|
||||
case "$CURRENT_LEVEL" in
|
||||
@@ -495,7 +495,7 @@ elif [[ "$TARGET_LEVEL" -lt "$CURRENT_LEVEL" ]]; then
|
||||
rm_state_set "rm_stopped_containers" "$STOPPED_LIST"
|
||||
|
||||
if [[ "$NEW_LEVEL" -gt 0 ]]; then
|
||||
warn "De-escalated to level $NEW_LEVEL (${LEVEL_NAMES[$NEW_LEVEL]}) — ${RM_RECOVER_CYCLES:-3} more cycles to fully clear"
|
||||
warn "De-escalated to level $NEW_LEVEL (${LEVEL_NAMES[$NEW_LEVEL]}) — ${RW_RECOVER_CYCLES:-3} more cycles to fully clear"
|
||||
else
|
||||
log "All pressure cleared — system at normal operation ✅"
|
||||
notify "Resource Manager: pressure resolved on $(hostname) ($MY_ID) — system back to normal" \
|
||||
@@ -520,4 +520,4 @@ fi
|
||||
rm_state_set "rm_paused_containers" "$PAUSED_LIST"
|
||||
rm_state_set "rm_stopped_containers" "$STOPPED_LIST"
|
||||
# Touch state file each run so docker_watchdog stale guard sees fresh mtime
|
||||
touch "$RM_STATE_FILE" 2>/dev/null
|
||||
touch "$RW_STATE_FILE" 2>/dev/null
|
||||
@@ -131,7 +131,7 @@ if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo "$ICON_GEAR /var/log warn: ${SYS_WATCHDOG_LOG_PCT}%"
|
||||
echo "$ICON_GEAR /tmp warn: ${SYS_WATCHDOG_TMP_PCT}%"
|
||||
echo "$ICON_MEM RAM reboot: < ${SYS_WATCHDOG_MEM_GB}GB (+ strikes)"
|
||||
echo " (RAM warn/shutdown/recover managed by resource_manager.sh)"
|
||||
echo " (RAM warn/shutdown/recover managed by resource_watchdog.sh)"
|
||||
echo "$ICON_ZFS ARC pinned: ${SYS_WATCHDOG_ARC_PINNED_PCT}%"
|
||||
echo "$ICON_GEAR Load multiplier: ${SYS_WATCHDOG_LOAD_MULTIPLIER}x (= $(( TOTAL_CORES * SYS_WATCHDOG_LOAD_MULTIPLIER )) on $TOTAL_CORES cores)"
|
||||
echo "$ICON_GEAR Zombie limit: ${SYS_WATCHDOG_ZOMBIE_LIMIT}"
|
||||
@@ -558,7 +558,7 @@ echo "━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
TRIGGERS+=("tmp=${TMP_USED}%")
|
||||
fi
|
||||
|
||||
# ── RAM — reboot tier only (warn/shutdown/recover handled by resource_manager.sh) ──────────
|
||||
# ── RAM — reboot tier only (warn/shutdown/recover handled by resource_watchdog.sh) ──────────
|
||||
if [[ "$SYS_WATCHDOG_CHECK_RAM" == true ]]; then
|
||||
MEM_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo)
|
||||
MEM_GB=$(( MEM_KB / 1024 / 1024 ))
|
||||
|
||||
Reference in New Issue
Block a user