Revert orchestrator + skip_list_manager to original folders
watchdog_orchestrator.sh stays in Orchestrators/ — it's an orchestrator, not a watchdog. watchdog_skip_list_manager.sh stays in Tools/ — it's a management utility. Only the 4 watchdog scripts belong in Watchdogs/: docker_watchdog.sh, resource_watchdog.sh, storage_watchdog.sh, system_watchdog.sh
This commit is contained in:
Gmer4Lfe
@@ -1,194 +0,0 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ============================ Watchdog Orchestrator ===========================================
|
||||
# ==============================================================================================
|
||||
# Runs WATCHDOG_ORCHESTRATOR_SCRIPTS in order each cron cycle.
|
||||
# Schedule: * * * * * (every minute via User Scripts plugin)
|
||||
#
|
||||
# ── EXECUTION ORDER ───────────────────────────────────────────────────────────────────────────
|
||||
# Driven by WATCHDOG_ORCHESTRATOR_SCRIPTS in master.conf — add, remove, or reorder there.
|
||||
# Default: resource_watchdog → docker_watchdog → system_watchdog
|
||||
#
|
||||
# ── WHY ORDER MATTERS ─────────────────────────────────────────────────────────────────────────
|
||||
# Resource Watchdog first — frees RAM and CPU before healing attempts container restarts.
|
||||
# Containers restarted into a resource-pressured system just fail again.
|
||||
# Docker Watchdog second — restarts with pressure already reduced, more likely to stabilise.
|
||||
# System Watchdog last — only triggers if prior layers could not resolve the issue.
|
||||
# Rebooting without first reducing pressure may reboot into the same state.
|
||||
#
|
||||
# ── STARTUP GRACE ─────────────────────────────────────────────────────────────────────────────
|
||||
# No action until system uptime >= WATCHDOG_STARTUP_GRACE seconds.
|
||||
# Prevents false positives from containers still starting at array launch.
|
||||
# Each sub-script enforces this independently — orchestrator exits early to avoid log noise.
|
||||
#
|
||||
# ── OVERLAP PROTECTION ────────────────────────────────────────────────────────────────────────
|
||||
# acquire_lock() — exits immediately if a prior cycle is still in progress.
|
||||
# Prevents pile-up when a cycle runs long (daemon restart attempt = 30s, etc.).
|
||||
#
|
||||
# ── REPLACES ──────────────────────────────────────────────────────────────────────────────────
|
||||
# Continuous loops previously in system_watchdog.sh and docker_watchdog.sh.
|
||||
# Those scripts are now single-pass — this orchestrator provides the cadence.
|
||||
# Remove system_watchdog.sh and docker_watchdog.sh from ARRAY_START_SCRIPTS.
|
||||
#
|
||||
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||||
# WATCHDOG_ORCHESTRATOR_SCRIPTS — watchdogs to run, in order
|
||||
# WATCHDOG_STARTUP_GRACE — seconds after boot before checks activate
|
||||
# WATCHDOG_ORCHESTRATOR_HEARTBEAT — periodic heartbeat log toggle
|
||||
# WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS — heartbeat interval in hours
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# watchdog_orchestrator.sh — normal run (called by cron every minute)
|
||||
# watchdog_orchestrator.sh --dry-run — pass --dry-run to all sub-scripts
|
||||
# watchdog_orchestrator.sh --status — show script paths and current grace state
|
||||
# watchdog_orchestrator.sh --log — verbose output from all sub-scripts
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
|
||||
source "$ECOSYSTEM_ROOT/load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Skip immediately if another cycle is still running — no pile-up
|
||||
acquire_lock
|
||||
|
||||
detect_hosts
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — passing --dry-run to all sub-scripts"
|
||||
|
||||
# Derive a display name from a script path: "resource_watchdog.sh" → "Resource Watchdog"
|
||||
_watchdog_display_name() {
|
||||
local path="$1"
|
||||
local base="${path##*/}"
|
||||
base="${base%.sh}"
|
||||
base="${base//_/ }"
|
||||
echo "$base" | awk '{for(i=1;i<=NF;i++) $i=toupper(substr($i,1,1)) substr($i,2); print}'
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY WATCHDOG ORCHESTRATOR STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo ""
|
||||
|
||||
UPTIME_S=$(awk '{print int($1)}' /proc/uptime)
|
||||
if [[ "$UPTIME_S" -lt "$WATCHDOG_STARTUP_GRACE" ]]; then
|
||||
warn "Within startup grace — $(format_duration $UPTIME_S) / $(format_duration $WATCHDOG_STARTUP_GRACE)"
|
||||
else
|
||||
echo "Past startup grace — $(format_duration $UPTIME_S) uptime"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "── Sub-scripts ──"
|
||||
for entry in "${WATCHDOG_ORCHESTRATOR_SCRIPTS[@]}"; do
|
||||
local_path="$ECOSYSTEM_ROOT/$entry"
|
||||
label="$(_watchdog_display_name "$entry")"
|
||||
if [[ -f "$local_path" ]]; then
|
||||
[[ -x "$local_path" ]] && icon="$ICON_DONE" || icon="$ICON_WARN"
|
||||
echo " $icon $label — ${local_path##*/}"
|
||||
else
|
||||
echo " $ICON_ERROR $label — NOT FOUND: $local_path"
|
||||
fi
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo " Schedule: * * * * * (every minute via User Scripts)"
|
||||
echo " Heartbeat: ${WATCHDOG_ORCHESTRATOR_HEARTBEAT:-true} / every ${WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS:-1}hr"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Startup Grace ━━━
|
||||
# ==============================================================================================
|
||||
UPTIME_SECONDS=$(awk '{print int($1)}' /proc/uptime)
|
||||
if [[ "$UPTIME_SECONDS" -lt "$WATCHDOG_STARTUP_GRACE" ]]; then
|
||||
log "Startup grace — ${UPTIME_SECONDS}s / ${WATCHDOG_STARTUP_GRACE}s — skipping cycle"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Run Watchdog Cycle ━━━
|
||||
# ==============================================================================================
|
||||
CYCLE_START=$(date +%s)
|
||||
PASS=()
|
||||
FAIL=()
|
||||
|
||||
run_watchdog() {
|
||||
local name="$1" script="$2"
|
||||
|
||||
if [[ ! -f "$script" ]]; then
|
||||
error "$name — not found: $script"
|
||||
FAIL+=("$name:missing")
|
||||
return 1
|
||||
fi
|
||||
|
||||
[[ ! -x "$script" ]] && chmod +x "$script"
|
||||
|
||||
local extra_args=()
|
||||
[[ "$DRY_RUN" == true ]] && extra_args+=("--dry-run")
|
||||
[[ "$VERBOSE" == true ]] && extra_args+=("--log")
|
||||
|
||||
log "$ICON_START $name"
|
||||
if bash "$script" "${extra_args[@]}"; then
|
||||
PASS+=("$name")
|
||||
return 0
|
||||
else
|
||||
error "$name — non-zero exit"
|
||||
FAIL+=("$name")
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
for _entry in "${WATCHDOG_ORCHESTRATOR_SCRIPTS[@]}"; do
|
||||
run_watchdog "$(_watchdog_display_name "$_entry")" "$ECOSYSTEM_ROOT/$_entry"
|
||||
done
|
||||
|
||||
CYCLE_END=$(date +%s)
|
||||
DURATION=$(( CYCLE_END - CYCLE_START ))
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Heartbeat ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "${WATCHDOG_ORCHESTRATOR_HEARTBEAT:-true}" == true ]]; then
|
||||
HB_SECONDS=$(( ${WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS:-1} * 3600 ))
|
||||
HB_COUNT_FILE="/tmp/watchdog_orch_hb.count"
|
||||
HB_COUNT=$(cat "$HB_COUNT_FILE" 2>/dev/null || echo 0)
|
||||
HB_COUNT=$(( HB_COUNT + 1 ))
|
||||
echo "$HB_COUNT" > "$HB_COUNT_FILE"
|
||||
# Each cron run = ~60s — use count × 60 as uptime approximation
|
||||
HB_ELAPSED=$(( HB_COUNT * 60 ))
|
||||
if [[ "$HB_SECONDS" -gt 0 ]] && (( HB_ELAPSED % HB_SECONDS < 60 )) && [[ "$HB_COUNT" -gt 1 ]]; then
|
||||
HB_HR=$(( HB_ELAPSED / 3600 ))
|
||||
warn "♥ watchdog_orchestrator alive — $MY_ID — ~${HB_HR}hr ($(date '+%H:%M:%S'))"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary — only shown on failures or --log ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "${#FAIL[@]}" -gt 0 || "$VERBOSE" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY WATCHDOG CYCLE — $MY_ID — $(date '+%H:%M:%S') ━━━━━"
|
||||
for p in "${PASS[@]}"; do log " $ICON_DONE $p"; done
|
||||
for f in "${FAIL[@]}"; do error " $ICON_ERROR $f"; done
|
||||
echo "$ICON_TIME Duration: $(format_duration $DURATION)"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
if [[ "${#FAIL[@]}" -gt 0 ]]; then
|
||||
notify "Watchdog cycle failure on $(hostname) ($MY_ID) — ${FAIL[*]}" \
|
||||
"Watchdog Orchestrator" "warning"
|
||||
fi
|
||||
fi
|
||||
@@ -1,286 +0,0 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# =========================== Watchdog Skip List Manager =======================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# View and manage the persistent container skip list used by docker_watchdog.sh.
|
||||
# docker_watchdog.sh adds a container to the skip list when it exceeds
|
||||
# WATCHDOG_CONTAINER_RESTART_LIMIT restarts within WATCHDOG_CONTAINER_RESTART_WINDOW
|
||||
# hours — prevents infinite restart loops on containers that keep crashing.
|
||||
#
|
||||
# Skip list persists on /boot/config (survives reboots). Auto-clears when
|
||||
# docker_watchdog.sh sees the container running on a later cycle. Use this
|
||||
# script to clear manually after fixing the underlying problem.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Skip List Lifecycle
|
||||
# 1. Container crashes repeatedly → watchdog adds to skip list, notifies
|
||||
# 2. Watchdog stops restarting the container on subsequent cycles
|
||||
# 3a. If container recovers on its own (Docker restart policy), watchdog
|
||||
# sees it running, removes from skip list automatically
|
||||
# 3b. If stuck stopped → fix the root cause, clear via this script, then
|
||||
# docker start ContainerName manually
|
||||
# 4. Watchdog monitors normally on next cycle. If it crashes again → re-added.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Single Instance Lock
|
||||
# acquire_lock prevents concurrent access with docker_watchdog.sh writing
|
||||
# the same files.
|
||||
#
|
||||
# Active Watchdog Detection
|
||||
# Warns if docker_watchdog.sh is currently running when a clear is attempted
|
||||
# — the watchdog could re-add the container to the skip list within seconds.
|
||||
#
|
||||
# Docker Timeout
|
||||
# DOCKER_TIMEOUT caps docker inspect calls against a hung daemon.
|
||||
#
|
||||
# Confirmation Required
|
||||
# Interactive mode prompts for YES before clearing. Use --force for scripts.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# STATE FILES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# SYS_WATCHDOG_FAILED_FILE — persistent container skip list (on /boot/config)
|
||||
# WATCHDOG_CONTAINER_RESTART_LOG — restart history used for loop detection
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# watchdog_skip_list_manager.sh [--status]
|
||||
# Show skip list, container states, and recent restart history.
|
||||
#
|
||||
# watchdog_skip_list_manager.sh --clear ContainerName
|
||||
# Remove a specific container from the skip list and clear its restart history.
|
||||
# Prompts for YES unless --force is passed.
|
||||
#
|
||||
# watchdog_skip_list_manager.sh --clear-all
|
||||
# Clear all skip lists and all restart history.
|
||||
# Prompts for YES unless --force is passed.
|
||||
#
|
||||
# All actions support --dry-run (show what would change) and --force (skip prompt).
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
DOCKER_TIMEOUT=15
|
||||
|
||||
# ── Parse action flags before parse_args ──────────────────────────────────────────────────────
|
||||
ACTION="status"
|
||||
TARGET_CONTAINER=""
|
||||
FORCE=false
|
||||
FILTERED_ARGS=()
|
||||
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--clear-all) ACTION="clear-all" ;;
|
||||
--clear) ACTION="clear" ;;
|
||||
--status) ACTION="status" ;;
|
||||
--force) FORCE=true ;;
|
||||
*)
|
||||
if [[ "$ACTION" == "clear" && -z "$TARGET_CONTAINER" ]]; then
|
||||
TARGET_CONTAINER="$arg"
|
||||
else
|
||||
FILTERED_ARGS+=("$arg")
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
parse_args "${FILTERED_ARGS[@]}"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
validate_unraid_cmd \
|
||||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||||
"" "" \
|
||||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||
|
||||
acquire_lock
|
||||
|
||||
if ! command -v docker &>/dev/null; then
|
||||
error "Docker command not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# detect_hosts() sets MY_ID — used in output
|
||||
detect_hosts
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
[[ "$FORCE" == true ]] && warn "FORCE mode — confirmation prompt skipped"
|
||||
|
||||
# Ensure state files exist
|
||||
touch "$SYS_WATCHDOG_FAILED_FILE" "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status — always shown regardless of action ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_WATCHDOG Skip List Status — $MY_ID ━━━"
|
||||
|
||||
SKIP_COUNT=$(grep -c "." "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null || echo 0)
|
||||
SKIP_COUNT="${SKIP_COUNT//[^0-9]/}"; SKIP_COUNT="${SKIP_COUNT:-0}"
|
||||
RESTART_COUNT=$(wc -l < "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null || echo 0)
|
||||
RESTART_COUNT="${RESTART_COUNT//[^0-9]/}"; RESTART_COUNT="${RESTART_COUNT:-0}"
|
||||
|
||||
# docker_watchdog.sh running check
|
||||
WATCHDOG_RUNNING=false
|
||||
if pgrep -f "docker_watchdog.sh" >/dev/null 2>&1; then
|
||||
WATCHDOG_RUNNING=true
|
||||
warn "docker_watchdog.sh is currently RUNNING"
|
||||
[[ "$ACTION" != "status" ]] && \
|
||||
warn "Clearing during an active cycle — watchdog may re-add container on next iteration"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
if [[ "$SKIP_COUNT" -eq 0 ]]; then
|
||||
echo "Skip list: empty — all containers monitored normally ✅"
|
||||
else
|
||||
warn "$SKIP_COUNT container(s) on skip list — manual intervention needed:"
|
||||
echo ""
|
||||
while IFS= read -r container; do
|
||||
[[ -z "$container" ]] && continue
|
||||
STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
|
||||
'{{.State.Running}}' "$container" 2>/dev/null || echo "unknown")
|
||||
case "$STATUS" in
|
||||
true)
|
||||
echo " $ICON_RUNNING $container — RUNNING (watchdog will auto-clear next cycle)"
|
||||
;;
|
||||
false)
|
||||
echo " $ICON_NOT_RUNNING $container — STOPPED — fix and start manually"
|
||||
;;
|
||||
*)
|
||||
echo " $ICON_WARN $container — not found on this server"
|
||||
;;
|
||||
esac
|
||||
done < "$SYS_WATCHDOG_FAILED_FILE"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "━━━ $ICON_WATCHDOG Restart History ━━━"
|
||||
if [[ "$RESTART_COUNT" -eq 0 ]]; then
|
||||
echo "No restart history"
|
||||
else
|
||||
echo "$RESTART_COUNT restart entries (window: ${WATCHDOG_CONTAINER_RESTART_WINDOW}h)"
|
||||
echo ""
|
||||
awk -F'|' '{counts[$1]++} END {
|
||||
for (c in counts)
|
||||
printf " %-30s %d restart(s)\n", c, counts[c]
|
||||
}' "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | sort
|
||||
fi
|
||||
|
||||
[[ "$ACTION" == "status" ]] && exit 0
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Clear All ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$ACTION" == "clear-all" ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_TRASH Clear All Skip Lists ━━━"
|
||||
warn "This will clear the skip list and restart history for ALL containers"
|
||||
echo ""
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
if [[ "$FORCE" == true ]]; then
|
||||
log "FORCE flag set — skipping confirmation"
|
||||
elif [[ -t 0 ]]; then
|
||||
read -r -p "Type YES to confirm: " CONFIRM
|
||||
if [[ "$CONFIRM" != "YES" ]]; then
|
||||
warn "Cancelled"
|
||||
exit 0
|
||||
fi
|
||||
else
|
||||
error "Non-interactive mode — use --force flag to skip confirmation"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
> "$SYS_WATCHDOG_FAILED_FILE"
|
||||
> "$WATCHDOG_CONTAINER_RESTART_LOG"
|
||||
warn "Skip list cleared ✅"
|
||||
warn "Restart history cleared ✅"
|
||||
[[ "$WATCHDOG_RUNNING" == true ]] && \
|
||||
warn "Note: watchdog is running — containers will be monitored on next cycle"
|
||||
notify "Watchdog skip list cleared on $(hostname) ($MY_ID) — all containers will be monitored normally" \
|
||||
"Watchdog Manager" "warning"
|
||||
else
|
||||
warn "DRY RUN — would clear: $SYS_WATCHDOG_FAILED_FILE"
|
||||
warn "DRY RUN — would clear: $WATCHDOG_CONTAINER_RESTART_LOG"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Clear Specific Container ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$ACTION" == "clear" ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_TRASH Clear Container: $TARGET_CONTAINER ━━━"
|
||||
|
||||
if [[ -z "$TARGET_CONTAINER" ]]; then
|
||||
error "No container specified"
|
||||
error "Usage: watchdog_skip_list_manager.sh --clear ContainerName"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Remove from skip list
|
||||
if ! grep -q "^${TARGET_CONTAINER}$" "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null; then
|
||||
warn "$TARGET_CONTAINER is not on the skip list"
|
||||
else
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
sed -i "/^${TARGET_CONTAINER}$/d" "$SYS_WATCHDOG_FAILED_FILE"
|
||||
warn "$TARGET_CONTAINER removed from skip list ✅"
|
||||
else
|
||||
warn "DRY RUN — would remove $TARGET_CONTAINER from skip list"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Clear restart history for this container
|
||||
HIST_COUNT=$(grep -c "^${TARGET_CONTAINER}|" \
|
||||
"$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null || echo 0)
|
||||
HIST_COUNT="${HIST_COUNT//[^0-9]/}"; HIST_COUNT="${HIST_COUNT:-0}"
|
||||
|
||||
if [[ "$HIST_COUNT" -gt 0 ]]; then
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
sed -i "/^${TARGET_CONTAINER}|/d" "$WATCHDOG_CONTAINER_RESTART_LOG"
|
||||
warn "Cleared $HIST_COUNT restart history entries for $TARGET_CONTAINER ✅"
|
||||
else
|
||||
warn "DRY RUN — would clear $HIST_COUNT restart history entries"
|
||||
fi
|
||||
else
|
||||
log "No restart history for $TARGET_CONTAINER"
|
||||
fi
|
||||
|
||||
[[ "$WATCHDOG_RUNNING" == true ]] && \
|
||||
warn "Note: watchdog is running — $TARGET_CONTAINER may be re-added if still failing"
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_INFO Next Steps ━━━"
|
||||
echo " 1. Fix whatever was causing $TARGET_CONTAINER to fail"
|
||||
echo " 2. Start it manually: docker start $TARGET_CONTAINER"
|
||||
echo " 3. docker_watchdog.sh monitors it on the next cycle"
|
||||
echo " 4. If it crashes again → watchdog adds it back and notifies"
|
||||
notify "$TARGET_CONTAINER cleared from watchdog skip list on $(hostname) ($MY_ID)" \
|
||||
"Watchdog Manager" "warning"
|
||||
fi
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY DONE — $MY_ID ━━━━━"
|
||||
Reference in New Issue
Block a user