massive update to rsync stop
This commit is contained in:
+280
-67
@@ -2,12 +2,28 @@
|
|||||||
# -----------------------------------------------------------------------------------------------
|
# -----------------------------------------------------------------------------------------------
|
||||||
# --------------------------------- Rsync Stop Script ------------------------------------------
|
# --------------------------------- Rsync Stop Script ------------------------------------------
|
||||||
# -----------------------------------------------------------------------------------------------
|
# -----------------------------------------------------------------------------------------------
|
||||||
# Stops all running rsync processes on both the local and remote server.
|
# Stops rsync intelligently — auto-detects what's running and acts accordingly.
|
||||||
# Used during array stop or manually when rsync needs to be forcefully terminated.
|
#
|
||||||
# If rsync processes were killed locally, checks all profile containers and restarts any
|
# Default behavior (just run it):
|
||||||
# that were left stopped by the interrupted rsync run.
|
# Detects if an orchestrator (daily/weekly) is running
|
||||||
# Remote is killed but left in whatever container state it is in — secondary is self-healing.
|
# If yes → kills rsync subprocess only
|
||||||
# Supports --dry-run to preview what would be killed without making changes.
|
# orchestrator sees rsync died → moves to next share or exits cleanly
|
||||||
|
# If no → kills rsync processes directly (solo rsync.sh run)
|
||||||
|
# Cleans stale lock files
|
||||||
|
# Recovers any containers left stopped by interrupted rsync
|
||||||
|
#
|
||||||
|
# --full-stop flag (nuclear):
|
||||||
|
# Kills orchestrator first → then rsync
|
||||||
|
# Use when: you want everything dead immediately
|
||||||
|
# daily/weekly loop will NOT continue to next share
|
||||||
|
#
|
||||||
|
# Both local and remote are handled in one run.
|
||||||
|
# Remote containers left as-is — docker_watchdog.sh handles remote recovery.
|
||||||
|
#
|
||||||
|
# Flags:
|
||||||
|
# (none) ← smart mode — auto-detects, rsync-only if orchestrator running
|
||||||
|
# --full-stop ← nuclear — kill orchestrator + rsync
|
||||||
|
# --dry-run ← preview without changes
|
||||||
# -----------------------------------------------------------------------------------------------
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
@@ -15,7 +31,18 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|||||||
source "$SCRIPT_DIR/../Master.conf"
|
source "$SCRIPT_DIR/../Master.conf"
|
||||||
source "$SCRIPT_DIR/../common.sh"
|
source "$SCRIPT_DIR/../common.sh"
|
||||||
|
|
||||||
parse_args "$@"
|
# Check for --full-stop before parse_args
|
||||||
|
FULL_STOP=false
|
||||||
|
FILTERED_ARGS=()
|
||||||
|
for arg in "$@"; do
|
||||||
|
if [[ "$arg" == "--full-stop" ]]; then
|
||||||
|
FULL_STOP=true
|
||||||
|
else
|
||||||
|
FILTERED_ARGS+=("$arg")
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
parse_args "${FILTERED_ARGS[@]}"
|
||||||
|
|
||||||
# -----------------------------------------------------------------------------------------------
|
# -----------------------------------------------------------------------------------------------
|
||||||
# ━━━ $ICON_GEAR Setup ━━━
|
# ━━━ $ICON_GEAR Setup ━━━
|
||||||
@@ -26,18 +53,202 @@ echo "━━━ $ICON_GEAR Setup ━━━"
|
|||||||
detect_hosts
|
detect_hosts
|
||||||
resolve_remote_ip
|
resolve_remote_ip
|
||||||
|
|
||||||
# Check connectivity but do not exit on failure — remote may already be going down
|
|
||||||
REMOTE_REACHABLE=true
|
REMOTE_REACHABLE=true
|
||||||
if ! ping -c1 -W3 "$REMOTE_SERVER" &>/dev/null; then
|
if ! ping -c1 -W3 "$REMOTE_SERVER" &>/dev/null; then
|
||||||
warn "$ICON_PING Remote $REMOTE_SERVER_NAME is unreachable — will skip remote kill"
|
warn "$ICON_PING Remote $REMOTE_SERVER_NAME unreachable — will skip remote"
|
||||||
REMOTE_REACHABLE=false
|
REMOTE_REACHABLE=false
|
||||||
else
|
else
|
||||||
info "$ICON_PING $REMOTE_SERVER_NAME is reachable"
|
info "$ICON_PING $REMOTE_SERVER_NAME reachable"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
[[ "$SHOW_STATUS" == true ]] && show_status && exit 0
|
[[ "$SHOW_STATUS" == true ]] && show_status && exit 0
|
||||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||||
|
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
# ━━━ Auto-detect orchestrators ━━━
|
||||||
|
# Check if daily or weekly is running on local and remote
|
||||||
|
# This determines default behavior
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
# detect_rsync_parent — scans all lock files, finds which running process has rsync as a child
|
||||||
|
# No hardcoded list — works for any orchestrator automatically
|
||||||
|
#
|
||||||
|
# Returns: "script_name:parent_pid" if found, empty if rsync running standalone
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
detect_rsync_parent() {
|
||||||
|
local found=""
|
||||||
|
|
||||||
|
# Get all rsync PIDs running locally
|
||||||
|
local rsync_pids
|
||||||
|
rsync_pids=$(pgrep -x rsync 2>/dev/null || true)
|
||||||
|
[[ -z "$rsync_pids" ]] && echo "" && return
|
||||||
|
|
||||||
|
# Scan all lock files in LOCK_DIR
|
||||||
|
for lockfile in "$LOCK_DIR"/*.lock; do
|
||||||
|
[[ -f "$lockfile" ]] || continue
|
||||||
|
|
||||||
|
local content pid locked_name
|
||||||
|
content=$(cat "$lockfile" 2>/dev/null)
|
||||||
|
pid="${content%%:*}"
|
||||||
|
locked_name="${content##*:}"
|
||||||
|
|
||||||
|
# Skip if PID dead or is itself a rsync lock
|
||||||
|
[[ -z "$pid" ]] && continue
|
||||||
|
! kill -0 "$pid" 2>/dev/null && continue
|
||||||
|
[[ "$locked_name" == rsync_* ]] && continue
|
||||||
|
|
||||||
|
# Check if any rsync PID is a child of this lock's PID
|
||||||
|
local children
|
||||||
|
children=$(cat /proc/"$pid"/task/"$pid"/children 2>/dev/null || \
|
||||||
|
tr ' ' '\n' < /proc/"$pid"/children 2>/dev/null || true)
|
||||||
|
|
||||||
|
# Walk the child tree — rsync may be a grandchild (bash → rsync.sh → rsync)
|
||||||
|
local all_descendants
|
||||||
|
all_descendants=$(pgrep -P "$pid" 2>/dev/null || true)
|
||||||
|
|
||||||
|
# Check if any rsync PID is in the descendants
|
||||||
|
while IFS= read -r rsync_pid; do
|
||||||
|
[[ -z "$rsync_pid" ]] && continue
|
||||||
|
if echo "$all_descendants" | grep -qw "$rsync_pid" 2>/dev/null || \
|
||||||
|
[[ "$(cat /proc/"$rsync_pid"/status 2>/dev/null | awk '/^PPid:/{print $2}')" == "$pid" ]]; then
|
||||||
|
found="$locked_name:$pid"
|
||||||
|
break 2
|
||||||
|
fi
|
||||||
|
done <<< "$rsync_pids"
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "$found"
|
||||||
|
}
|
||||||
|
|
||||||
|
detect_rsync_parent_remote() {
|
||||||
|
[[ "$REMOTE_REACHABLE" != true ]] && echo "" && return
|
||||||
|
|
||||||
|
# Run the same logic on remote via SSH
|
||||||
|
local found
|
||||||
|
found=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" bash << 'REMOTE_SCRIPT'
|
||||||
|
LOCK_DIR="/tmp/unraid_locks"
|
||||||
|
rsync_pids=$(pgrep -x rsync 2>/dev/null || true)
|
||||||
|
[[ -z "$rsync_pids" ]] && exit 0
|
||||||
|
|
||||||
|
for lockfile in "$LOCK_DIR"/*.lock; do
|
||||||
|
[[ -f "$lockfile" ]] || continue
|
||||||
|
content=$(cat "$lockfile" 2>/dev/null)
|
||||||
|
pid="${content%%:*}"
|
||||||
|
locked_name="${content##*:}"
|
||||||
|
[[ -z "$pid" ]] && continue
|
||||||
|
! kill -0 "$pid" 2>/dev/null && continue
|
||||||
|
[[ "$locked_name" == rsync_* ]] && continue
|
||||||
|
|
||||||
|
all_descendants=$(pgrep -P "$pid" 2>/dev/null || true)
|
||||||
|
while IFS= read -r rsync_pid; do
|
||||||
|
[[ -z "$rsync_pid" ]] && continue
|
||||||
|
ppid=$(awk '/^PPid:/{print $2}' /proc/"$rsync_pid"/status 2>/dev/null)
|
||||||
|
if echo "$all_descendants" | grep -qw "$rsync_pid" 2>/dev/null || \
|
||||||
|
[[ "$ppid" == "$pid" ]]; then
|
||||||
|
echo "$locked_name:$pid"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
done <<< "$rsync_pids"
|
||||||
|
done
|
||||||
|
REMOTE_SCRIPT
|
||||||
|
2>/dev/null)
|
||||||
|
echo "$found"
|
||||||
|
}
|
||||||
|
|
||||||
|
LOCAL_ORCH=$(detect_rsync_parent)
|
||||||
|
REMOTE_ORCH=""
|
||||||
|
[[ "$REMOTE_REACHABLE" == true ]] && REMOTE_ORCH=$(detect_rsync_parent_remote)
|
||||||
|
|
||||||
|
# Determine mode
|
||||||
|
if [[ "$FULL_STOP" == true ]]; then
|
||||||
|
MODE="full-stop"
|
||||||
|
info "Mode: FULL STOP — orchestrator + rsync will be killed"
|
||||||
|
elif [[ -n "$LOCAL_ORCH" ]] || [[ -n "$REMOTE_ORCH" ]]; then
|
||||||
|
MODE="rsync-only"
|
||||||
|
[[ -n "$LOCAL_ORCH" ]] && info "Detected local orchestrator: ${LOCAL_ORCH%%:*} — rsync-only mode"
|
||||||
|
[[ -n "$REMOTE_ORCH" ]] && info "Detected remote orchestrator: ${REMOTE_ORCH%%:*} — rsync-only mode"
|
||||||
|
info "Orchestrator will continue after rsync is killed"
|
||||||
|
info "Use --full-stop to also kill the orchestrator"
|
||||||
|
else
|
||||||
|
MODE="rsync-only"
|
||||||
|
info "No orchestrator detected — killing rsync directly"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
# ━━━ $ICON_STOP Kill Orchestrators (full-stop only) ━━━
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
ORCHESTRATORS_KILLED=()
|
||||||
|
REMOTE_ORCHESTRATORS_KILLED=()
|
||||||
|
|
||||||
|
kill_orchestrator() {
|
||||||
|
local script_name="$1"
|
||||||
|
local pid="$2"
|
||||||
|
local lockfile="$LOCK_DIR/${script_name}.lock"
|
||||||
|
|
||||||
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
|
warn "DRY RUN — would kill $script_name (PID $pid)"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
kill -TERM "$pid" 2>/dev/null
|
||||||
|
sleep 2
|
||||||
|
kill -0 "$pid" 2>/dev/null && kill -KILL "$pid" 2>/dev/null
|
||||||
|
sleep 1
|
||||||
|
|
||||||
|
if ! kill -0 "$pid" 2>/dev/null; then
|
||||||
|
success "$script_name stopped ✅"
|
||||||
|
rm -f "$lockfile"
|
||||||
|
return 0
|
||||||
|
else
|
||||||
|
error "Failed to kill $script_name (PID $pid)"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
if [[ "$MODE" == "full-stop" ]]; then
|
||||||
|
echo ""
|
||||||
|
echo "━━━ $ICON_STOP Orchestrators ━━━"
|
||||||
|
|
||||||
|
# Local
|
||||||
|
if [[ -n "$LOCAL_ORCH" ]]; then
|
||||||
|
name="${LOCAL_ORCH%%:*}"
|
||||||
|
pid="${LOCAL_ORCH##*:}"
|
||||||
|
info "Killing local: $name (PID $pid)"
|
||||||
|
if kill_orchestrator "$name" "$pid"; then
|
||||||
|
ORCHESTRATORS_KILLED+=("$name")
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
info "No local orchestrator running"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Remote
|
||||||
|
if [[ "$REMOTE_REACHABLE" == true ]] && [[ -n "$REMOTE_ORCH" ]]; then
|
||||||
|
name="${REMOTE_ORCH%%:*}"
|
||||||
|
pid="${REMOTE_ORCH##*:}"
|
||||||
|
lockfile="$LOCK_DIR/${name}.lock"
|
||||||
|
info "Killing remote: $name (PID $pid)"
|
||||||
|
if [[ "$DRY_RUN" == false ]]; then
|
||||||
|
ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||||||
|
"kill -TERM '$pid' 2>/dev/null; sleep 2; \
|
||||||
|
kill -0 '$pid' 2>/dev/null && kill -KILL '$pid' 2>/dev/null; \
|
||||||
|
rm -f '$lockfile'" 2>/dev/null
|
||||||
|
success "Remote $name stopped ✅"
|
||||||
|
REMOTE_ORCHESTRATORS_KILLED+=("$name")
|
||||||
|
else
|
||||||
|
warn "DRY RUN — would kill remote $name (PID $pid)"
|
||||||
|
fi
|
||||||
|
elif [[ "$REMOTE_REACHABLE" == true ]]; then
|
||||||
|
info "No remote orchestrator running"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Wait for subprocesses to settle
|
||||||
|
if [[ ${#ORCHESTRATORS_KILLED[@]} -gt 0 ]] || \
|
||||||
|
[[ ${#REMOTE_ORCHESTRATORS_KILLED[@]} -gt 0 ]]; then
|
||||||
|
info "Waiting 3s for subprocesses to settle..."
|
||||||
|
sleep 3
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
# -----------------------------------------------------------------------------------------------
|
# -----------------------------------------------------------------------------------------------
|
||||||
# ━━━ $ICON_STOP Local Rsync ━━━
|
# ━━━ $ICON_STOP Local Rsync ━━━
|
||||||
# -----------------------------------------------------------------------------------------------
|
# -----------------------------------------------------------------------------------------------
|
||||||
@@ -48,22 +259,32 @@ LOCAL_KILLED=false
|
|||||||
LOCAL_PIDS=$(pgrep -x rsync || true)
|
LOCAL_PIDS=$(pgrep -x rsync || true)
|
||||||
|
|
||||||
if [[ -z "$LOCAL_PIDS" ]]; then
|
if [[ -z "$LOCAL_PIDS" ]]; then
|
||||||
info "No rsync processes running locally — nothing to kill"
|
info "No rsync processes running locally"
|
||||||
else
|
else
|
||||||
info "Found rsync processes locally: $(echo "$LOCAL_PIDS" | tr '\n' ' ')"
|
info "Found PIDs: $(echo "$LOCAL_PIDS" | tr '\n' ' ')"
|
||||||
|
|
||||||
if [[ "$DRY_RUN" == true ]]; then
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
warn "DRY RUN — would kill local rsync processes"
|
warn "DRY RUN — would kill local rsync"
|
||||||
else
|
else
|
||||||
if pkill -x rsync; then
|
if pkill -x rsync; then
|
||||||
success "Local rsync processes killed"
|
success "Local rsync killed ✅"
|
||||||
LOCAL_KILLED=true
|
LOCAL_KILLED=true
|
||||||
else
|
else
|
||||||
warn "pkill returned non-zero — processes may have already exited"
|
warn "pkill non-zero — may have already exited"
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# Clean stale rsync lock files
|
||||||
|
for lockfile in "$LOCK_DIR"/rsync_*.lock; do
|
||||||
|
[[ -f "$lockfile" ]] || continue
|
||||||
|
content=$(cat "$lockfile" 2>/dev/null)
|
||||||
|
pid="${content%%:*}"
|
||||||
|
if [[ -n "$pid" ]] && ! kill -0 "$pid" 2>/dev/null; then
|
||||||
|
info "Cleaning stale lock: $(basename "$lockfile")"
|
||||||
|
[[ "$DRY_RUN" == false ]] && rm -f "$lockfile"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
# -----------------------------------------------------------------------------------------------
|
# -----------------------------------------------------------------------------------------------
|
||||||
# ━━━ $ICON_STOP Remote Rsync ━━━
|
# ━━━ $ICON_STOP Remote Rsync ━━━
|
||||||
# -----------------------------------------------------------------------------------------------
|
# -----------------------------------------------------------------------------------------------
|
||||||
@@ -73,43 +294,41 @@ echo "━━━ $ICON_STOP Remote Rsync ($REMOTE_SERVER_NAME) ━━━"
|
|||||||
REMOTE_KILLED=false
|
REMOTE_KILLED=false
|
||||||
|
|
||||||
if [[ "$REMOTE_REACHABLE" == false ]]; then
|
if [[ "$REMOTE_REACHABLE" == false ]]; then
|
||||||
warn "Skipping remote kill — $REMOTE_SERVER_NAME unreachable"
|
warn "Skipping — $REMOTE_SERVER_NAME unreachable"
|
||||||
else
|
else
|
||||||
REMOTE_PIDS=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
REMOTE_PIDS=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||||||
"pgrep -x rsync || true" 2>/dev/null || true)
|
"pgrep -x rsync || true" 2>/dev/null || true)
|
||||||
|
|
||||||
if [[ -z "$REMOTE_PIDS" ]]; then
|
if [[ -z "$REMOTE_PIDS" ]]; then
|
||||||
info "No rsync processes running on $REMOTE_SERVER_NAME — nothing to kill"
|
info "No rsync running on $REMOTE_SERVER_NAME"
|
||||||
else
|
else
|
||||||
info "Found rsync processes on $REMOTE_SERVER_NAME: $(echo "$REMOTE_PIDS" | tr '\n' ' ')"
|
info "Found PIDs on $REMOTE_SERVER_NAME: $(echo "$REMOTE_PIDS" | tr '\n' ' ')"
|
||||||
|
|
||||||
if [[ "$DRY_RUN" == true ]]; then
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
warn "DRY RUN — would kill remote rsync processes"
|
warn "DRY RUN — would kill remote rsync"
|
||||||
else
|
else
|
||||||
if ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" "pkill -x rsync || true" 2>/dev/null; then
|
if ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" "pkill -x rsync || true" 2>/dev/null; then
|
||||||
success "Remote rsync processes killed"
|
success "Remote rsync killed ✅"
|
||||||
REMOTE_KILLED=true
|
REMOTE_KILLED=true
|
||||||
else
|
else
|
||||||
warn "Remote pkill returned non-zero — processes may have already exited"
|
warn "Remote pkill non-zero — may have already exited"
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# -----------------------------------------------------------------------------------------------
|
# -----------------------------------------------------------------------------------------------
|
||||||
# ━━━ $ICON_START $ICON_CONTAINERS Local Container Recovery ━━━
|
# ━━━ $ICON_START $ICON_CONTAINERS Container Recovery ━━━
|
||||||
# Only runs if rsync was actually killed locally — containers may have been left stopped
|
# Restart containers left stopped by interrupted rsync
|
||||||
# by the interrupted rsync run. Checks all containers across all profiles and restarts
|
# Only runs if something was actually killed locally
|
||||||
# any that are currently stopped. Remote containers are left in their current state.
|
# Remote containers left as-is — docker_watchdog.sh handles remote
|
||||||
# -----------------------------------------------------------------------------------------------
|
# -----------------------------------------------------------------------------------------------
|
||||||
CONTAINERS_RESTARTED=()
|
CONTAINERS_RESTARTED=()
|
||||||
|
|
||||||
if [[ "$LOCAL_KILLED" == true ]]; then
|
if [[ "$LOCAL_KILLED" == true ]] || [[ ${#ORCHESTRATORS_KILLED[@]} -gt 0 ]]; then
|
||||||
echo ""
|
echo ""
|
||||||
echo "━━━ $ICON_START $ICON_CONTAINERS Local Container Recovery ━━━"
|
echo "━━━ $ICON_START $ICON_CONTAINERS Container Recovery ━━━"
|
||||||
info "Rsync was killed locally — checking all profile containers..."
|
info "Checking all profile containers..."
|
||||||
|
|
||||||
# Build deduplicated list of all containers across all profiles
|
|
||||||
declare -A SEEN
|
declare -A SEEN
|
||||||
ALL_CONTAINERS=()
|
ALL_CONTAINERS=()
|
||||||
|
|
||||||
@@ -125,35 +344,29 @@ if [[ "$LOCAL_KILLED" == true ]]; then
|
|||||||
done
|
done
|
||||||
|
|
||||||
if [[ ${#ALL_CONTAINERS[@]} -eq 0 ]]; then
|
if [[ ${#ALL_CONTAINERS[@]} -eq 0 ]]; then
|
||||||
info "No containers defined across any profile — skipping recovery"
|
info "No containers defined — skipping recovery"
|
||||||
else
|
else
|
||||||
for c in "${ALL_CONTAINERS[@]}"; do
|
for c in "${ALL_CONTAINERS[@]}"; do
|
||||||
info "Checking $c..."
|
|
||||||
|
|
||||||
STATUS=$(docker inspect -f '{{.State.Running}}' "$c" 2>/dev/null || echo "unknown")
|
STATUS=$(docker inspect -f '{{.State.Running}}' "$c" 2>/dev/null || echo "unknown")
|
||||||
|
|
||||||
if [[ "$STATUS" == "true" ]]; then
|
if [[ "$STATUS" == "true" ]]; then
|
||||||
echo "$ICON_RUNNING $c is running — no action needed"
|
info "$ICON_RUNNING $c — running ✅"
|
||||||
elif [[ "$STATUS" == "false" ]]; then
|
elif [[ "$STATUS" == "false" ]]; then
|
||||||
echo "$ICON_NOT_RUNNING $c is stopped — restarting..."
|
warn "$ICON_NOT_RUNNING $c — stopped, restarting..."
|
||||||
|
|
||||||
if [[ "$DRY_RUN" == true ]]; then
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
warn "DRY RUN — would restart $c"
|
warn "DRY RUN — would restart $c"
|
||||||
else
|
else
|
||||||
if docker start "$c" >/dev/null 2>&1; then
|
if docker start "$c" >/dev/null 2>&1; then
|
||||||
echo "$ICON_STARTED $c restarted"
|
success "$c restarted ✅"
|
||||||
CONTAINERS_RESTARTED+=("$c")
|
CONTAINERS_RESTARTED+=("$c")
|
||||||
else
|
else
|
||||||
error "Failed to restart $c"
|
error "Failed to restart $c"
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
else
|
else
|
||||||
warn "$c state unknown — may not exist on this machine, skipping"
|
info "$c not found on this host — skipping"
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
fi
|
fi
|
||||||
elif [[ "$DRY_RUN" == false ]]; then
|
|
||||||
info "No local rsync was killed — skipping container recovery"
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# -----------------------------------------------------------------------------------------------
|
# -----------------------------------------------------------------------------------------------
|
||||||
@@ -161,39 +374,39 @@ fi
|
|||||||
# -----------------------------------------------------------------------------------------------
|
# -----------------------------------------------------------------------------------------------
|
||||||
echo ""
|
echo ""
|
||||||
echo "━━━━━ $ICON_SUMMARY RSYNC STOP SUMMARY ━━━━━"
|
echo "━━━━━ $ICON_SUMMARY RSYNC STOP SUMMARY ━━━━━"
|
||||||
|
echo " Mode: $MODE"
|
||||||
|
echo ""
|
||||||
|
|
||||||
echo "$ICON_HOST Local ($LOCAL_SERVER_NAME):"
|
echo "$ICON_HOST Local ($LOCAL_SERVER_NAME):"
|
||||||
if [[ "$LOCAL_KILLED" == true ]]; then
|
[[ ${#ORCHESTRATORS_KILLED[@]} -gt 0 ]] && \
|
||||||
echo " $ICON_STOPPED Rsync killed"
|
echo " $ICON_STOPPED Orchestrators killed: ${ORCHESTRATORS_KILLED[*]}"
|
||||||
elif [[ "$DRY_RUN" == true ]]; then
|
[[ "$LOCAL_KILLED" == true ]] && \
|
||||||
echo " $ICON_WARN Dry run — no changes made"
|
echo " $ICON_STOPPED Rsync killed" || \
|
||||||
else
|
echo " $ICON_SUCCESS No rsync was running"
|
||||||
echo " $ICON_SUCCESS No rsync running"
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "$ICON_NET Remote ($REMOTE_SERVER_NAME):"
|
echo "$ICON_NET Remote ($REMOTE_SERVER_NAME):"
|
||||||
if [[ "$REMOTE_REACHABLE" == false ]]; then
|
if [[ "$REMOTE_REACHABLE" == false ]]; then
|
||||||
echo " $ICON_WARN Unreachable — state unknown"
|
echo " $ICON_WARN Unreachable — skipped"
|
||||||
elif [[ "$REMOTE_KILLED" == true ]]; then
|
|
||||||
echo " $ICON_STOPPED Rsync killed"
|
|
||||||
else
|
else
|
||||||
echo " $ICON_SUCCESS No rsync running"
|
[[ ${#REMOTE_ORCHESTRATORS_KILLED[@]} -gt 0 ]] && \
|
||||||
|
echo " $ICON_STOPPED Orchestrators killed: ${REMOTE_ORCHESTRATORS_KILLED[*]}"
|
||||||
|
[[ "$REMOTE_KILLED" == true ]] && \
|
||||||
|
echo " $ICON_STOPPED Rsync killed" || \
|
||||||
|
echo " $ICON_SUCCESS No rsync was running"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [[ ${#CONTAINERS_RESTARTED[@]} -gt 0 ]]; then
|
[[ ${#CONTAINERS_RESTARTED[@]} -gt 0 ]] && \
|
||||||
echo "$ICON_CONTAINERS Containers restarted locally:"
|
echo "$ICON_CONTAINERS Containers recovered: ${CONTAINERS_RESTARTED[*]}"
|
||||||
for c in "${CONTAINERS_RESTARTED[@]}"; do
|
|
||||||
echo " $ICON_STARTED $c"
|
|
||||||
done
|
|
||||||
elif [[ "$LOCAL_KILLED" == true ]]; then
|
|
||||||
echo "$ICON_CONTAINERS No containers needed restarting"
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "$ICON_TIME Dry Run: $DRY_RUN"
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
|
echo "$ICON_WARN Status: DRY RUN — no changes made"
|
||||||
|
else
|
||||||
|
echo "$ICON_DONE Status: $ICON_SUCCESS DONE"
|
||||||
|
fi
|
||||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||||
|
|
||||||
# Notify based on what happened
|
if [[ "$LOCAL_KILLED" == true ]] || [[ "$REMOTE_KILLED" == true ]] || \
|
||||||
if [[ "$LOCAL_KILLED" == true ]] || [[ "$REMOTE_KILLED" == true ]]; then
|
[[ ${#ORCHESTRATORS_KILLED[@]} -gt 0 ]]; then
|
||||||
local_status=$([[ "$LOCAL_KILLED" == true ]] && echo "killed" || echo "clean")
|
notify "Rsync stopped on $(hostname) — mode: $MODE — ${#CONTAINERS_RESTARTED[@]} containers recovered" \
|
||||||
remote_status=$([[ "$REMOTE_KILLED" == true ]] && echo "killed" || echo "clean")
|
"Rsync Stop" "warning"
|
||||||
notify "Rsync stopped — local: $local_status remote: $remote_status — ${#CONTAINERS_RESTARTED[@]} containers recovered" "Rsync Stop" "warning"
|
|
||||||
fi
|
fi
|
||||||
Reference in New Issue
Block a user