192 lines
7.6 KiB
Bash
192 lines
7.6 KiB
Bash
#!/bin/bash
|
|
# -----------------------------------------------------------------------------------------------
|
|
# --------------------------------- Rsync Stop Script ------------------------------------------
|
|
# -----------------------------------------------------------------------------------------------
|
|
# Stops all running rsync processes on both the local and remote server.
|
|
# Used during array stop or manually when rsync needs to be forcefully terminated.
|
|
# If rsync processes were killed locally, checks all profile containers and restarts any
|
|
# that were left stopped by the interrupted rsync run.
|
|
# Remote is killed but left in whatever container state it is in — secondary is self-healing.
|
|
# Supports --dry-run to preview what would be killed without making changes.
|
|
# -----------------------------------------------------------------------------------------------
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
source "$SCRIPT_DIR/../Master.conf"
|
|
source "$SCRIPT_DIR/../common.sh"
|
|
|
|
parse_args "$@"
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# ━━━ $ICON_GEAR Setup ━━━
|
|
# -----------------------------------------------------------------------------------------------
|
|
echo ""
|
|
echo "━━━ $ICON_GEAR Setup ━━━"
|
|
|
|
detect_hosts
|
|
resolve_remote_ip
|
|
|
|
# Check connectivity but do not exit on failure — remote may already be going down
|
|
REMOTE_REACHABLE=true
|
|
if ! ping -c1 -W3 "$REMOTE_SERVER" &>/dev/null; then
|
|
warn "$ICON_PING Remote $REMOTE_SERVER_NAME is unreachable — will skip remote kill"
|
|
REMOTE_REACHABLE=false
|
|
else
|
|
info "$ICON_PING $REMOTE_SERVER_NAME is reachable"
|
|
fi
|
|
|
|
[[ "$SHOW_STATUS" == true ]] && show_status && exit 0
|
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# ━━━ $ICON_STOP Local Rsync ━━━
|
|
# -----------------------------------------------------------------------------------------------
|
|
echo ""
|
|
echo "━━━ $ICON_STOP Local Rsync ━━━"
|
|
|
|
LOCAL_KILLED=false
|
|
LOCAL_PIDS=$(pgrep -x rsync || true)
|
|
|
|
if [[ -z "$LOCAL_PIDS" ]]; then
|
|
info "No rsync processes running locally — nothing to kill"
|
|
else
|
|
info "Found rsync processes locally: $(echo "$LOCAL_PIDS" | tr '\n' ' ')"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would kill local rsync processes"
|
|
else
|
|
if pkill -x rsync; then
|
|
success "Local rsync processes killed"
|
|
LOCAL_KILLED=true
|
|
else
|
|
warn "pkill returned non-zero — processes may have already exited"
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# ━━━ $ICON_STOP Remote Rsync ━━━
|
|
# -----------------------------------------------------------------------------------------------
|
|
echo ""
|
|
echo "━━━ $ICON_STOP Remote Rsync ($REMOTE_SERVER_NAME) ━━━"
|
|
|
|
REMOTE_KILLED=false
|
|
|
|
if [[ "$REMOTE_REACHABLE" == false ]]; then
|
|
warn "Skipping remote kill — $REMOTE_SERVER_NAME unreachable"
|
|
else
|
|
REMOTE_PIDS=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
|
"pgrep -x rsync || true" 2>/dev/null || true)
|
|
|
|
if [[ -z "$REMOTE_PIDS" ]]; then
|
|
info "No rsync processes running on $REMOTE_SERVER_NAME — nothing to kill"
|
|
else
|
|
info "Found rsync processes on $REMOTE_SERVER_NAME: $(echo "$REMOTE_PIDS" | tr '\n' ' ')"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would kill remote rsync processes"
|
|
else
|
|
if ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" "pkill -x rsync || true" 2>/dev/null; then
|
|
success "Remote rsync processes killed"
|
|
REMOTE_KILLED=true
|
|
else
|
|
warn "Remote pkill returned non-zero — processes may have already exited"
|
|
fi
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# ━━━ $ICON_START $ICON_CONTAINERS Local Container Recovery ━━━
|
|
# Only runs if rsync was actually killed locally — containers may have been left stopped
|
|
# by the interrupted rsync run. Checks all containers across all profiles and restarts
|
|
# any that are currently stopped. Remote containers are left in their current state.
|
|
# -----------------------------------------------------------------------------------------------
|
|
CONTAINERS_RESTARTED=()
|
|
|
|
if [[ "$LOCAL_KILLED" == true ]]; then
|
|
echo ""
|
|
echo "━━━ $ICON_START $ICON_CONTAINERS Local Container Recovery ━━━"
|
|
info "Rsync was killed locally — checking all profile containers..."
|
|
|
|
# Build deduplicated list of all containers across all profiles
|
|
declare -A SEEN
|
|
ALL_CONTAINERS=()
|
|
|
|
for profile_containers in "${PROFILE_CRITICAL_CONTAINER_NAMES[@]}"; do
|
|
read -r -a container_list <<< "$profile_containers"
|
|
for c in "${container_list[@]}"; do
|
|
[[ -z "$c" ]] && continue
|
|
if [[ -z "${SEEN[$c]:-}" ]]; then
|
|
SEEN[$c]=1
|
|
ALL_CONTAINERS+=("$c")
|
|
fi
|
|
done
|
|
done
|
|
|
|
if [[ ${#ALL_CONTAINERS[@]} -eq 0 ]]; then
|
|
info "No containers defined across any profile — skipping recovery"
|
|
else
|
|
for c in "${ALL_CONTAINERS[@]}"; do
|
|
info "Checking $c..."
|
|
|
|
STATUS=$(docker inspect -f '{{.State.Running}}' "$c" 2>/dev/null || echo "unknown")
|
|
|
|
if [[ "$STATUS" == "true" ]]; then
|
|
echo "$ICON_RUNNING $c is running — no action needed"
|
|
elif [[ "$STATUS" == "false" ]]; then
|
|
echo "$ICON_NOT_RUNNING $c is stopped — restarting..."
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would restart $c"
|
|
else
|
|
if docker start "$c" >/dev/null 2>&1; then
|
|
echo "$ICON_STARTED $c restarted"
|
|
CONTAINERS_RESTARTED+=("$c")
|
|
else
|
|
error "Failed to restart $c"
|
|
fi
|
|
fi
|
|
else
|
|
warn "$c state unknown — may not exist on this machine, skipping"
|
|
fi
|
|
done
|
|
fi
|
|
elif [[ "$DRY_RUN" == false ]]; then
|
|
info "No local rsync was killed — skipping container recovery"
|
|
fi
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# ━━━ $ICON_SUMMARY Summary ━━━
|
|
# -----------------------------------------------------------------------------------------------
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY RSYNC STOP SUMMARY ━━━━━"
|
|
echo "$ICON_HOST Local ($LOCAL_SERVER_NAME):"
|
|
if [[ "$LOCAL_KILLED" == true ]]; then
|
|
echo " $ICON_STOPPED Rsync killed"
|
|
elif [[ "$DRY_RUN" == true ]]; then
|
|
echo " $ICON_WARN Dry run — no changes made"
|
|
else
|
|
echo " $ICON_SUCCESS No rsync running"
|
|
fi
|
|
|
|
echo "$ICON_NET Remote ($REMOTE_SERVER_NAME):"
|
|
if [[ "$REMOTE_REACHABLE" == false ]]; then
|
|
echo " $ICON_WARN Unreachable — state unknown"
|
|
elif [[ "$REMOTE_KILLED" == true ]]; then
|
|
echo " $ICON_STOPPED Rsync killed"
|
|
else
|
|
echo " $ICON_SUCCESS No rsync running"
|
|
fi
|
|
|
|
if [[ ${#CONTAINERS_RESTARTED[@]} -gt 0 ]]; then
|
|
echo "$ICON_CONTAINERS Containers restarted locally:"
|
|
for c in "${CONTAINERS_RESTARTED[@]}"; do
|
|
echo " $ICON_STARTED $c"
|
|
done
|
|
elif [[ "$LOCAL_KILLED" == true ]]; then
|
|
echo "$ICON_CONTAINERS No containers needed restarting"
|
|
fi
|
|
|
|
echo "$ICON_TIME Dry Run: $DRY_RUN"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" |