diff --git a/unRAID_Essentials/rsync_stop.sh b/unRAID_Essentials/rsync_stop.sh index 2a6745f..f4a350f 100644 --- a/unRAID_Essentials/rsync_stop.sh +++ b/unRAID_Essentials/rsync_stop.sh @@ -2,12 +2,28 @@ # ----------------------------------------------------------------------------------------------- # --------------------------------- Rsync Stop Script ------------------------------------------ # ----------------------------------------------------------------------------------------------- -# Stops all running rsync processes on both the local and remote server. -# Used during array stop or manually when rsync needs to be forcefully terminated. -# If rsync processes were killed locally, checks all profile containers and restarts any -# that were left stopped by the interrupted rsync run. -# Remote is killed but left in whatever container state it is in — secondary is self-healing. -# Supports --dry-run to preview what would be killed without making changes. +# Stops rsync intelligently — auto-detects what's running and acts accordingly. +# +# Default behavior (just run it): +# Detects if an orchestrator (daily/weekly) is running +# If yes → kills rsync subprocess only +# orchestrator sees rsync died → moves to next share or exits cleanly +# If no → kills rsync processes directly (solo rsync.sh run) +# Cleans stale lock files +# Recovers any containers left stopped by interrupted rsync +# +# --full-stop flag (nuclear): +# Kills orchestrator first → then rsync +# Use when: you want everything dead immediately +# daily/weekly loop will NOT continue to next share +# +# Both local and remote are handled in one run. +# Remote containers left as-is — docker_watchdog.sh handles remote recovery. +# +# Flags: +# (none) ← smart mode — auto-detects, rsync-only if orchestrator running +# --full-stop ← nuclear — kill orchestrator + rsync +# --dry-run ← preview without changes # ----------------------------------------------------------------------------------------------- SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -15,7 +31,18 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/../Master.conf" source "$SCRIPT_DIR/../common.sh" -parse_args "$@" +# Check for --full-stop before parse_args +FULL_STOP=false +FILTERED_ARGS=() +for arg in "$@"; do + if [[ "$arg" == "--full-stop" ]]; then + FULL_STOP=true + else + FILTERED_ARGS+=("$arg") + fi +done + +parse_args "${FILTERED_ARGS[@]}" # ----------------------------------------------------------------------------------------------- # ━━━ $ICON_GEAR Setup ━━━ @@ -26,18 +53,202 @@ echo "━━━ $ICON_GEAR Setup ━━━" detect_hosts resolve_remote_ip -# Check connectivity but do not exit on failure — remote may already be going down REMOTE_REACHABLE=true if ! ping -c1 -W3 "$REMOTE_SERVER" &>/dev/null; then - warn "$ICON_PING Remote $REMOTE_SERVER_NAME is unreachable — will skip remote kill" + warn "$ICON_PING Remote $REMOTE_SERVER_NAME unreachable — will skip remote" REMOTE_REACHABLE=false else - info "$ICON_PING $REMOTE_SERVER_NAME is reachable" + info "$ICON_PING $REMOTE_SERVER_NAME reachable" fi [[ "$SHOW_STATUS" == true ]] && show_status && exit 0 [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made" +# ----------------------------------------------------------------------------------------------- +# ━━━ Auto-detect orchestrators ━━━ +# Check if daily or weekly is running on local and remote +# This determines default behavior +# ----------------------------------------------------------------------------------------------- +# ----------------------------------------------------------------------------------------------- +# detect_rsync_parent — scans all lock files, finds which running process has rsync as a child +# No hardcoded list — works for any orchestrator automatically +# +# Returns: "script_name:parent_pid" if found, empty if rsync running standalone +# ----------------------------------------------------------------------------------------------- +detect_rsync_parent() { + local found="" + + # Get all rsync PIDs running locally + local rsync_pids + rsync_pids=$(pgrep -x rsync 2>/dev/null || true) + [[ -z "$rsync_pids" ]] && echo "" && return + + # Scan all lock files in LOCK_DIR + for lockfile in "$LOCK_DIR"/*.lock; do + [[ -f "$lockfile" ]] || continue + + local content pid locked_name + content=$(cat "$lockfile" 2>/dev/null) + pid="${content%%:*}" + locked_name="${content##*:}" + + # Skip if PID dead or is itself a rsync lock + [[ -z "$pid" ]] && continue + ! kill -0 "$pid" 2>/dev/null && continue + [[ "$locked_name" == rsync_* ]] && continue + + # Check if any rsync PID is a child of this lock's PID + local children + children=$(cat /proc/"$pid"/task/"$pid"/children 2>/dev/null || \ + tr ' ' '\n' < /proc/"$pid"/children 2>/dev/null || true) + + # Walk the child tree — rsync may be a grandchild (bash → rsync.sh → rsync) + local all_descendants + all_descendants=$(pgrep -P "$pid" 2>/dev/null || true) + + # Check if any rsync PID is in the descendants + while IFS= read -r rsync_pid; do + [[ -z "$rsync_pid" ]] && continue + if echo "$all_descendants" | grep -qw "$rsync_pid" 2>/dev/null || \ + [[ "$(cat /proc/"$rsync_pid"/status 2>/dev/null | awk '/^PPid:/{print $2}')" == "$pid" ]]; then + found="$locked_name:$pid" + break 2 + fi + done <<< "$rsync_pids" + done + + echo "$found" +} + +detect_rsync_parent_remote() { + [[ "$REMOTE_REACHABLE" != true ]] && echo "" && return + + # Run the same logic on remote via SSH + local found + found=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" bash << 'REMOTE_SCRIPT' +LOCK_DIR="/tmp/unraid_locks" +rsync_pids=$(pgrep -x rsync 2>/dev/null || true) +[[ -z "$rsync_pids" ]] && exit 0 + +for lockfile in "$LOCK_DIR"/*.lock; do + [[ -f "$lockfile" ]] || continue + content=$(cat "$lockfile" 2>/dev/null) + pid="${content%%:*}" + locked_name="${content##*:}" + [[ -z "$pid" ]] && continue + ! kill -0 "$pid" 2>/dev/null && continue + [[ "$locked_name" == rsync_* ]] && continue + + all_descendants=$(pgrep -P "$pid" 2>/dev/null || true) + while IFS= read -r rsync_pid; do + [[ -z "$rsync_pid" ]] && continue + ppid=$(awk '/^PPid:/{print $2}' /proc/"$rsync_pid"/status 2>/dev/null) + if echo "$all_descendants" | grep -qw "$rsync_pid" 2>/dev/null || \ + [[ "$ppid" == "$pid" ]]; then + echo "$locked_name:$pid" + exit 0 + fi + done <<< "$rsync_pids" +done +REMOTE_SCRIPT + 2>/dev/null) + echo "$found" +} + +LOCAL_ORCH=$(detect_rsync_parent) +REMOTE_ORCH="" +[[ "$REMOTE_REACHABLE" == true ]] && REMOTE_ORCH=$(detect_rsync_parent_remote) + +# Determine mode +if [[ "$FULL_STOP" == true ]]; then + MODE="full-stop" + info "Mode: FULL STOP — orchestrator + rsync will be killed" +elif [[ -n "$LOCAL_ORCH" ]] || [[ -n "$REMOTE_ORCH" ]]; then + MODE="rsync-only" + [[ -n "$LOCAL_ORCH" ]] && info "Detected local orchestrator: ${LOCAL_ORCH%%:*} — rsync-only mode" + [[ -n "$REMOTE_ORCH" ]] && info "Detected remote orchestrator: ${REMOTE_ORCH%%:*} — rsync-only mode" + info "Orchestrator will continue after rsync is killed" + info "Use --full-stop to also kill the orchestrator" +else + MODE="rsync-only" + info "No orchestrator detected — killing rsync directly" +fi + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_STOP Kill Orchestrators (full-stop only) ━━━ +# ----------------------------------------------------------------------------------------------- +ORCHESTRATORS_KILLED=() +REMOTE_ORCHESTRATORS_KILLED=() + +kill_orchestrator() { + local script_name="$1" + local pid="$2" + local lockfile="$LOCK_DIR/${script_name}.lock" + + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would kill $script_name (PID $pid)" + return 0 + fi + + kill -TERM "$pid" 2>/dev/null + sleep 2 + kill -0 "$pid" 2>/dev/null && kill -KILL "$pid" 2>/dev/null + sleep 1 + + if ! kill -0 "$pid" 2>/dev/null; then + success "$script_name stopped ✅" + rm -f "$lockfile" + return 0 + else + error "Failed to kill $script_name (PID $pid)" + return 1 + fi +} + +if [[ "$MODE" == "full-stop" ]]; then + echo "" + echo "━━━ $ICON_STOP Orchestrators ━━━" + + # Local + if [[ -n "$LOCAL_ORCH" ]]; then + name="${LOCAL_ORCH%%:*}" + pid="${LOCAL_ORCH##*:}" + info "Killing local: $name (PID $pid)" + if kill_orchestrator "$name" "$pid"; then + ORCHESTRATORS_KILLED+=("$name") + fi + else + info "No local orchestrator running" + fi + + # Remote + if [[ "$REMOTE_REACHABLE" == true ]] && [[ -n "$REMOTE_ORCH" ]]; then + name="${REMOTE_ORCH%%:*}" + pid="${REMOTE_ORCH##*:}" + lockfile="$LOCK_DIR/${name}.lock" + info "Killing remote: $name (PID $pid)" + if [[ "$DRY_RUN" == false ]]; then + ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ + "kill -TERM '$pid' 2>/dev/null; sleep 2; \ + kill -0 '$pid' 2>/dev/null && kill -KILL '$pid' 2>/dev/null; \ + rm -f '$lockfile'" 2>/dev/null + success "Remote $name stopped ✅" + REMOTE_ORCHESTRATORS_KILLED+=("$name") + else + warn "DRY RUN — would kill remote $name (PID $pid)" + fi + elif [[ "$REMOTE_REACHABLE" == true ]]; then + info "No remote orchestrator running" + fi + + # Wait for subprocesses to settle + if [[ ${#ORCHESTRATORS_KILLED[@]} -gt 0 ]] || \ + [[ ${#REMOTE_ORCHESTRATORS_KILLED[@]} -gt 0 ]]; then + info "Waiting 3s for subprocesses to settle..." + sleep 3 + fi +fi + # ----------------------------------------------------------------------------------------------- # ━━━ $ICON_STOP Local Rsync ━━━ # ----------------------------------------------------------------------------------------------- @@ -48,22 +259,32 @@ LOCAL_KILLED=false LOCAL_PIDS=$(pgrep -x rsync || true) if [[ -z "$LOCAL_PIDS" ]]; then - info "No rsync processes running locally — nothing to kill" + info "No rsync processes running locally" else - info "Found rsync processes locally: $(echo "$LOCAL_PIDS" | tr '\n' ' ')" - + info "Found PIDs: $(echo "$LOCAL_PIDS" | tr '\n' ' ')" if [[ "$DRY_RUN" == true ]]; then - warn "DRY RUN — would kill local rsync processes" + warn "DRY RUN — would kill local rsync" else if pkill -x rsync; then - success "Local rsync processes killed" + success "Local rsync killed ✅" LOCAL_KILLED=true else - warn "pkill returned non-zero — processes may have already exited" + warn "pkill non-zero — may have already exited" fi fi fi +# Clean stale rsync lock files +for lockfile in "$LOCK_DIR"/rsync_*.lock; do + [[ -f "$lockfile" ]] || continue + content=$(cat "$lockfile" 2>/dev/null) + pid="${content%%:*}" + if [[ -n "$pid" ]] && ! kill -0 "$pid" 2>/dev/null; then + info "Cleaning stale lock: $(basename "$lockfile")" + [[ "$DRY_RUN" == false ]] && rm -f "$lockfile" + fi +done + # ----------------------------------------------------------------------------------------------- # ━━━ $ICON_STOP Remote Rsync ━━━ # ----------------------------------------------------------------------------------------------- @@ -73,43 +294,41 @@ echo "━━━ $ICON_STOP Remote Rsync ($REMOTE_SERVER_NAME) ━━━" REMOTE_KILLED=false if [[ "$REMOTE_REACHABLE" == false ]]; then - warn "Skipping remote kill — $REMOTE_SERVER_NAME unreachable" + warn "Skipping — $REMOTE_SERVER_NAME unreachable" else REMOTE_PIDS=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ "pgrep -x rsync || true" 2>/dev/null || true) if [[ -z "$REMOTE_PIDS" ]]; then - info "No rsync processes running on $REMOTE_SERVER_NAME — nothing to kill" + info "No rsync running on $REMOTE_SERVER_NAME" else - info "Found rsync processes on $REMOTE_SERVER_NAME: $(echo "$REMOTE_PIDS" | tr '\n' ' ')" - + info "Found PIDs on $REMOTE_SERVER_NAME: $(echo "$REMOTE_PIDS" | tr '\n' ' ')" if [[ "$DRY_RUN" == true ]]; then - warn "DRY RUN — would kill remote rsync processes" + warn "DRY RUN — would kill remote rsync" else if ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" "pkill -x rsync || true" 2>/dev/null; then - success "Remote rsync processes killed" + success "Remote rsync killed ✅" REMOTE_KILLED=true else - warn "Remote pkill returned non-zero — processes may have already exited" + warn "Remote pkill non-zero — may have already exited" fi fi fi fi # ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_START $ICON_CONTAINERS Local Container Recovery ━━━ -# Only runs if rsync was actually killed locally — containers may have been left stopped -# by the interrupted rsync run. Checks all containers across all profiles and restarts -# any that are currently stopped. Remote containers are left in their current state. +# ━━━ $ICON_START $ICON_CONTAINERS Container Recovery ━━━ +# Restart containers left stopped by interrupted rsync +# Only runs if something was actually killed locally +# Remote containers left as-is — docker_watchdog.sh handles remote # ----------------------------------------------------------------------------------------------- CONTAINERS_RESTARTED=() -if [[ "$LOCAL_KILLED" == true ]]; then +if [[ "$LOCAL_KILLED" == true ]] || [[ ${#ORCHESTRATORS_KILLED[@]} -gt 0 ]]; then echo "" - echo "━━━ $ICON_START $ICON_CONTAINERS Local Container Recovery ━━━" - info "Rsync was killed locally — checking all profile containers..." + echo "━━━ $ICON_START $ICON_CONTAINERS Container Recovery ━━━" + info "Checking all profile containers..." - # Build deduplicated list of all containers across all profiles declare -A SEEN ALL_CONTAINERS=() @@ -125,35 +344,29 @@ if [[ "$LOCAL_KILLED" == true ]]; then done if [[ ${#ALL_CONTAINERS[@]} -eq 0 ]]; then - info "No containers defined across any profile — skipping recovery" + info "No containers defined — skipping recovery" else for c in "${ALL_CONTAINERS[@]}"; do - info "Checking $c..." - STATUS=$(docker inspect -f '{{.State.Running}}' "$c" 2>/dev/null || echo "unknown") - if [[ "$STATUS" == "true" ]]; then - echo "$ICON_RUNNING $c is running — no action needed" + info "$ICON_RUNNING $c — running ✅" elif [[ "$STATUS" == "false" ]]; then - echo "$ICON_NOT_RUNNING $c is stopped — restarting..." - + warn "$ICON_NOT_RUNNING $c — stopped, restarting..." if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would restart $c" else if docker start "$c" >/dev/null 2>&1; then - echo "$ICON_STARTED $c restarted" + success "$c restarted ✅" CONTAINERS_RESTARTED+=("$c") else error "Failed to restart $c" fi fi else - warn "$c state unknown — may not exist on this machine, skipping" + info "$c not found on this host — skipping" fi done fi -elif [[ "$DRY_RUN" == false ]]; then - info "No local rsync was killed — skipping container recovery" fi # ----------------------------------------------------------------------------------------------- @@ -161,39 +374,39 @@ fi # ----------------------------------------------------------------------------------------------- echo "" echo "━━━━━ $ICON_SUMMARY RSYNC STOP SUMMARY ━━━━━" +echo " Mode: $MODE" +echo "" + echo "$ICON_HOST Local ($LOCAL_SERVER_NAME):" -if [[ "$LOCAL_KILLED" == true ]]; then - echo " $ICON_STOPPED Rsync killed" -elif [[ "$DRY_RUN" == true ]]; then - echo " $ICON_WARN Dry run — no changes made" -else - echo " $ICON_SUCCESS No rsync running" -fi +[[ ${#ORCHESTRATORS_KILLED[@]} -gt 0 ]] && \ + echo " $ICON_STOPPED Orchestrators killed: ${ORCHESTRATORS_KILLED[*]}" +[[ "$LOCAL_KILLED" == true ]] && \ + echo " $ICON_STOPPED Rsync killed" || \ + echo " $ICON_SUCCESS No rsync was running" echo "$ICON_NET Remote ($REMOTE_SERVER_NAME):" if [[ "$REMOTE_REACHABLE" == false ]]; then - echo " $ICON_WARN Unreachable — state unknown" -elif [[ "$REMOTE_KILLED" == true ]]; then - echo " $ICON_STOPPED Rsync killed" + echo " $ICON_WARN Unreachable — skipped" else - echo " $ICON_SUCCESS No rsync running" + [[ ${#REMOTE_ORCHESTRATORS_KILLED[@]} -gt 0 ]] && \ + echo " $ICON_STOPPED Orchestrators killed: ${REMOTE_ORCHESTRATORS_KILLED[*]}" + [[ "$REMOTE_KILLED" == true ]] && \ + echo " $ICON_STOPPED Rsync killed" || \ + echo " $ICON_SUCCESS No rsync was running" fi -if [[ ${#CONTAINERS_RESTARTED[@]} -gt 0 ]]; then - echo "$ICON_CONTAINERS Containers restarted locally:" - for c in "${CONTAINERS_RESTARTED[@]}"; do - echo " $ICON_STARTED $c" - done -elif [[ "$LOCAL_KILLED" == true ]]; then - echo "$ICON_CONTAINERS No containers needed restarting" -fi +[[ ${#CONTAINERS_RESTARTED[@]} -gt 0 ]] && \ + echo "$ICON_CONTAINERS Containers recovered: ${CONTAINERS_RESTARTED[*]}" -echo "$ICON_TIME Dry Run: $DRY_RUN" +if [[ "$DRY_RUN" == true ]]; then + echo "$ICON_WARN Status: DRY RUN — no changes made" +else + echo "$ICON_DONE Status: $ICON_SUCCESS DONE" +fi echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -# Notify based on what happened -if [[ "$LOCAL_KILLED" == true ]] || [[ "$REMOTE_KILLED" == true ]]; then - local_status=$([[ "$LOCAL_KILLED" == true ]] && echo "killed" || echo "clean") - remote_status=$([[ "$REMOTE_KILLED" == true ]] && echo "killed" || echo "clean") - notify "Rsync stopped — local: $local_status remote: $remote_status — ${#CONTAINERS_RESTARTED[@]} containers recovered" "Rsync Stop" "warning" +if [[ "$LOCAL_KILLED" == true ]] || [[ "$REMOTE_KILLED" == true ]] || \ + [[ ${#ORCHESTRATORS_KILLED[@]} -gt 0 ]]; then + notify "Rsync stopped on $(hostname) — mode: $MODE — ${#CONTAINERS_RESTARTED[@]} containers recovered" \ + "Rsync Stop" "warning" fi \ No newline at end of file