#!/bin/bash
# -----------------------------------------------------------------------------------------------
# --------------------------------- Rsync Stop Script ------------------------------------------
# -----------------------------------------------------------------------------------------------
# Stops rsync intelligently — auto-detects what's running and acts accordingly.
#
# Default behavior (just run it):
#   Detects if an orchestrator (daily/weekly) is running
#   If yes → kills rsync subprocess only
#             orchestrator sees rsync died → moves to next share or exits cleanly
#   If no  → kills rsync processes directly (solo rsync.sh run)
#   Cleans stale lock files
#   Recovers any containers left stopped by interrupted rsync
#
# --full-stop flag (nuclear):
#   Kills orchestrator first → then rsync
#   Use when: you want everything dead immediately
#   daily/weekly loop will NOT continue to next share
#
# Both local and remote are handled in one run.
# Remote containers left as-is — docker_watchdog.sh handles remote recovery.
#
# Flags:
#   (none)       ← smart mode — auto-detects, rsync-only if orchestrator running
#   --full-stop  ← nuclear — kill orchestrator + rsync
#   --dry-run    ← preview without changes
# -----------------------------------------------------------------------------------------------

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

source "$SCRIPT_DIR/../Master.conf"
source "$SCRIPT_DIR/../common.sh"

# Check for --full-stop before parse_args
FULL_STOP=false
FILTERED_ARGS=()
for arg in "$@"; do
    if [[ "$arg" == "--full-stop" ]]; then
        FULL_STOP=true
    else
        FILTERED_ARGS+=("$arg")
    fi
done

parse_args "${FILTERED_ARGS[@]}"

# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Setup ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_GEAR Setup ━━━"

detect_hosts
resolve_remote_ip

REMOTE_REACHABLE=true
if ! ping -c1 -W3 "$REMOTE_SERVER" &>/dev/null; then
    warn "$ICON_PING Remote $REMOTE_SERVER_NAME unreachable — will skip remote"
    REMOTE_REACHABLE=false
else
    info "$ICON_PING $REMOTE_SERVER_NAME reachable"
fi

[[ "$SHOW_STATUS" == true ]] && show_status && exit 0
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"

# -----------------------------------------------------------------------------------------------
# ━━━ Auto-detect orchestrators ━━━
# Check if daily or weekly is running on local and remote
# This determines default behavior
# -----------------------------------------------------------------------------------------------
# -----------------------------------------------------------------------------------------------
# detect_rsync_parent — scans all lock files, finds which running process has rsync as a child
# No hardcoded list — works for any orchestrator automatically
#
# Returns: "script_name:parent_pid" if found, empty if rsync running standalone
# -----------------------------------------------------------------------------------------------
detect_rsync_parent() {
    local found=""

    # Get all rsync PIDs running locally
    local rsync_pids
    rsync_pids=$(pgrep -x rsync 2>/dev/null || true)
    [[ -z "$rsync_pids" ]] && echo "" && return

    # Scan all lock files in LOCK_DIR
    for lockfile in "$LOCK_DIR"/*.lock; do
        [[ -f "$lockfile" ]] || continue

        local content pid locked_name
        content=$(cat "$lockfile" 2>/dev/null)
        pid="${content%%:*}"
        locked_name="${content##*:}"

        # Skip if PID dead or is itself a rsync lock
        [[ -z "$pid" ]] && continue
        ! kill -0 "$pid" 2>/dev/null && continue
        [[ "$locked_name" == rsync_* ]] && continue

        # Check if any rsync PID is a child of this lock's PID
        local children
        children=$(cat /proc/"$pid"/task/"$pid"/children 2>/dev/null || \
                   tr ' ' '\n' < /proc/"$pid"/children 2>/dev/null || true)

        # Walk the child tree — rsync may be a grandchild (bash → rsync.sh → rsync)
        local all_descendants
        all_descendants=$(pgrep -P "$pid" 2>/dev/null || true)

        # Check if any rsync PID is in the descendants
        while IFS= read -r rsync_pid; do
            [[ -z "$rsync_pid" ]] && continue
            if echo "$all_descendants" | grep -qw "$rsync_pid" 2>/dev/null || \
               [[ "$(cat /proc/"$rsync_pid"/status 2>/dev/null | awk '/^PPid:/{print $2}')" == "$pid" ]]; then
                found="$locked_name:$pid"
                break 2
            fi
        done <<< "$rsync_pids"
    done

    echo "$found"
}

detect_rsync_parent_remote() {
    [[ "$REMOTE_REACHABLE" != true ]] && echo "" && return

    # Run the same logic on remote via SSH
    local found
    found=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" bash << 'REMOTE_SCRIPT'
LOCK_DIR="/tmp/unraid_locks"
rsync_pids=$(pgrep -x rsync 2>/dev/null || true)
[[ -z "$rsync_pids" ]] && exit 0

for lockfile in "$LOCK_DIR"/*.lock; do
    [[ -f "$lockfile" ]] || continue
    content=$(cat "$lockfile" 2>/dev/null)
    pid="${content%%:*}"
    locked_name="${content##*:}"
    [[ -z "$pid" ]] && continue
    ! kill -0 "$pid" 2>/dev/null && continue
    [[ "$locked_name" == rsync_* ]] && continue

    all_descendants=$(pgrep -P "$pid" 2>/dev/null || true)
    while IFS= read -r rsync_pid; do
        [[ -z "$rsync_pid" ]] && continue
        ppid=$(awk '/^PPid:/{print $2}' /proc/"$rsync_pid"/status 2>/dev/null)
        if echo "$all_descendants" | grep -qw "$rsync_pid" 2>/dev/null || \
           [[ "$ppid" == "$pid" ]]; then
            echo "$locked_name:$pid"
            exit 0
        fi
    done <<< "$rsync_pids"
done
REMOTE_SCRIPT
    2>/dev/null)
    echo "$found"
}

LOCAL_ORCH=$(detect_rsync_parent)
REMOTE_ORCH=""
[[ "$REMOTE_REACHABLE" == true ]] && REMOTE_ORCH=$(detect_rsync_parent_remote)

# Determine mode
if [[ "$FULL_STOP" == true ]]; then
    MODE="full-stop"
    info "Mode: FULL STOP — orchestrator + rsync will be killed"
elif [[ -n "$LOCAL_ORCH" ]] || [[ -n "$REMOTE_ORCH" ]]; then
    MODE="rsync-only"
    [[ -n "$LOCAL_ORCH" ]] && info "Detected local orchestrator: ${LOCAL_ORCH%%:*} — rsync-only mode"
    [[ -n "$REMOTE_ORCH" ]] && info "Detected remote orchestrator: ${REMOTE_ORCH%%:*} — rsync-only mode"
    info "Orchestrator will continue after rsync is killed"
    info "Use --full-stop to also kill the orchestrator"
else
    MODE="rsync-only"
    info "No orchestrator detected — killing rsync directly"
fi

# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_STOP Kill Orchestrators (full-stop only) ━━━
# -----------------------------------------------------------------------------------------------
ORCHESTRATORS_KILLED=()
REMOTE_ORCHESTRATORS_KILLED=()

kill_orchestrator() {
    local script_name="$1"
    local pid="$2"
    local lockfile="$LOCK_DIR/${script_name}.lock"

    if [[ "$DRY_RUN" == true ]]; then
        warn "DRY RUN — would kill $script_name (PID $pid)"
        return 0
    fi

    kill -TERM "$pid" 2>/dev/null
    sleep 2
    kill -0 "$pid" 2>/dev/null && kill -KILL "$pid" 2>/dev/null
    sleep 1

    if ! kill -0 "$pid" 2>/dev/null; then
        success "$script_name stopped ✅"
        rm -f "$lockfile"
        return 0
    else
        error "Failed to kill $script_name (PID $pid)"
        return 1
    fi
}

if [[ "$MODE" == "full-stop" ]]; then
    echo ""
    echo "━━━ $ICON_STOP Orchestrators ━━━"

    # Local
    if [[ -n "$LOCAL_ORCH" ]]; then
        name="${LOCAL_ORCH%%:*}"
        pid="${LOCAL_ORCH##*:}"
        info "Killing local: $name (PID $pid)"
        if kill_orchestrator "$name" "$pid"; then
            ORCHESTRATORS_KILLED+=("$name")
        fi
    else
        info "No local orchestrator running"
    fi

    # Remote
    if [[ "$REMOTE_REACHABLE" == true ]] && [[ -n "$REMOTE_ORCH" ]]; then
        name="${REMOTE_ORCH%%:*}"
        pid="${REMOTE_ORCH##*:}"
        lockfile="$LOCK_DIR/${name}.lock"
        info "Killing remote: $name (PID $pid)"
        if [[ "$DRY_RUN" == false ]]; then
            ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
                "kill -TERM '$pid' 2>/dev/null; sleep 2; \
                 kill -0 '$pid' 2>/dev/null && kill -KILL '$pid' 2>/dev/null; \
                 rm -f '$lockfile'" 2>/dev/null
            success "Remote $name stopped ✅"
            REMOTE_ORCHESTRATORS_KILLED+=("$name")
        else
            warn "DRY RUN — would kill remote $name (PID $pid)"
        fi
    elif [[ "$REMOTE_REACHABLE" == true ]]; then
        info "No remote orchestrator running"
    fi

    # Wait for subprocesses to settle
    if [[ ${#ORCHESTRATORS_KILLED[@]} -gt 0 ]] || \
       [[ ${#REMOTE_ORCHESTRATORS_KILLED[@]} -gt 0 ]]; then
        info "Waiting 3s for subprocesses to settle..."
        sleep 3
    fi
fi

# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_STOP Local Rsync ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_STOP Local Rsync ━━━"

LOCAL_KILLED=false
LOCAL_PIDS=$(pgrep -x rsync || true)

if [[ -z "$LOCAL_PIDS" ]]; then
    info "No rsync processes running locally"
else
    info "Found PIDs: $(echo "$LOCAL_PIDS" | tr '\n' ' ')"
    if [[ "$DRY_RUN" == true ]]; then
        warn "DRY RUN — would kill local rsync"
    else
        if pkill -x rsync; then
            success "Local rsync killed ✅"
            LOCAL_KILLED=true
        else
            warn "pkill non-zero — may have already exited"
        fi
    fi
fi

# Clean stale rsync lock files
for lockfile in "$LOCK_DIR"/rsync_*.lock; do
    [[ -f "$lockfile" ]] || continue
    content=$(cat "$lockfile" 2>/dev/null)
    pid="${content%%:*}"
    if [[ -n "$pid" ]] && ! kill -0 "$pid" 2>/dev/null; then
        info "Cleaning stale lock: $(basename "$lockfile")"
        [[ "$DRY_RUN" == false ]] && rm -f "$lockfile"
    fi
done

# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_STOP Remote Rsync ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_STOP Remote Rsync ($REMOTE_SERVER_NAME) ━━━"

REMOTE_KILLED=false

if [[ "$REMOTE_REACHABLE" == false ]]; then
    warn "Skipping — $REMOTE_SERVER_NAME unreachable"
else
    REMOTE_PIDS=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
        "pgrep -x rsync || true" 2>/dev/null || true)

    if [[ -z "$REMOTE_PIDS" ]]; then
        info "No rsync running on $REMOTE_SERVER_NAME"
    else
        info "Found PIDs on $REMOTE_SERVER_NAME: $(echo "$REMOTE_PIDS" | tr '\n' ' ')"
        if [[ "$DRY_RUN" == true ]]; then
            warn "DRY RUN — would kill remote rsync"
        else
            if ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" "pkill -x rsync || true" 2>/dev/null; then
                success "Remote rsync killed ✅"
                REMOTE_KILLED=true
            else
                warn "Remote pkill non-zero — may have already exited"
            fi
        fi
    fi
fi

# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_START $ICON_CONTAINERS Container Recovery ━━━
# Restart containers left stopped by interrupted rsync
# Only runs if something was actually killed locally
# Remote containers left as-is — docker_watchdog.sh handles remote
# -----------------------------------------------------------------------------------------------
CONTAINERS_RESTARTED=()

if [[ "$LOCAL_KILLED" == true ]] || [[ ${#ORCHESTRATORS_KILLED[@]} -gt 0 ]]; then
    echo ""
    echo "━━━ $ICON_START $ICON_CONTAINERS Container Recovery ━━━"
    info "Checking all profile containers..."

    declare -A SEEN
    ALL_CONTAINERS=()

    for profile_containers in "${PROFILE_CRITICAL_CONTAINER_NAMES[@]}"; do
        read -r -a container_list <<< "$profile_containers"
        for c in "${container_list[@]}"; do
            [[ -z "$c" ]] && continue
            if [[ -z "${SEEN[$c]:-}" ]]; then
                SEEN[$c]=1
                ALL_CONTAINERS+=("$c")
            fi
        done
    done

    if [[ ${#ALL_CONTAINERS[@]} -eq 0 ]]; then
        info "No containers defined — skipping recovery"
    else
        for c in "${ALL_CONTAINERS[@]}"; do
            STATUS=$(docker inspect -f '{{.State.Running}}' "$c" 2>/dev/null || echo "unknown")
            if [[ "$STATUS" == "true" ]]; then
                info "$ICON_RUNNING $c — running ✅"
            elif [[ "$STATUS" == "false" ]]; then
                warn "$ICON_NOT_RUNNING $c — stopped, restarting..."
                if [[ "$DRY_RUN" == true ]]; then
                    warn "DRY RUN — would restart $c"
                else
                    if docker start "$c" >/dev/null 2>&1; then
                        success "$c restarted ✅"
                        CONTAINERS_RESTARTED+=("$c")
                    else
                        error "Failed to restart $c"
                    fi
                fi
            else
                info "$c not found on this host — skipping"
            fi
        done
    fi
fi

# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Summary ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━━━ $ICON_SUMMARY RSYNC STOP SUMMARY ━━━━━"
echo "  Mode: $MODE"
echo ""

echo "$ICON_HOST  Local ($LOCAL_SERVER_NAME):"
[[ ${#ORCHESTRATORS_KILLED[@]} -gt 0 ]] && \
    echo "         $ICON_STOPPED Orchestrators killed: ${ORCHESTRATORS_KILLED[*]}"
[[ "$LOCAL_KILLED" == true ]] && \
    echo "         $ICON_STOPPED Rsync killed" || \
    echo "         $ICON_SUCCESS No rsync was running"

echo "$ICON_NET   Remote ($REMOTE_SERVER_NAME):"
if [[ "$REMOTE_REACHABLE" == false ]]; then
    echo "         $ICON_WARN    Unreachable — skipped"
else
    [[ ${#REMOTE_ORCHESTRATORS_KILLED[@]} -gt 0 ]] && \
        echo "         $ICON_STOPPED Orchestrators killed: ${REMOTE_ORCHESTRATORS_KILLED[*]}"
    [[ "$REMOTE_KILLED" == true ]] && \
        echo "         $ICON_STOPPED Rsync killed" || \
        echo "         $ICON_SUCCESS No rsync was running"
fi

[[ ${#CONTAINERS_RESTARTED[@]} -gt 0 ]] && \
    echo "$ICON_CONTAINERS Containers recovered: ${CONTAINERS_RESTARTED[*]}"

if [[ "$DRY_RUN" == true ]]; then
    echo "$ICON_WARN  Status: DRY RUN — no changes made"
else
    echo "$ICON_DONE  Status: $ICON_SUCCESS DONE"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

if [[ "$LOCAL_KILLED" == true ]] || [[ "$REMOTE_KILLED" == true ]] || \
   [[ ${#ORCHESTRATORS_KILLED[@]} -gt 0 ]]; then
    notify "Rsync stopped on $(hostname) — mode: $MODE — ${#CONTAINERS_RESTARTED[@]} containers recovered" \
        "Rsync Stop" "warning"
fi