#!/bin/bash
# -----------------------------------------------------------------------------------------------
# ----------------------- Common Helpers for Unraid Scripts -------------------------------------
# -----------------------------------------------------------------------------------------------

# ----------------------------- Logging -----------------------------
log() {
    [[ "$ENABLE_LOGGING" == true ]] && echo "[LOG] $(date '+%Y-%m-%d %H:%M:%S') $*"
}

# ----------------------------- Argument Parsing -----------------------------
parse_args() {
    DRY_RUN=${DRY_RUN:-false}
    for ARG in "$@"; do
        if [[ "$ARG" == *=* ]]; then
            VAR_NAME="${ARG%%=*}"
            VAR_VALUE="${ARG#*=}"
            if declare -p "$VAR_NAME" &>/dev/null; then
                printf -v "$VAR_NAME" '%s' "$VAR_VALUE"
                log "Overriding $VAR_NAME -> $VAR_VALUE"
            else
                echo "Warning: Unknown variable $VAR_NAME, ignoring."
            fi
        else
            case "$ARG" in
                --dry-run|-n) DRY_RUN=true ;;
                --help|-h) echo "Usage: script [--dry-run|-n] [VAR=value ...]"; exit 0 ;;
                *) echo "Warning: Unknown argument $ARG" ;;
            esac
        fi
    done
}

require_var() {
    local var="$1"
    if [[ -z "${!var:-}" ]]; then
        echo "Error: Required variable $var is not set."
        exit 1
    fi
}

validate_int() {
    local name="$1"
    local value="$2"
    if ! [[ "${value:-}" =~ ^[0-9]+$ ]]; then
        echo "Error: $name must be a non-negative integer."
        exit 1
    fi
}

# ----------------------------- Host Detection & SSH -----------------------------
detect_hosts() {
    LOCAL_HOSTNAME="$(hostname)"
    if [[ "$LOCAL_HOSTNAME" == "$HOST1" ]]; then
        LOCAL_SERVER_NAME="$HOST1"
        REMOTE_SERVER_NAME="$HOST2"
    elif [[ "$LOCAL_HOSTNAME" == "$HOST2" ]]; then
        LOCAL_SERVER_NAME="$HOST2"
        REMOTE_SERVER_NAME="$HOST1"
    else
        echo "Error: Local hostname ($LOCAL_HOSTNAME) not recognized."
        exit 1
    fi

    declare -A SSH_KEYS
    SSH_KEYS["$HOST1|$HOST2"]="$HOST1_SSH_KEY"
    SSH_KEYS["$HOST2|$HOST1"]="$HOST2_SSH_KEY"

    KEY_ID="$LOCAL_SERVER_NAME|$REMOTE_SERVER_NAME"
    SSH_KEY="${SSH_KEYS[$KEY_ID]}"
    if [[ -z "$SSH_KEY" ]]; then
        echo "Error: No SSH key defined for $LOCAL_SERVER_NAME → $REMOTE_SERVER_NAME"
        exit 1
    fi

    echo "Local server: $LOCAL_SERVER_NAME, Remote server: $REMOTE_SERVER_NAME, SSH key: $SSH_KEY"
    log "Detected hosts successfully"
}

# ----------------------------- Remote Server IP -----------------------------
resolve_remote_ip() {
    REMOTE_SERVER=$(tailscale ip -4 "$REMOTE_SERVER_NAME" 2>/dev/null)
    if [[ -z "$REMOTE_SERVER" ]]; then
        echo "Error: Could not resolve Tailscale IP for $REMOTE_SERVER_NAME"
        exit 1
    fi
    echo "Remote IP: $REMOTE_SERVER"
    log "Resolved Tailscale IP successfully"
}

# ----------------------------- Container Helpers -----------------------------
declare -a RUNNING_CONTAINERS=()

stop_containers() {
    echo "Checking critical containers on $REMOTE_SERVER_NAME..."
    RUNNING_CONTAINERS=()
    for container in "${CRITICAL_CONTAINER_NAMES[@]}"; do
        STATUS=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
            "docker inspect -f '{{.State.Running}}' $container 2>/dev/null" || echo "false")
        log "Container $container running: $STATUS"
        if [[ "$STATUS" == "true" ]]; then
            echo "Stopping $container"
            RUNNING_CONTAINERS+=("$container")
            ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" "docker stop $container"
        fi
    done
}

start_containers() {
    echo "Restarting containers on $REMOTE_SERVER_NAME..."
    for container in "${RUNNING_CONTAINERS[@]}"; do
        if [[ " ${DELAYED_CONTAINERS[*]} " == *" $container "* ]]; then
            echo "Delaying start of $container by ${CONTAINER_DELAY}s..."
            sleep "$CONTAINER_DELAY"
        fi
        ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" "docker start $container"
        log "Container $container started"
    done
}

# ----------------------------- Rsync Options Builder -----------------------------
get_rsync_opts() {
    if [[ -n "${PROFILE_RSYNC_OPTS[$PROFILE_NAME]:-}" ]]; then
        read -r -a RSYNC_OPTS <<< "${PROFILE_RSYNC_OPTS[$PROFILE_NAME]}"
        log "Using profile-specific rsync options: ${RSYNC_OPTS[*]}"
    else
        RSYNC_OPTS=("${DEFAULT_RSYNC_OPTS[@]}")
        log "Using default rsync options: ${RSYNC_OPTS[*]}"
    fi
}
3️⃣ rsync.sh

Minimal user-facing echo + detailed logging via log:

#!/bin/bash
set -e

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

# Load configs & helpers
source "$SCRIPT_DIR/../Master.conf"
source "$SCRIPT_DIR/../common.sh"

# ----------------------------- Input -----------------------------
DIRECTORY="$1"
if [[ -z "$DIRECTORY" ]]; then
    echo "Usage: $0 <directory-to-sync> [--dry-run] [VAR=value ...]"
    exit 1
fi
shift
parse_args "$@"

# ----------------------------- Hosts & IP -----------------------------
detect_hosts
resolve_remote_ip

# ----------------------------- Profile -----------------------------
PROFILE_NAME=$(basename "$DIRECTORY" | tr '[:upper:]' '[:lower:]')

MAX_RSYNC_PROCS=${PROFILE_MAX_PROCS[$PROFILE_NAME]:-$MAX_RSYNC_PROCS}
BW_LIMIT=${PROFILE_BW_LIMIT[$PROFILE_NAME]:-$BW_LIMIT}
CRITICAL_CONTAINER_NAMES=(${PROFILE_CRITICAL_CONTAINER_NAMES[$PROFILE_NAME]})
DELAYED_CONTAINERS=(${PROFILE_DELAYED_CONTAINERS[$PROFILE_NAME]})
CONTAINER_DELAY=${PROFILE_CONTAINER_DELAY[$PROFILE_NAME]:-$CONTAINER_DELAY}
RETRY_COUNT=${PROFILE_RETRY_COUNT[$PROFILE_NAME]:-$RETRY_COUNT}
SLEEP=${PROFILE_SLEEP[$PROFILE_NAME]:-$SLEEP}
EXCLUDE_DIRS=(${PROFILE_EXCLUDE_DIRS[$PROFILE_NAME]:-${EXCLUDE_DIRS[@]}})

echo "Detected profile: $PROFILE_NAME"
log "Profile settings: MAX_RSYNC_PROCS=$MAX_RSYNC_PROCS, BW_LIMIT=$BW_LIMIT, CONTAINERS=${CRITICAL_CONTAINER_NAMES[*]}, EXCLUDES=${EXCLUDE_DIRS[*]}, DRY_RUN=$DRY_RUN"

# ----------------------------- Rsync Wait -----------------------------
wait_for_rsync() {
    for attempt in $(seq 1 "$RETRY_COUNT"); do
        RSYNC_COUNT=$(pgrep -fc "rsync.*root@${REMOTE_SERVER}" || true)
        log "Current rsync processes: $RSYNC_COUNT"
        if [ "$RSYNC_COUNT" -ge "$MAX_RSYNC_PROCS" ]; then
            echo "Waiting for available rsync slot ($RSYNC_COUNT active)..."
            sleep "$SLEEP"
        else
            return 0
        fi
    done
    echo "Too many rsync processes, exiting."
    exit 1
}

# ----------------------------- Main -----------------------------
main() {
    echo "Starting sync: $DIRECTORY → $REMOTE_SERVER"
    wait_for_rsync
    stop_containers
    get_rsync_opts

    for ex in "${EXCLUDE_DIRS[@]}"; do RSYNC_OPTS+=(--exclude="$ex"); done
    [ "$DRY_RUN" = true ] && RSYNC_OPTS+=("--dry-run")
    log "Rsync options: ${RSYNC_OPTS[*]}"

    for attempt in $(seq 1 "$RETRY_COUNT"); do
        echo "Rsync attempt $attempt/$RETRY_COUNT..."
        if rsync "${RSYNC_OPTS[@]}" -e "ssh -i \"$SSH_KEY\" -T -o Compression=no -o IPQoS=throughput" \
            "$DIRECTORY" "root@${REMOTE_SERVER}:$DIRECTORY"; then
            echo "Rsync completed successfully."
            break
        else
            echo "Rsync attempt $attempt failed."
            if [ "$attempt" -lt "$RETRY_COUNT" ]; then
                echo "Retrying in $SLEEP seconds..."
                sleep "$SLEEP"
            else
                echo "All $RETRY_COUNT rsync attempts failed."
                start_containers
                exit 1
            fi
        fi
    done

    start_containers
    echo "Sync complete."
    log "[$LOCAL_SERVER_NAME] Sync finished for $DIRECTORY → $REMOTE_SERVER"
}

main