#!/bin/bash
# -----------------------------------------------------------------------------------------------
# ----------------- UNRAID OPS COMMON LIBRARY (STABLE FRAMEWORK v1) ----------------------------
# -----------------------------------------------------------------------------------------------
# Version:  2.3
# -----------------------------------------------------------------------------------------------
# Changelog:
#   v1.0 — Initial stable framework
#   v1.1 — format_duration moved here from daily_sync.sh for shared use
#           SSH_KEY collision resolved — gitea key renamed GITEA_SSH_KEY in Master.conf
#           Version and changelog tracking added
#   v1.2 — Consistent function header comment blocks across all functions
#           check_connectivity added as standalone function
#           check_connectivity friendlier error output with tailscale hint
#   v1.3 — check_remote_rootfs added — aborts if remote rootfs exceeds ROOTFS_WARN threshold
#           check_remote_share added — aborts if target directory is missing or empty on remote
#           Both protect against rsync running when remote array is down or drives are missing
#   v1.4 — check_remote_disks added — verifies all physical disks backing a share are mounted
#           Discovers disk layout automatically at runtime, no configuration required
#           Aborts if any single disk backing the share is offline or unmounted
#   v1.5 — Full icon set expanded — each operation and state has its own distinct icon
#           All function output updated to use correct icon per context
#           Icons grouped and commented by category for clarity
#   v1.6 — ICON_CONTAINERS added — 📦 anchors all container sections for visual consistency
#           ICON_NOT_RUNNING changed to ⭕ — distinct from ICON_STOPPED 🔴
#           Section dividers updated from --- to ━━━ for cleaner log readability
#           Summary passed/failed lines use ICON_SUCCESS and ICON_ERROR consistently
#   v1.7 — ICON_MOVER added for mover operations
#           ICON_CONTAINERS replaces ICON_DOCKER for docker/container operations
#           validate_int added — reusable integer validation for any script
#   v1.8 — ICON_PHP added for PHP-FPM operations
#   v1.9 — ICON_REBOOT added for server reboot operations
#   v2.0 — ICON_PLUGIN added for User Scripts plugin operations
#   v2.1 — ICON_ZFS and ICON_MEM added for ZFS and memory diagnostics
#           Diagnostics icon group added to icon block
#   v2.2 — ICON_WATCHDOG added for Docker watchdog monitoring operations
#   v2.3 — ICON_NOTIFY added for notification operations
#           notify() added — shared notification function supporting unRAID native and Discord
#           NOTIFY_UNRAID and DISCORD_WEBHOOK configured in Master.conf
# -----------------------------------------------------------------------------------------------

# -----------------------------------------------------------------------------------------------
# ICONS
# Each icon has one job — do not reuse across different contexts.
# -----------------------------------------------------------------------------------------------

# System / Host
ICON_HOST="🖥️"          # host detection
ICON_NET="🌐"           # network / IP resolution
ICON_PING="📡"          # connectivity check
ICON_GEAR="⚙️"          # setup section header / profile load

# Health Checks
ICON_DISK="💾"          # disk checks
ICON_HEALTH="🩺"        # rootfs / share health checks
ICON_SHIELD="🛡️"        # pre-flight section header

# Containers
ICON_CONTAINERS="📦"    # container section anchor — paired with action icon for direction
ICON_STOP="⛔"          # stop command being issued
ICON_STOPPED="🔴"       # container confirmed stopped
ICON_START="▶️"         # start command being issued
ICON_STARTED="💚"       # container confirmed started
ICON_RUNNING="🟢"       # container already running when checked
ICON_NOT_RUNNING="⭕"   # container already stopped when checked — distinct from ICON_STOPPED

# Transfer
ICON_SYNC="🔄"          # transfer section header
ICON_RUN="🚀"           # sync starting / rsync attempt
ICON_RETRY="🔁"         # retry attempt
ICON_DONE="🏁"          # transfer complete

# Summary
ICON_SUMMARY="📋"       # summary section header
ICON_TIME="⏱️"          # duration line

# System Operations
ICON_MOVER="🔃"         # mover operations
ICON_REBOOT="⚡"        # server reboot operations
ICON_PLUGIN="🧩"        # user scripts plugin operations
ICON_PHP="👥"           # PHP-FPM operations

# Diagnostics
ICON_ZFS="📊"           # ZFS ARC statistics
ICON_MEM="🧠"           # memory status
ICON_WATCHDOG="🐾"      # docker watchdog monitoring operations

# Notifications
ICON_NOTIFY="🔔"        # notification operations

# Output
ICON_INFO="ℹ️"
ICON_WARN="⚠️"
ICON_ERROR="❌"
ICON_SUCCESS="✅"

# -----------------------------------------------------------------------------------------------
# OUTPUT HELPERS
# Standardised output functions used across all scripts.
# log() is gated by ENABLE_LOGGING — set in Master.conf or via --log flag.
# -----------------------------------------------------------------------------------------------
info()    { echo "$ICON_INFO  [INFO]  $*"; }
warn()    { echo "$ICON_WARN  [WARN]  $*"; }
error()   { echo "$ICON_ERROR [ERROR] $*"; }
success() { echo "$ICON_SUCCESS [OK]  $*"; }

log() {
    [[ "${ENABLE_LOGGING:-false}" == true ]] && echo "[LOG] $*"
}

# -----------------------------------------------------------------------------------------------
# NOTIFICATION
# Sends a notification via unRAID native system and/or Discord webhook.
# Both channels are optional and independently controlled via Master.conf.
# unRAID native: requires NOTIFY_UNRAID=true and the dynamix notify script to be present.
# Discord: requires DISCORD_WEBHOOK to be set to a valid webhook URL.
# Severity levels: normal, warning, alert — maps to unRAID notification severity.
# Usage: notify "message" "subject" "severity"
#        notify "Rsync failed: Movies" "Rsync Alert" "alert"
#        notify "Daily sync complete" "Daily Sync" "normal"
# -----------------------------------------------------------------------------------------------
notify() {
    local message="$1"
    local subject="${2:-unRAID Notification}"
    local severity="${3:-normal}"

    log "$ICON_NOTIFY Sending notification: $subject — $message"

    # unRAID native notification
    if [[ "${NOTIFY_UNRAID:-false}" == true ]]; then
        local notify_script="/usr/local/emhttp/plugins/dynamix/scripts/notify"
        if [[ -x "$notify_script" ]]; then
            "$notify_script" -s "$subject" -d "$message" -i "$severity" 2>/dev/null
            log "$ICON_NOTIFY unRAID notification sent"
        else
            log "$ICON_NOTIFY unRAID notify script not found — skipping"
        fi
    fi

    # Discord webhook notification
    if [[ -n "${DISCORD_WEBHOOK:-}" ]]; then
        local hostname
        hostname=$(hostname)
        local payload
        payload=$(printf '{"content": "%s — **%s**\\n%s"}' \
            "$ICON_NOTIFY" "$subject" "$message")
        if curl -s -H "Content-Type: application/json" \
            -d "$payload" "$DISCORD_WEBHOOK" >/dev/null 2>&1; then
            log "$ICON_NOTIFY Discord notification sent"
        else
            warn "Discord notification failed — check DISCORD_WEBHOOK in Master.conf"
        fi
    fi
}

# -----------------------------------------------------------------------------------------------
# DURATION FORMATTER
# Converts raw seconds into a human readable string — e.g. 10m53s or 47s
# Used by rsync.sh summary and daily_sync.sh summary.
# Sourced from common.sh so both scripts share the same implementation.
# -----------------------------------------------------------------------------------------------
format_duration() {
    local secs=$1
    local mins=$((secs / 60))
    local rem=$((secs % 60))
    [[ $mins -gt 0 ]] && echo "${mins}m${rem}s" || echo "${rem}s"
}

# -----------------------------------------------------------------------------------------------
# ARG PARSER
# Processes all flags and key=value pairs passed to any script.
# Positional arguments (directory paths) are separated before calling this — see rsync.sh.
# Supported flags: --dry-run, --log, --no-log, --status, --help
# Supported key=value: LOG=true/false, or any declared variable e.g. BW_LIMIT=5000
# Unparsed positional args are returned in PARSED_ARGS array.
# -----------------------------------------------------------------------------------------------
parse_args() {
    ENABLE_LOGGING=${ENABLE_LOGGING:-false}
    DRY_RUN=${DRY_RUN:-false}
    SHOW_STATUS=${SHOW_STATUS:-false}

    CLEAN_ARGS=()

    for ARG in "$@"; do
        if [[ "$ARG" == *=* ]]; then
            VAR="${ARG%%=*}"
            VAL="${ARG#*=}"

            case "$VAR" in
                LOG)
                    [[ "$VAL" == "true" ]]  && ENABLE_LOGGING=true
                    [[ "$VAL" == "false" ]] && ENABLE_LOGGING=false
                    ;;
                *)
                    if declare -p "$VAR" &>/dev/null; then
                        printf -v "$VAR" '%s' "$VAL"
                        log "Set $VAR=$VAL"
                    else
                        warn "Unknown variable: $VAR"
                    fi
                    ;;
            esac
        else
            case "$ARG" in
                --dry-run|-n)       DRY_RUN=true ;;
                --log)              ENABLE_LOGGING=true ;;
                --no-log)           ENABLE_LOGGING=false ;;
                --status|--summary) SHOW_STATUS=true ;;
                --help|-h)
                    echo "Usage: script <dir> [--dry-run] [--log] [--status]"
                    exit 0
                    ;;
                *) CLEAN_ARGS+=("$ARG") ;;
            esac
        fi
    done

    PARSED_ARGS=("${CLEAN_ARGS[@]}")
}

# -----------------------------------------------------------------------------------------------
# VALIDATION
# Checks that a required variable is set and non-empty.
# Usage: require_var VAR_NAME
# Exits with error if the variable is missing.
# -----------------------------------------------------------------------------------------------
require_var() {
    [[ -z "${!1:-}" ]] && error "Missing required: $1" && exit 1
}

# -----------------------------------------------------------------------------------------------
# INTEGER VALIDATION
# Checks that a variable contains a valid positive integer.
# Exits with a clear error if the value is missing, empty, or not a number.
# Usage: validate_int VAR_NAME "$VAR_VALUE"
# Example: validate_int MOVER_STOP_TIMEOUT "$MOVER_STOP_TIMEOUT"
# -----------------------------------------------------------------------------------------------
validate_int() {
    local name="$1"
    local value="$2"

    if [[ -z "$value" ]]; then
        error "$name is not set — check Master.conf"
        exit 1
    fi

    if ! [[ "$value" =~ ^[0-9]+$ ]]; then
        error "$name must be a positive integer — got: '$value'"
        exit 1
    fi

    log "$name validated: $value"
}

# -----------------------------------------------------------------------------------------------
# HOST DETECTION
# Determines local and remote server names by comparing hostname against HOST1/HOST2.
# Sets LOCAL_SERVER_NAME, REMOTE_SERVER_NAME, and SSH_KEY for the current run direction.
# Both HOST1 and HOST2 must be defined in Master.conf.
# -----------------------------------------------------------------------------------------------
detect_hosts() {
    LOCAL_HOSTNAME="$(hostname)"

    if [[ "$LOCAL_HOSTNAME" == "$HOST1" ]]; then
        LOCAL_SERVER_NAME="$HOST1"
        REMOTE_SERVER_NAME="$HOST2"
    elif [[ "$LOCAL_HOSTNAME" == "$HOST2" ]]; then
        LOCAL_SERVER_NAME="$HOST2"
        REMOTE_SERVER_NAME="$HOST1"
    else
        error "Unknown host: $LOCAL_HOSTNAME"
        exit 1
    fi

    declare -A SSH_KEYS
    SSH_KEYS["$HOST1|$HOST2"]="$HOST1_SSH_KEY"
    SSH_KEYS["$HOST2|$HOST1"]="$HOST2_SSH_KEY"

    SSH_KEY="${SSH_KEYS[$LOCAL_SERVER_NAME|$REMOTE_SERVER_NAME]}"

    [[ -z "$SSH_KEY" ]] && error "Missing SSH key mapping for $LOCAL_SERVER_NAME → $REMOTE_SERVER_NAME" && exit 1

    info "$ICON_HOST Host: $LOCAL_SERVER_NAME → $REMOTE_SERVER_NAME"
}

# -----------------------------------------------------------------------------------------------
# REMOTE IP RESOLUTION
# Resolves the Tailscale IPv4 address of the remote server.
# Sets REMOTE_SERVER used by all subsequent SSH and rsync calls.
# Exits if resolution fails — likely means Tailscale is down or peer is offline.
# -----------------------------------------------------------------------------------------------
resolve_remote_ip() {
    log "Resolving remote IP for $REMOTE_SERVER_NAME..."
    REMOTE_SERVER=$(tailscale ip -4 "$REMOTE_SERVER_NAME" 2>/dev/null)

    [[ -z "$REMOTE_SERVER" ]] && error "Failed to resolve Tailscale IP for $REMOTE_SERVER_NAME" && exit 1

    info "$ICON_NET Remote IP: $REMOTE_SERVER"
}

# -----------------------------------------------------------------------------------------------
# CONNECTIVITY CHECK
# Pings the remote server to confirm it is reachable before starting any transfers.
# Prevents the rsync retry loop from burning all attempts against an unreachable host.
# If ping fails, prints a tailscale status hint to aid diagnosis before exiting.
# -----------------------------------------------------------------------------------------------
check_connectivity() {
    log "Checking connectivity to $REMOTE_SERVER..."
    if ! ping -c1 -W3 "$REMOTE_SERVER" &>/dev/null; then
        error "$ICON_PING Remote $REMOTE_SERVER ($REMOTE_SERVER_NAME) is unreachable"
        info  "Hint: tailscale status | grep $REMOTE_SERVER_NAME"
        exit 1
    fi
    info "$ICON_PING $REMOTE_SERVER_NAME is reachable"
}

# -----------------------------------------------------------------------------------------------
# REMOTE ROOTFS SPACE CHECK
# Checks the remote server's rootfs usage before any rsync runs.
# If the array is down or drives are missing, rsync writes land on rootfs instead of the array —
# this can fill the remote filesystem rapidly and crash the server.
# Threshold is set by ROOTFS_WARN in Master.conf (recommended: 75).
# Aborts cleanly with a clear error showing current usage vs threshold.
# -----------------------------------------------------------------------------------------------
check_remote_rootfs() {
    log "Checking remote rootfs usage..."

    REMOTE_USAGE=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
        "df / --output=pcent | tail -1 | tr -d ' %'" 2>/dev/null)

    if [[ -z "$REMOTE_USAGE" ]]; then
        error "Could not retrieve rootfs usage from $REMOTE_SERVER_NAME"
        exit 1
    fi

    if [[ "$REMOTE_USAGE" -ge "${ROOTFS_WARN:-75}" ]]; then
        echo ""
        error "$ICON_HEALTH Remote rootfs is ${REMOTE_USAGE}% full — threshold is ${ROOTFS_WARN:-75}%"
        warn  "Array may be down or drives missing on $REMOTE_SERVER_NAME"
        info  "Hint: Check array status on $REMOTE_SERVER_NAME before retrying"
        echo ""
        exit 1
    fi

    info "$ICON_HEALTH Remote rootfs: ${REMOTE_USAGE}% used (threshold: ${ROOTFS_WARN:-75}%)"
}

# -----------------------------------------------------------------------------------------------
# REMOTE SHARE VALIDATION
# Verifies that the target directory exists and is not empty on the remote server.
# Catches the scenario where the array is mounted but drives are not backing the share —
# the path exists as an empty mountpoint, which would cause --delete to wipe the remote.
# Called with the specific directory being synced so each share is checked individually.
# Usage: check_remote_share "/mnt/user/Movies"
# -----------------------------------------------------------------------------------------------
check_remote_share() {
    local dir="$1"

    log "Checking remote share: $dir..."

    SHARE_EXISTS=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
        "[[ -d '$dir' ]] && echo yes || echo no" 2>/dev/null)

    if [[ "$SHARE_EXISTS" != "yes" ]]; then
        echo ""
        error "$ICON_HEALTH Remote share does not exist: $dir"
        warn  "Array may not be started or share is not configured on $REMOTE_SERVER_NAME"
        info  "Hint: Check shares and array status on $REMOTE_SERVER_NAME before retrying"
        echo ""
        exit 1
    fi

    SHARE_EMPTY=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
        "[[ -z \"\$(ls -A '$dir' 2>/dev/null)\" ]] && echo yes || echo no" 2>/dev/null)

    if [[ "$SHARE_EMPTY" == "yes" ]]; then
        echo ""
        warn  "$ICON_HEALTH Remote share exists but is empty: $dir"
        warn  "Drives may not be mounted on $REMOTE_SERVER_NAME — aborting to protect data"
        info  "Hint: Verify array and drive assignments on $REMOTE_SERVER_NAME before retrying"
        echo ""
        exit 1
    fi

    info "$ICON_HEALTH Remote share verified: $dir"
}

# -----------------------------------------------------------------------------------------------
# REMOTE DISK CHECK
# Verifies that all physical disks backing a share are online and mounted on the remote server.
# Discovers disk layout automatically at runtime by finding all /mnt/diskN/sharename paths —
# no configuration required, works for any share regardless of how many disks it spans.
# Aborts if any single disk backing the share is offline — partial disk failure means
# incomplete data which could result in files being deleted by --delete during sync.
# Usage: check_remote_disks "/mnt/user/Movies"
# -----------------------------------------------------------------------------------------------
check_remote_disks() {
    local dir="$1"
    local share_name
    share_name=$(basename "$dir")

    info "$ICON_DISK Checking disks backing $share_name on $REMOTE_SERVER_NAME..."

    DISK_PATHS=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
        "ls -d /mnt/disk*/$share_name 2>/dev/null" 2>/dev/null)

    if [[ -z "$DISK_PATHS" ]]; then
        echo ""
        error "$ICON_DISK No disks found backing share $share_name on $REMOTE_SERVER_NAME"
        warn  "Share may not exist on any disk or array may not be started"
        info  "Hint: Check array and share configuration on $REMOTE_SERVER_NAME"
        echo ""
        exit 1
    fi

    local all_ok=true

    while IFS= read -r disk_share_path; do
        local disk_mount
        disk_mount=$(dirname "$disk_share_path")
        local disk_name
        disk_name=$(basename "$disk_mount")

        MOUNTED=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
            "mountpoint -q '$disk_mount' && echo yes || echo no" 2>/dev/null)

        if [[ "$MOUNTED" == "yes" ]]; then
            info "$ICON_DISK $disk_name $ICON_RUNNING — $share_name present"
        else
            error "$ICON_DISK $disk_name $ICON_STOPPED — $share_name missing or incomplete"
            all_ok=false
        fi
    done <<< "$DISK_PATHS"

    if [[ "$all_ok" == false ]]; then
        echo ""
        error "One or more disks backing $share_name are offline on $REMOTE_SERVER_NAME"
        warn  "Aborting to prevent partial or destructive sync"
        info  "Hint: Check disk assignments and array status on $REMOTE_SERVER_NAME before retrying"
        echo ""
        exit 1
    fi

    success "All disks backing $share_name are online"
}

# -----------------------------------------------------------------------------------------------
# CONTAINER MANAGEMENT — STOP
# Stops all containers listed in CRITICAL_CONTAINER_NAMES on the remote server.
# Only stops containers that are currently running — skips those already stopped.
# Tracks stopped containers in RUNNING_CONTAINERS for restart after rsync completes.
# CRITICAL_CONTAINER_NAMES must be a bash array — rsync.sh handles conversion from profile strings.
# -----------------------------------------------------------------------------------------------
RUNNING_CONTAINERS=()

stop_containers() {
    if [[ ${#CRITICAL_CONTAINER_NAMES[@]} -eq 0 ]] || \
       [[ "${CRITICAL_CONTAINER_NAMES[*]}" == "" ]]; then
        log "No containers configured for this profile, skipping stop."
        return
    fi

    info "Stopping containers..."
    RUNNING_CONTAINERS=()

    for c in "${CRITICAL_CONTAINER_NAMES[@]}"; do
        [[ -z "$c" ]] && continue

        info "Checking $c..."

        STATUS=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
            "docker inspect -f '{{.State.Running}}' $c 2>/dev/null" 2>/dev/null || echo "false")

        if [[ "$STATUS" == "true" ]]; then
            echo "$ICON_STOP Stopping $c..."
            RUNNING_CONTAINERS+=("$c")

            if ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" "docker stop $c" >/dev/null; then
                echo "$ICON_STOPPED $c stopped"
            else
                error "Failed to stop $c"
            fi
        else
            echo "$ICON_NOT_RUNNING $c is not running, skipping"
        fi
    done
}

# -----------------------------------------------------------------------------------------------
# CONTAINER MANAGEMENT — START
# Restarts only the containers that were running before rsync and were stopped by stop_containers.
# Containers listed in DELAYED_CONTAINERS receive a sleep of CONTAINER_DELAY seconds before
# starting — useful for dependencies like Authelia that need upstream services ready first.
# DELAYED_CONTAINERS must be a bash array — rsync.sh handles conversion from profile strings.
# -----------------------------------------------------------------------------------------------
start_containers() {
    if [[ ${#RUNNING_CONTAINERS[@]} -eq 0 ]]; then
        log "No containers to restart."
        return
    fi

    info "Starting containers..."

    for c in "${RUNNING_CONTAINERS[@]}"; do
        [[ -z "$c" ]] && continue

        local needs_delay=false
        for d in "${DELAYED_CONTAINERS[@]}"; do
            if [[ "$c" == "$d" ]]; then
                needs_delay=true
                break
            fi
        done

        if [[ "$needs_delay" == true ]]; then
            info "Waiting ${CONTAINER_DELAY}s before starting $c..."
            sleep "$CONTAINER_DELAY"
        fi

        echo "$ICON_START Starting $c..."
        if ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" "docker start $c" >/dev/null; then
            echo "$ICON_STARTED $c started"
        else
            error "Failed to start $c"
        fi
    done
}

# -----------------------------------------------------------------------------------------------
# RSYNC OPTIONS
# Loads rsync options for the current profile from PROFILE_RSYNC_OPTS in Master.conf.
# If no profile match is found, falls back to DEFAULT_RSYNC_OPTS.
# Note: profile opts do NOT inherit from defaults — all desired flags must be listed explicitly.
# Sets RSYNC_OPTS array used directly in the rsync call in rsync.sh.
# -----------------------------------------------------------------------------------------------
get_rsync_opts() {
    if [[ -n "${PROFILE_RSYNC_OPTS[$PROFILE_NAME]:-}" ]]; then
        read -r -a RSYNC_OPTS <<< "${PROFILE_RSYNC_OPTS[$PROFILE_NAME]}"
        log "Using profile rsync opts for $PROFILE_NAME: ${RSYNC_OPTS[*]}"
    else
        RSYNC_OPTS=("${DEFAULT_RSYNC_OPTS[@]}")
        log "Using default rsync opts: ${RSYNC_OPTS[*]}"
    fi
}

# -----------------------------------------------------------------------------------------------
# STATUS DISPLAY
# Prints a summary of the current runtime configuration.
# Triggered by --status or --summary flag passed to any script.
# Useful for verifying profile resolution and variable state before a live run.
# -----------------------------------------------------------------------------------------------
show_status() {
    echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
    echo "Local:      $LOCAL_SERVER_NAME"
    echo "Remote:     $REMOTE_SERVER_NAME"
    echo "IP:         $REMOTE_SERVER"
    echo "Profile:    $PROFILE_NAME"
    echo "DryRun:     $DRY_RUN"
    echo "Logging:    $ENABLE_LOGGING"
    echo "Containers: ${CRITICAL_CONTAINER_NAMES[*]}"
    echo "Delayed:    ${DELAYED_CONTAINERS[*]}"
    echo "Excludes:   ${EXCLUDE_DIRS[*]}"
    echo "━━━━━━━━━━━━━━━━━━━━━━━"
}