#!/bin/bash
# -----------------------------------------------------------------------------------------------
# ----------------- UNRAID OPS COMMON LIBRARY (STABLE FRAMEWORK v1) ----------------------------
# -----------------------------------------------------------------------------------------------
# Version:  2.7
# -----------------------------------------------------------------------------------------------
# Changelog:
#   v1.0 — Initial stable framework
#   v1.1 — format_duration moved here from daily_sync.sh for shared use
#           SSH_KEY collision resolved — gitea key renamed GITEA_SSH_KEY in Master.conf
#           Version and changelog tracking added
#   v1.2 — Consistent function header comment blocks across all functions
#           check_connectivity added as standalone function
#           check_connectivity friendlier error output with tailscale hint
#   v1.3 — check_remote_rootfs added — aborts if remote rootfs exceeds ROOTFS_WARN threshold
#           check_remote_share added — aborts if target directory is missing or empty on remote
#           Both protect against rsync running when remote array is down or drives are missing
#   v1.4 — check_remote_disks added — verifies all physical disks backing a share are mounted
#           Discovers disk layout automatically at runtime, no configuration required
#           Aborts if any single disk backing the share is offline or unmounted
#   v1.5 — Full icon set expanded — each operation and state has its own distinct icon
#           All function output updated to use correct icon per context
#           Icons grouped and commented by category for clarity
#   v1.6 — ICON_CONTAINERS added — 📦 anchors all container sections for visual consistency
#           ICON_NOT_RUNNING changed to ⭕ — distinct from ICON_STOPPED 🔴
#           Section dividers updated from --- to ━━━ for cleaner log readability
#           Summary passed/failed lines use ICON_SUCCESS and ICON_ERROR consistently
#   v1.7 — ICON_MOVER added for mover operations
#           ICON_CONTAINERS replaces ICON_DOCKER for docker/container operations
#           validate_int added — reusable integer validation for any script
#   v1.8 — ICON_PHP added for PHP-FPM operations
#   v1.9 — ICON_REBOOT added for server reboot operations
#   v2.0 — ICON_PLUGIN added for User Scripts plugin operations
#   v2.1 — ICON_ZFS and ICON_MEM added for ZFS and memory diagnostics
#           Diagnostics icon group added to icon block
#   v2.2 — ICON_WATCHDOG added for Docker watchdog monitoring operations
#   v2.3 — ICON_NOTIFY added for notification operations
#           notify() added — shared notification function supporting unRAID native and Discord
#           NOTIFY_UNRAID and DISCORD_WEBHOOK configured in Master.conf
#   v2.4 — ICON_CLEAN, ICON_TRASH added for media cleaner operations
#           ICON_PERMS, ICON_UNLOCKED added for media permissions operations
#           ICON_REBOOT_SMART added for smart conditional reboot
#   v2.5 — ICON_RAM added for ramdisk operations
#           ICON_LINK added for symlink state and management
#           Transcode scripts group added to ecosystem
#   v2.6 — ICON_FAILOVER added for failover operations
#           check_local_array added — verifies local /mnt/user is mounted and healthy
#           check_remote_array added — verifies remote /mnt/user is mounted and healthy
#           check_remote_docker added — verifies remote Docker daemon is responding
#           ping_remote added — non-fatal ping returning status for failover use
#           ping_internet added — non-fatal external ping for failover use
#   v2.7 — ICON_WEBGUI added for WebGUI watchdog operations
#           ICON_DOCKER_NET added for Docker network connect operations
# -----------------------------------------------------------------------------------------------

# -----------------------------------------------------------------------------------------------
# ICONS
# Each icon has one job — do not reuse across different contexts.
# -----------------------------------------------------------------------------------------------

# System / Host
ICON_HOST="🖥️"          # host detection
ICON_NET="🌐"           # network / IP resolution
ICON_PING="📡"          # connectivity check
ICON_GEAR="⚙️"          # setup section header / profile load

# Health Checks
ICON_DISK="💾"          # disk checks
ICON_HEALTH="🩺"        # rootfs / share health checks
ICON_SHIELD="🛡️"        # pre-flight section header

# Containers
ICON_CONTAINERS="📦"    # container section anchor
ICON_STOP="⛔"          # stop command being issued
ICON_STOPPED="🔴"       # container confirmed stopped
ICON_START="▶️"         # start command being issued
ICON_STARTED="💚"       # container confirmed started
ICON_RUNNING="🟢"       # container already running when checked
ICON_NOT_RUNNING="⭕"   # container already stopped when checked

# Transfer
ICON_SYNC="🔄"          # transfer section header
ICON_RUN="🚀"           # sync starting / rsync attempt
ICON_RETRY="🔁"         # retry attempt
ICON_DONE="🏁"          # transfer complete

# Summary
ICON_SUMMARY="📋"       # summary section header
ICON_TIME="⏱️"          # duration line

# System Operations
ICON_MOVER="🔃"         # mover operations
ICON_REBOOT="⚡"        # scheduled server reboot
ICON_REBOOT_SMART="🚨"  # smart conditional reboot triggered
ICON_PLUGIN="🧩"        # user scripts plugin operations
ICON_PHP="👥"           # PHP-FPM operations
ICON_WEBGUI="💻"        # WebGUI / nginx / emhttp operations

# Diagnostics
ICON_ZFS="📊"           # ZFS ARC statistics
ICON_MEM="🧠"           # memory status
ICON_WATCHDOG="🐾"      # docker watchdog monitoring operations

# Media Operations
ICON_CLEAN="🧹"         # media cleaner operations
ICON_TRASH="🗑️"         # files being deleted
ICON_PERMS="🔐"         # permissions operation / section header
ICON_UNLOCKED="🔓"      # permissions successfully applied to a share

# Transcode Operations
ICON_RAM="💨"           # ramdisk operations — fast ephemeral storage
ICON_LINK="🔗"          # symlink state and management

# Failover Operations
ICON_FAILOVER="🔀"      # failover state changes and operations

# Docker Network Operations
ICON_DOCKER_NET="🔌"    # Docker network connect operations

# Notifications
ICON_NOTIFY="🔔"        # notification operations

# Output
ICON_INFO="ℹ️"
ICON_WARN="⚠️"
ICON_ERROR="❌"
ICON_SUCCESS="✅"

# -----------------------------------------------------------------------------------------------
# OUTPUT HELPERS
# -----------------------------------------------------------------------------------------------
info()    { echo "$ICON_INFO  [INFO]  $*"; }
warn()    { echo "$ICON_WARN  [WARN]  $*"; }
error()   { echo "$ICON_ERROR [ERROR] $*"; }
success() { echo "$ICON_SUCCESS [OK]  $*"; }

log() {
    [[ "${ENABLE_LOGGING:-false}" == true ]] && echo "[LOG] $*"
}

# -----------------------------------------------------------------------------------------------
# NOTIFICATION
# Usage: notify "message" "subject" "severity"
# Severity: normal, warning, alert
# -----------------------------------------------------------------------------------------------
notify() {
    local message="$1"
    local subject="${2:-unRAID Notification}"
    local severity="${3:-normal}"

    log "$ICON_NOTIFY Sending notification: $subject — $message"

    if [[ "${NOTIFY_UNRAID:-false}" == true ]]; then
        local notify_script="/usr/local/emhttp/plugins/dynamix/scripts/notify"
        if [[ -x "$notify_script" ]]; then
            "$notify_script" -s "$subject" -d "$message" -i "$severity" 2>/dev/null
            log "$ICON_NOTIFY unRAID notification sent"
        else
            log "$ICON_NOTIFY unRAID notify script not found — skipping"
        fi
    fi

    if [[ -n "${DISCORD_WEBHOOK:-}" ]]; then
        local payload
        payload=$(printf '{"content": "%s — **%s**\\n%s"}' \
            "$ICON_NOTIFY" "$subject" "$message")
        if curl -s -H "Content-Type: application/json" \
            -d "$payload" "$DISCORD_WEBHOOK" >/dev/null 2>&1; then
            log "$ICON_NOTIFY Discord notification sent"
        else
            warn "Discord notification failed — check DISCORD_WEBHOOK in Master.conf"
        fi
    fi
}

# -----------------------------------------------------------------------------------------------
# DURATION FORMATTER
# -----------------------------------------------------------------------------------------------
format_duration() {
    local secs=$1
    local mins=$((secs / 60))
    local rem=$((secs % 60))
    [[ $mins -gt 0 ]] && echo "${mins}m${rem}s" || echo "${rem}s"
}

# -----------------------------------------------------------------------------------------------
# ARG PARSER
# -----------------------------------------------------------------------------------------------
parse_args() {
    ENABLE_LOGGING=${ENABLE_LOGGING:-false}
    DRY_RUN=${DRY_RUN:-false}
    SHOW_STATUS=${SHOW_STATUS:-false}
    CLEAN_ARGS=()

    for ARG in "$@"; do
        if [[ "$ARG" == *=* ]]; then
            VAR="${ARG%%=*}"
            VAL="${ARG#*=}"
            case "$VAR" in
                LOG)
                    [[ "$VAL" == "true" ]]  && ENABLE_LOGGING=true
                    [[ "$VAL" == "false" ]] && ENABLE_LOGGING=false
                    ;;
                *)
                    if declare -p "$VAR" &>/dev/null; then
                        printf -v "$VAR" '%s' "$VAL"
                        log "Set $VAR=$VAL"
                    else
                        warn "Unknown variable: $VAR"
                    fi
                    ;;
            esac
        else
            case "$ARG" in
                --dry-run|-n)       DRY_RUN=true ;;
                --log)              ENABLE_LOGGING=true ;;
                --no-log)           ENABLE_LOGGING=false ;;
                --status|--summary) SHOW_STATUS=true ;;
                --help|-h)
                    echo "Usage: script [--dry-run] [--log] [--status]"
                    exit 0
                    ;;
                *) CLEAN_ARGS+=("$ARG") ;;
            esac
        fi
    done

    PARSED_ARGS=("${CLEAN_ARGS[@]}")
}

# -----------------------------------------------------------------------------------------------
# VALIDATION
# -----------------------------------------------------------------------------------------------
require_var() {
    [[ -z "${!1:-}" ]] && error "Missing required: $1" && exit 1
}

validate_int() {
    local name="$1" value="$2"
    if [[ -z "$value" ]]; then
        error "$name is not set — check Master.conf"
        exit 1
    fi
    if ! [[ "$value" =~ ^[0-9]+$ ]]; then
        error "$name must be a positive integer — got: '$value'"
        exit 1
    fi
    log "$name validated: $value"
}

# -----------------------------------------------------------------------------------------------
# HOST DETECTION
# -----------------------------------------------------------------------------------------------
detect_hosts() {
    LOCAL_HOSTNAME="$(hostname)"

    if [[ "$LOCAL_HOSTNAME" == "$HOST1" ]]; then
        LOCAL_SERVER_NAME="$HOST1"
        REMOTE_SERVER_NAME="$HOST2"
    elif [[ "$LOCAL_HOSTNAME" == "$HOST2" ]]; then
        LOCAL_SERVER_NAME="$HOST2"
        REMOTE_SERVER_NAME="$HOST1"
    else
        error "Unknown host: $LOCAL_HOSTNAME"
        exit 1
    fi

    declare -A SSH_KEYS
    SSH_KEYS["$HOST1|$HOST2"]="$HOST1_SSH_KEY"
    SSH_KEYS["$HOST2|$HOST1"]="$HOST2_SSH_KEY"
    SSH_KEY="${SSH_KEYS[$LOCAL_SERVER_NAME|$REMOTE_SERVER_NAME]}"

    [[ -z "$SSH_KEY" ]] && error "Missing SSH key mapping" && exit 1

    info "$ICON_HOST Host: $LOCAL_SERVER_NAME → $REMOTE_SERVER_NAME"
}

# -----------------------------------------------------------------------------------------------
# REMOTE IP RESOLUTION
# -----------------------------------------------------------------------------------------------
resolve_remote_ip() {
    log "Resolving remote IP for $REMOTE_SERVER_NAME..."
    REMOTE_SERVER=$(tailscale ip -4 "$REMOTE_SERVER_NAME" 2>/dev/null)
    [[ -z "$REMOTE_SERVER" ]] && error "Failed to resolve Tailscale IP for $REMOTE_SERVER_NAME" && exit 1
    info "$ICON_NET Remote IP: $REMOTE_SERVER"
}

# -----------------------------------------------------------------------------------------------
# CONNECTIVITY CHECK — fatal, used by rsync scripts
# -----------------------------------------------------------------------------------------------
check_connectivity() {
    log "Checking connectivity to $REMOTE_SERVER..."
    if ! ping -c1 -W3 "$REMOTE_SERVER" &>/dev/null; then
        error "$ICON_PING Remote $REMOTE_SERVER ($REMOTE_SERVER_NAME) is unreachable"
        info  "Hint: tailscale status | grep $REMOTE_SERVER_NAME"
        exit 1
    fi
    info "$ICON_PING $REMOTE_SERVER_NAME is reachable"
}

# -----------------------------------------------------------------------------------------------
# PING REMOTE — non-fatal, used by failover
# Returns 0 if reachable, 1 if not
# -----------------------------------------------------------------------------------------------
ping_remote() {
    ping -c2 -W3 "$REMOTE_SERVER" &>/dev/null
}

# -----------------------------------------------------------------------------------------------
# PING INTERNET — non-fatal, used by failover
# Returns 0 if internet reachable, 1 if not
# -----------------------------------------------------------------------------------------------
ping_internet() {
    ping -c2 -W3 "${EXTERNAL_IP:-8.8.8.8}" &>/dev/null
}

# -----------------------------------------------------------------------------------------------
# LOCAL ARRAY CHECK — non-fatal, returns status
# Used by failover before starting remote containers locally
# -----------------------------------------------------------------------------------------------
check_local_array() {
    log "Checking local array..."
    if ! mountpoint -q /mnt/user 2>/dev/null; then
        error "$ICON_DISK Local array is not started — /mnt/user is not mounted"
        return 1
    fi
    local file_count
    file_count=$(ls /mnt/user 2>/dev/null | wc -l)
    if [[ "$file_count" -eq 0 ]]; then
        error "$ICON_DISK Local array appears empty — shares may not be available"
        return 1
    fi
    log "Local array is healthy"
    return 0
}

# -----------------------------------------------------------------------------------------------
# REMOTE ARRAY CHECK — non-fatal, returns status
# Used before handback rsync
# -----------------------------------------------------------------------------------------------
check_remote_array() {
    log "Checking remote array on $REMOTE_SERVER_NAME..."
    local result
    result=$(ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \
        "mountpoint -q /mnt/user && echo yes || echo no" 2>/dev/null)
    if [[ "$result" != "yes" ]]; then
        error "$ICON_DISK Remote array not started on $REMOTE_SERVER_NAME"
        return 1
    fi
    log "Remote array is healthy"
    return 0
}

# -----------------------------------------------------------------------------------------------
# REMOTE DOCKER CHECK — non-fatal, returns status
# Used before starting containers on remote
# -----------------------------------------------------------------------------------------------
check_remote_docker() {
    log "Checking remote Docker daemon on $REMOTE_SERVER_NAME..."
    if ! ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \
        "timeout 10 docker ps" >/dev/null 2>&1; then
        error "$ICON_CONTAINERS Remote Docker daemon not responding on $REMOTE_SERVER_NAME"
        return 1
    fi
    log "Remote Docker daemon is healthy"
    return 0
}

# -----------------------------------------------------------------------------------------------
# REMOTE ROOTFS SPACE CHECK — fatal
# -----------------------------------------------------------------------------------------------
check_remote_rootfs() {
    log "Checking remote rootfs usage..."
    REMOTE_USAGE=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
        "df / --output=pcent | tail -1 | tr -d ' %'" 2>/dev/null)

    if [[ -z "$REMOTE_USAGE" ]]; then
        error "Could not retrieve rootfs usage from $REMOTE_SERVER_NAME"
        exit 1
    fi

    if [[ "$REMOTE_USAGE" -ge "${ROOTFS_WARN:-75}" ]]; then
        error "$ICON_HEALTH Remote rootfs ${REMOTE_USAGE}% — threshold ${ROOTFS_WARN:-75}%"
        warn  "Array may be down or drives missing on $REMOTE_SERVER_NAME"
        exit 1
    fi

    info "$ICON_HEALTH Remote rootfs: ${REMOTE_USAGE}% (threshold: ${ROOTFS_WARN:-75}%)"
}

# -----------------------------------------------------------------------------------------------
# REMOTE SHARE VALIDATION — fatal
# Usage: check_remote_share "/mnt/user/Movies"
# -----------------------------------------------------------------------------------------------
check_remote_share() {
    local dir="$1"
    log "Checking remote share: $dir..."

    SHARE_EXISTS=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
        "[[ -d '$dir' ]] && echo yes || echo no" 2>/dev/null)

    if [[ "$SHARE_EXISTS" != "yes" ]]; then
        error "$ICON_HEALTH Remote share does not exist: $dir"
        exit 1
    fi

    SHARE_EMPTY=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
        "[[ -z \"\$(ls -A '$dir' 2>/dev/null)\" ]] && echo yes || echo no" 2>/dev/null)

    if [[ "$SHARE_EMPTY" == "yes" ]]; then
        warn "$ICON_HEALTH Remote share exists but is empty: $dir — aborting to protect data"
        exit 1
    fi

    info "$ICON_HEALTH Remote share verified: $dir"
}

# -----------------------------------------------------------------------------------------------
# REMOTE DISK CHECK — fatal
# Usage: check_remote_disks "/mnt/user/Movies"
# -----------------------------------------------------------------------------------------------
check_remote_disks() {
    local dir="$1"
    local share_name
    share_name=$(basename "$dir")

    info "$ICON_DISK Checking disks backing $share_name on $REMOTE_SERVER_NAME..."

    DISK_PATHS=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
        "ls -d /mnt/disk*/$share_name 2>/dev/null" 2>/dev/null)

    if [[ -z "$DISK_PATHS" ]]; then
        error "$ICON_DISK No disks found backing $share_name on $REMOTE_SERVER_NAME"
        exit 1
    fi

    local all_ok=true
    while IFS= read -r disk_share_path; do
        local disk_mount disk_name
        disk_mount=$(dirname "$disk_share_path")
        disk_name=$(basename "$disk_mount")

        MOUNTED=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
            "mountpoint -q '$disk_mount' && echo yes || echo no" 2>/dev/null)

        if [[ "$MOUNTED" == "yes" ]]; then
            info "$ICON_DISK $disk_name $ICON_RUNNING — $share_name present"
        else
            error "$ICON_DISK $disk_name $ICON_STOPPED — $share_name missing"
            all_ok=false
        fi
    done <<< "$DISK_PATHS"

    if [[ "$all_ok" == false ]]; then
        error "One or more disks backing $share_name are offline on $REMOTE_SERVER_NAME"
        exit 1
    fi

    success "All disks backing $share_name are online"
}

# -----------------------------------------------------------------------------------------------
# CONTAINER MANAGEMENT — STOP (remote via SSH)
# -----------------------------------------------------------------------------------------------
RUNNING_CONTAINERS=()

stop_containers() {
    if [[ ${#CRITICAL_CONTAINER_NAMES[@]} -eq 0 ]] || \
       [[ "${CRITICAL_CONTAINER_NAMES[*]}" == "" ]]; then
        log "No containers configured for this profile, skipping stop."
        return
    fi

    info "Stopping containers..."
    RUNNING_CONTAINERS=()

    for c in "${CRITICAL_CONTAINER_NAMES[@]}"; do
        [[ -z "$c" ]] && continue
        STATUS=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
            "docker inspect -f '{{.State.Running}}' $c 2>/dev/null" 2>/dev/null || echo "false")

        if [[ "$STATUS" == "true" ]]; then
            echo "$ICON_STOP Stopping $c..."
            RUNNING_CONTAINERS+=("$c")
            if ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" "docker stop $c" >/dev/null; then
                echo "$ICON_STOPPED $c stopped"
            else
                error "Failed to stop $c"
            fi
        else
            echo "$ICON_NOT_RUNNING $c is not running, skipping"
        fi
    done
}

# -----------------------------------------------------------------------------------------------
# CONTAINER MANAGEMENT — START (remote via SSH)
# -----------------------------------------------------------------------------------------------
start_containers() {
    if [[ ${#RUNNING_CONTAINERS[@]} -eq 0 ]]; then
        log "No containers to restart."
        return
    fi

    info "Starting containers..."

    for c in "${RUNNING_CONTAINERS[@]}"; do
        [[ -z "$c" ]] && continue

        local needs_delay=false
        for d in "${DELAYED_CONTAINERS[@]}"; do
            [[ "$c" == "$d" ]] && needs_delay=true && break
        done

        if [[ "$needs_delay" == true ]]; then
            info "Waiting ${CONTAINER_DELAY}s before starting $c..."
            sleep "$CONTAINER_DELAY"
        fi

        echo "$ICON_START Starting $c..."
        if ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" "docker start $c" >/dev/null; then
            echo "$ICON_STARTED $c started"
        else
            error "Failed to start $c"
        fi
    done
}

# -----------------------------------------------------------------------------------------------
# RSYNC OPTIONS
# -----------------------------------------------------------------------------------------------
get_rsync_opts() {
    if [[ -n "${PROFILE_RSYNC_OPTS[$PROFILE_NAME]:-}" ]]; then
        read -r -a RSYNC_OPTS <<< "${PROFILE_RSYNC_OPTS[$PROFILE_NAME]}"
        log "Using profile rsync opts for $PROFILE_NAME: ${RSYNC_OPTS[*]}"
    else
        RSYNC_OPTS=("${DEFAULT_RSYNC_OPTS[@]}")
        log "Using default rsync opts: ${RSYNC_OPTS[*]}"
    fi
}

# -----------------------------------------------------------------------------------------------
# STATUS DISPLAY
# -----------------------------------------------------------------------------------------------
show_status() {
    echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
    echo "Local:      $LOCAL_SERVER_NAME"
    echo "Remote:     $REMOTE_SERVER_NAME"
    echo "IP:         $REMOTE_SERVER"
    echo "Profile:    ${PROFILE_NAME:-n/a}"
    echo "DryRun:     $DRY_RUN"
    echo "Logging:    $ENABLE_LOGGING"
    echo "Containers: ${CRITICAL_CONTAINER_NAMES[*]:-n/a}"
    echo "Delayed:    ${DELAYED_CONTAINERS[*]:-n/a}"
    echo "Excludes:   ${EXCLUDE_DIRS[*]:-n/a}"
    echo "━━━━━━━━━━━━━━━━━━━━━━━"
}