Varaverk/Transcodes/transcode_cleanup.sh

#!/bin/bash
# ==============================================================================================
# ================================= Transcode Cleanup ==========================================
# ==============================================================================================
#
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# Removes stale transcode files from both ramdisk and SSD fallback locations.
# Called by transcode_management.sh (Orchestrators/) before transcode_manager.sh —
# cleanup must run first so the manager sees real active-session usage, not
# inflated usage from stale files. Must be fast and non-blocking.
#
# A file is eligible for deletion only if ALL conditions are true:
#   1. Older than TRANSCODE_MAX_AGE minutes (mtime — last write time)
#   2. Not currently open by any process (checked via lsof pre-built map)
#
# ==============================================================================================
# DESIGN PRINCIPLES
# ==============================================================================================
#
# lsof Called Once, Not Per File
#   On a busy Live TV system the ramdisk contains thousands of HLS segment files.
#   Calling lsof once per file creates thousands of subprocess calls every 3 minutes.
#   lsof is called once per location to build a complete open-file map. All subsequent
#   checks are O(1) lookups against that map — thousands of files, one lsof call.
#
# No Session-Aware Cleanup
#   ffmpeg generates folder names independently of the media server API session IDs.
#   There is no reliable correlation between API session IDs and transcoding-temp
#   subfolder names. Attempting to correlate them would falsely treat active sessions
#   as ended. lsof is the correct check — if ffmpeg has a file open, it is active
#   regardless of folder naming or session state.
#
# transcoding-temp Is Never Deleted
#   If cleanup removes the empty transcoding-temp folder from the ramdisk, Emby
#   searches all accessible paths for an existing one, finds the SSD fallback version,
#   and routes all new sessions there until Emby restarts. The directory is excluded
#   from find by name — protected even when completely empty.
#
# Post-Cleanup Flip-Back
#   After removing stale files, checks whether ramdisk usage dropped below
#   RAMDISK_LOW_GB. If so — and symlink currently points at SSD — triggers a
#   flip back to ramdisk. This is the recovery path; the manager handles
#   the fill-up path.
#
# ==============================================================================================
# OPERATIONAL SAFEGUARDS
# ==============================================================================================
#
# Wait Lock
#   acquire_lock "wait" — waits if a previous cleanup run is still active rather
#   than exiting. The caller's 3-minute interval can overlap on a slow system.
#
# lsof Timeout
#   lsof call capped at 15 seconds per location — prevents blocking indefinitely
#   on a system with many open files.
#
# transcoding-temp Guard
#   `! -name "transcoding-temp"` in the find command — protected unconditionally.
#
# Silent by Default
#   Runs every 3 minutes — must not produce noise when healthy.
#
# ==============================================================================================
# CONFIGURATION
# ==============================================================================================
#
# master_host*.conf
#
#   HOST*_RAMDISK_PATH / HOST*_TRANSCODE_SSD / HOST*_RAMDISK_LOW_GB
#       Aliased by detect_hosts() → RAMDISK_PATH / TRANSCODE_SSD / RAMDISK_LOW_GB.
#
# master.conf
#
#   TRANSCODE_MAX_AGE
#       Minutes before an inactive transcode file is eligible for deletion. (default: 20)
#
#   TRANSCODE_ORPHAN_AGE
#       Minutes for orphan folder detection. (default: 30)
#
# ==============================================================================================
# RUNTIME MODES
# ==============================================================================================
#
# transcode_cleanup.sh
#     Remove stale files from ramdisk and SSD. Check for flip-back opportunity.
#
# transcode_cleanup.sh --dry-run
#     Show which files would be deleted. No deletions, no flip.
#
# transcode_cleanup.sh --status
#     Show current file counts, ages, and open-file status per location.
#
# transcode_cleanup.sh --log
#     Verbose per-file output including age, open status, and deletion result.
#
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

source "$SCRIPT_DIR/../load_config.sh"

parse_args "$@"

STATE_FILE="/tmp/transcode_state.db"

# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
    error "Must be run as root"
    exit 1
fi

validate_unraid_cmd \
    "/usr/local/emhttp/plugins/dynamix/scripts/notify" \
    "" "" \
    "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"

acquire_lock "wait"

# detect_hosts() sets MY_ID and aliases RAMDISK_PATH, TRANSCODE_SSD, RAMDISK_LOW_GB
detect_hosts

# lsof availability check
LSOF_AVAILABLE=false
if command -v lsof >/dev/null 2>&1; then
    LSOF_AVAILABLE=true
    log "lsof available — active file check enabled"
else
    warn "lsof not available — active file check skipped, all aged files eligible for deletion"
fi

log "Identity:  $MY_ID ($LOCAL_SERVER_NAME)"
log "Max age:   ${TRANSCODE_MAX_AGE} minutes"

# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
    echo ""
    echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
    echo "$ICON_HOST   Identity:     $MY_ID ($LOCAL_SERVER_NAME)"
    echo "$ICON_RAM    Ramdisk:      $RAMDISK_PATH"
    echo "$ICON_DISK   SSD fallback: $TRANSCODE_SSD"
    echo "$ICON_TRASH  Max age:      ${TRANSCODE_MAX_AGE} minutes"
    echo "$ICON_TRASH  Orphan age:   ${TRANSCODE_ORPHAN_AGE} minutes"
    echo "$ICON_RAM    Flip at:      ${RAMDISK_LOW_GB}GB"
    echo "$ICON_GEAR   lsof check:   $LSOF_AVAILABLE"
    echo "$ICON_GEAR   Dry Run:      $DRY_RUN"
    echo ""

    if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
        USAGE=$(df -BG "$RAMDISK_PATH" | awk 'NR==2 {print $3}')
        AVAIL=$(df -BG "$RAMDISK_PATH" | awk 'NR==2 {print $4}')
        echo "  $ICON_RAM  Ramdisk: mounted — $USAGE used / $AVAIL available ✅"
    else
        echo "  $ICON_RAM  Ramdisk: not mounted"
    fi

    CURRENT_TARGET=$(grep "^current_target=" "$STATE_FILE" 2>/dev/null | cut -d= -f2)
    echo "  $ICON_LINK Current target: ${CURRENT_TARGET:-unknown}"
    echo "━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi

[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be deleted"

# ==============================================================================================
# ── CLEANUP FUNCTION ──────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Scans a location and removes eligible files.
# Calls lsof ONCE per location — builds in-memory OPEN_FILES_MAP for O(1) lookup.
# Returns via LOCATION_REMOVED, LOCATION_FREED, LOCATION_SKIPPED, LOCATION_ACTIVE

cleanup_location() {
    local location="$1" label="$2" max_age="$3"
    local files_removed=0 bytes_freed=0 files_skipped=0 files_active=0

    if [[ ! -d "$location" ]]; then
        log "$label does not exist — skipping"
        LOCATION_REMOVED=0 LOCATION_FREED="0B" LOCATION_SKIPPED=0 LOCATION_ACTIVE=0
        return
    fi

    local file_count
    file_count=$(find "$location" -type f 2>/dev/null | wc -l)
    log "$label: $file_count files to scan (age threshold: ${max_age}min)"

    # Build in-memory open file map — O(1) lookup per file
    # One lsof call per location — never per file
    declare -A OPEN_FILES_MAP
    if [[ "$LSOF_AVAILABLE" == true ]]; then
        log "Building open file map for $label..."
        while IFS= read -r open_file; do
            [[ -n "$open_file" ]] && OPEN_FILES_MAP["$open_file"]=1
        done < <(timeout 15 lsof +D "$location" 2>/dev/null | awk 'NR>1 {print $9}' | sort -u)
        log "${#OPEN_FILES_MAP[@]} files currently open in $label"
    fi

    # Process aged files
    while IFS= read -r file; do
        [[ -z "$file" ]] && continue

        # O(1) open file check — in-memory map
        if [[ -n "${OPEN_FILES_MAP[$file]:-}" ]]; then
            (( files_active++ ))
            log "Skipping open file: $file"
            continue
        fi

        local file_size
        file_size=$(stat -c%s "$file" 2>/dev/null || echo 0)

        if [[ "$DRY_RUN" == true ]]; then
            log "DRY RUN — would delete: $(basename "$file")"
            (( files_skipped++ ))
        else
            if rm -f "$file" 2>/dev/null; then
                (( files_removed++ ))
                bytes_freed=$(( bytes_freed + file_size ))
                log "Deleted: $file"
            else
                warn "Could not delete: $file"
                (( files_skipped++ ))
            fi
        fi

    done < <(find "$location" -type f -mmin +"$max_age" 2>/dev/null)

    # Remove empty directories — but NEVER remove transcoding-temp
    # transcoding-temp must always exist on ramdisk so Emby finds it there first
    if [[ "$DRY_RUN" == false ]]; then
        find "$location" -mindepth 1 -type d -empty \
            ! -name "transcoding-temp" -delete 2>/dev/null
    fi

    # Format bytes freed
    local freed_human
    if (( bytes_freed > 1073741824 )); then
        freed_human=$(awk "BEGIN {printf \"%.1fGB\", $bytes_freed / 1073741824}")
    elif (( bytes_freed > 1048576 )); then
        freed_human=$(awk "BEGIN {printf \"%.1fMB\", $bytes_freed / 1048576}")
    elif (( bytes_freed > 0 )); then
        freed_human="${bytes_freed}B"
    else
        freed_human="0B"
    fi

    log "$label — removed $files_removed files ($freed_human freed) | active: $files_active | skipped: $files_skipped"

    LOCATION_REMOVED=$files_removed
    LOCATION_FREED=$freed_human
    LOCATION_SKIPPED=$files_skipped
    LOCATION_ACTIVE=$files_active
}

# ==============================================================================================
# ━━━ Transcode Cleanup ━━━
# ==============================================================================================
START=$(date +%s)
TOTAL_REMOVED=0 TOTAL_SKIPPED=0 TOTAL_ACTIVE=0
RAMDISK_FREED="0B" SSD_FREED="0B"

# Cleanup ramdisk
if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
    cleanup_location "$RAMDISK_PATH" "Ramdisk" "$TRANSCODE_MAX_AGE"
    TOTAL_REMOVED=$(( TOTAL_REMOVED + LOCATION_REMOVED ))
    TOTAL_SKIPPED=$(( TOTAL_SKIPPED + LOCATION_SKIPPED ))
    TOTAL_ACTIVE=$(( TOTAL_ACTIVE + LOCATION_ACTIVE ))
    RAMDISK_FREED=$LOCATION_FREED
else
    log "Ramdisk not mounted — skipping ramdisk cleanup"
fi

# Cleanup SSD fallback
if [[ -d "$TRANSCODE_SSD" ]]; then
    cleanup_location "$TRANSCODE_SSD" "SSD fallback" "$TRANSCODE_MAX_AGE"
    TOTAL_REMOVED=$(( TOTAL_REMOVED + LOCATION_REMOVED ))
    TOTAL_SKIPPED=$(( TOTAL_SKIPPED + LOCATION_SKIPPED ))
    TOTAL_ACTIVE=$(( TOTAL_ACTIVE + LOCATION_ACTIVE ))
    SSD_FREED=$LOCATION_FREED
else
    log "SSD fallback not found — skipping SSD cleanup"
fi

# Post-cleanup — check if ramdisk recovered enough to flip symlink back to ramdisk
if [[ "$DRY_RUN" == false ]] && mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
    RAMDISK_USED_KB=$(df "$RAMDISK_PATH" --output=used 2>/dev/null | tail -1 | tr -d ' ')
    RAMDISK_USED_GB=$(awk "BEGIN {printf \"%.2f\", ${RAMDISK_USED_KB:-0} / 1048576}")
    LOW_RECOVERED=$(awk "BEGIN {print ($RAMDISK_USED_GB < $RAMDISK_LOW_GB) ? 1 : 0}")
    CURRENT_TARGET=$(grep "^current_target=" "$STATE_FILE" 2>/dev/null | cut -d= -f2)

    if [[ "$LOW_RECOVERED" == "1" && "$CURRENT_TARGET" == "$TRANSCODE_SSD" ]]; then
        log "Ramdisk has space after cleanup (${RAMDISK_USED_GB}GB < ${RAMDISK_LOW_GB}GB) — triggering manager to flip back"
        bash "$SCRIPT_DIR/transcode_manager.sh" --no-log
    fi
fi

END=$(date +%s)

# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
echo "━━━━━ $ICON_SUMMARY TRANSCODE CLEANUP SUMMARY ━━━━━"
echo "$ICON_HOST   Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_RAM    Ramdisk freed: $RAMDISK_FREED"
echo "$ICON_DISK   SSD freed:     $SSD_FREED"
echo "$ICON_TRASH  Removed:       $TOTAL_REMOVED files"
echo "$ICON_RUNNING Active:       $TOTAL_ACTIVE files (open — protected)"
echo "$ICON_TRASH  Skipped:       $TOTAL_SKIPPED files"
echo "$ICON_TIME   Duration:      $(format_duration $(( END - START )))"

if [[ "$DRY_RUN" == true ]]; then
    warn "DRY RUN — no files deleted"
else
    log "$ICON_DONE Status: done ✅"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"