Gmer4Lfe
e8b114094a
Headers claimed protections the code never had, and several destructive paths had no guard against a collapsed config value.
393 lines
19 KiB
Bash
Executable File
393 lines
19 KiB
Bash
Executable File
#!/bin/bash
|
|
# ==============================================================================================
|
|
# ================================= Transcode Cleanup ==========================================
|
|
# ==============================================================================================
|
|
#
|
|
# PURPOSE
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Removes stale transcode files from both ramdisk and SSD fallback locations.
|
|
# Called by transcode_management.sh (Orchestrators/) before transcode_manager.sh —
|
|
# cleanup must run first so the manager sees real active-session usage, not
|
|
# inflated usage from stale files. Must be fast and non-blocking.
|
|
#
|
|
# A file is eligible for deletion only if ALL conditions are true:
|
|
# 1. Older than TRANSCODE_MAX_AGE minutes (mtime — last write time)
|
|
# 2. Not currently open by any process (checked via lsof pre-built map)
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL MODEL
|
|
# ==============================================================================================
|
|
#
|
|
# Two locations cleaned in sequence, each through the same routine:
|
|
#
|
|
# Ramdisk — only if mountpoint -q confirms it is actually mounted. An unmounted
|
|
# ramdisk means the underlying directory is the real filesystem, and
|
|
# cleaning it would delete from disk rather than from tmpfs.
|
|
# SSD fallback — only if the directory exists.
|
|
#
|
|
# Per location:
|
|
# 1. Path sanity check — refuse anything shallower than two components
|
|
# 2. Count total and age-eligible files (-mmin +TRANSCODE_MAX_AGE)
|
|
# 3. One lsof +D call → in-memory open-file map for the whole location
|
|
# 4. Per eligible file: open → skip and count as active; otherwise rm -f
|
|
# 5. Remove empty directories older than TRANSCODE_ORPHAN_AGE, never transcoding-temp
|
|
#
|
|
# Afterwards, if the ramdisk recovered enough headroom, the symlink is flipped back to
|
|
# it so new sessions return to RAM.
|
|
#
|
|
# ==============================================================================================
|
|
# DESIGN PRINCIPLES
|
|
# ==============================================================================================
|
|
#
|
|
# Two Independent Conditions, Both Required
|
|
# Age and open-file state are checked separately and a file must pass both. Age alone
|
|
# would delete a long-running session's segments; lsof alone cannot see HLS segments,
|
|
# which are written and closed atomically. Neither signal is sufficient on its own,
|
|
# which is exactly why both are applied rather than picking the better one.
|
|
#
|
|
# Mounted-Only Ramdisk Cleaning
|
|
# The ramdisk is only cleaned when it is genuinely mounted. If the tmpfs failed to
|
|
# mount, that same path is an ordinary directory on the array — cleaning it then would
|
|
# delete real files from disk while believing it was clearing RAM.
|
|
#
|
|
# lsof Called Once, Not Per File
|
|
# On a busy Live TV system the ramdisk contains thousands of HLS segment files.
|
|
# Calling lsof once per file creates thousands of subprocess calls every 7 minutes.
|
|
# lsof is called once per location to build a complete open-file map. All subsequent
|
|
# checks are O(1) lookups against that map — thousands of files, one lsof call.
|
|
#
|
|
# find -printf Instead of Per-File stat/basename (2026-07-17)
|
|
# Same fork-elimination principle as above, applied to the file-processing loop. find
|
|
# already has to stat() every entry to know it's -type f, so -printf '%s %p' gets the
|
|
# size for free during the walk instead of a separate stat fork per file. basename was
|
|
# also forking per file in the dry-run log line -- replaced with parameter expansion
|
|
# (${file##*/}). Same class of bug found and measured (~28-185x per call) in the arr
|
|
# cleanup scripts' classification loops the same day.
|
|
#
|
|
# No Session-Aware Cleanup
|
|
# ffmpeg generates folder names independently of the media server API session IDs.
|
|
# There is no reliable correlation between API session IDs and transcoding-temp
|
|
# subfolder names. Attempting to correlate them would falsely treat active sessions
|
|
# as ended. lsof is the correct check — if ffmpeg has a file open, it is active
|
|
# regardless of folder naming or session state.
|
|
#
|
|
# transcoding-temp Is Never Deleted
|
|
# If cleanup removes the empty transcoding-temp folder from the ramdisk, Emby
|
|
# searches all accessible paths for an existing one, finds the SSD fallback version,
|
|
# and routes all new sessions there until Emby restarts. The directory is excluded
|
|
# from find by name — protected even when completely empty.
|
|
#
|
|
# Post-Cleanup Flip-Back
|
|
# After removing stale files, checks whether ramdisk usage dropped below
|
|
# RAMDISK_LOW_GB. If so — and symlink currently points at SSD — triggers a
|
|
# flip back to ramdisk. This is the recovery path; the manager handles
|
|
# the fill-up path.
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL SAFEGUARDS
|
|
# ==============================================================================================
|
|
#
|
|
# Location Path Guard
|
|
# cleanup_location() refuses any path that is not absolute with at least two
|
|
# components. Neither caller's own check catches a collapsed value — mountpoint -q
|
|
# returns true for /, and -d is true for both / and /mnt — so the guard lives inside
|
|
# the function that does the deleting, covering both call sites.
|
|
#
|
|
#
|
|
# Wait Lock
|
|
# acquire_lock "wait" — waits if a previous cleanup run is still active rather
|
|
# than exiting. The caller's 7-minute interval can overlap on a slow system.
|
|
#
|
|
# lsof Timeout
|
|
# lsof call capped at 15 seconds per location — prevents blocking indefinitely
|
|
# on a system with many open files.
|
|
#
|
|
# transcoding-temp Guard
|
|
# `! -name "transcoding-temp"` in the find command — protected unconditionally.
|
|
#
|
|
# Silent by Default
|
|
# Runs every 7 minutes — must not produce noise when healthy.
|
|
#
|
|
# ==============================================================================================
|
|
# CONFIGURATION
|
|
# ==============================================================================================
|
|
#
|
|
# host*.conf
|
|
#
|
|
# HOST*_RAMDISK_PATH / HOST*_TRANSCODE_SSD / HOST*_RAMDISK_LOW_GB
|
|
# Aliased by detect_hosts() → RAMDISK_PATH / TRANSCODE_SSD / RAMDISK_LOW_GB.
|
|
#
|
|
# master.conf
|
|
#
|
|
# TRANSCODE_MAX_AGE
|
|
# Minutes before an inactive transcode file is eligible for deletion. (default: 20)
|
|
#
|
|
# TRANSCODE_ORPHAN_AGE
|
|
# Minutes for orphan folder detection. (default: 30)
|
|
#
|
|
# ==============================================================================================
|
|
# RUNTIME MODES
|
|
# ==============================================================================================
|
|
#
|
|
# transcode_cleanup.sh
|
|
# Remove stale files from ramdisk and SSD. Check for flip-back opportunity.
|
|
#
|
|
# transcode_cleanup.sh --dry-run
|
|
# Show which files would be deleted. No deletions, no flip.
|
|
#
|
|
# transcode_cleanup.sh --status
|
|
# Show current file counts, ages, and open-file status per location.
|
|
#
|
|
# transcode_cleanup.sh --log
|
|
# Verbose per-file output including age, open status, and deletion result.
|
|
#
|
|
# ==============================================================================================
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
source "$SCRIPT_DIR/../load_config.sh"
|
|
|
|
parse_args "$@"
|
|
|
|
STATE_FILE="${TRANSCODE_STATE_FILE:-${STATE_DIR:-/tmp}/transcode_state.db}"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Setup ━━━
|
|
# ==============================================================================================
|
|
if [[ "$EUID" -ne 0 ]]; then
|
|
error "Must be run as root"
|
|
exit 1
|
|
fi
|
|
|
|
|
|
acquire_lock "wait"
|
|
|
|
# detect_hosts() sets MY_ID and aliases RAMDISK_PATH, TRANSCODE_SSD, RAMDISK_LOW_GB
|
|
detect_hosts
|
|
|
|
# lsof availability check
|
|
LSOF_AVAILABLE=false
|
|
if command -v lsof >/dev/null 2>&1; then
|
|
LSOF_AVAILABLE=true
|
|
log "lsof available — active file check enabled"
|
|
else
|
|
warn "lsof not available — active file check skipped, all aged files eligible for deletion"
|
|
fi
|
|
|
|
log "Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
log "$ICON_GEAR Config: max-age=${TRANSCODE_MAX_AGE}min orphan-age=${TRANSCODE_ORPHAN_AGE}min flip-back-below=${RAMDISK_LOW_GB}GB"
|
|
log "$ICON_RAM Locations: ramdisk=$RAMDISK_PATH ssd=$TRANSCODE_SSD"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Status ━━━
|
|
# ==============================================================================================
|
|
if [[ "$SHOW_STATUS" == true ]]; then
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_RAM Ramdisk: $RAMDISK_PATH"
|
|
echo "$ICON_DISK SSD fallback: $TRANSCODE_SSD"
|
|
echo "$ICON_TRASH Max age: ${TRANSCODE_MAX_AGE} minutes"
|
|
echo "$ICON_TRASH Orphan age: ${TRANSCODE_ORPHAN_AGE} minutes"
|
|
echo "$ICON_RAM Flip at: ${RAMDISK_LOW_GB}GB"
|
|
echo "$ICON_GEAR lsof check: $LSOF_AVAILABLE"
|
|
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
|
echo ""
|
|
|
|
if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
|
|
USAGE=$(df -BG "$RAMDISK_PATH" | awk 'NR==2 {print $3}')
|
|
AVAIL=$(df -BG "$RAMDISK_PATH" | awk 'NR==2 {print $4}')
|
|
echo " $ICON_RAM Ramdisk: mounted — $USAGE used / $AVAIL available ✅"
|
|
else
|
|
echo " $ICON_RAM Ramdisk: not mounted"
|
|
fi
|
|
|
|
CURRENT_TARGET=$(grep "^current_target=" "$STATE_FILE" 2>/dev/null | cut -d= -f2)
|
|
echo " $ICON_LINK Current target: ${CURRENT_TARGET:-unknown}"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
fi
|
|
|
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be deleted"
|
|
|
|
# ==============================================================================================
|
|
# ── CLEANUP FUNCTION ──────────────────────────────────────────────────────────────────────────
|
|
# ==============================================================================================
|
|
# Scans a location and removes eligible files.
|
|
# Calls lsof ONCE per location — builds in-memory OPEN_FILES_MAP for O(1) lookup.
|
|
# Returns via LOCATION_REMOVED, LOCATION_FREED, LOCATION_SKIPPED, LOCATION_ACTIVE, LOCATION_FAILED
|
|
|
|
cleanup_location() {
|
|
local location="$1" label="$2" max_age="$3"
|
|
local files_removed=0 bytes_freed=0 files_skipped=0 files_active=0 files_streaming=0 files_too_young=0 files_failed=0
|
|
|
|
if [[ ! -d "$location" ]]; then
|
|
log "$label does not exist — skipping"
|
|
LOCATION_REMOVED=0 LOCATION_FREED="0B" LOCATION_SKIPPED=0 LOCATION_ACTIVE=0 LOCATION_STREAMING=0 LOCATION_TOO_YOUNG=0 LOCATION_FAILED=0
|
|
return
|
|
fi
|
|
|
|
# This function deletes every file under $location past the age gate. Neither caller's
|
|
# own check catches a collapsed path: mountpoint -q returns true for /, and -d is true
|
|
# for / and /mnt alike. Require at least two path components so a blank or truncated
|
|
# RAMDISK_PATH / TRANSCODE_SSD can never point this at a system directory.
|
|
local _loc_slashes="${location//[^\/]/}"
|
|
if [[ "$location" != /* || "${#_loc_slashes}" -lt 2 ]]; then
|
|
error "Refusing to clean unsafe location: '$location' ($label)"
|
|
notify "Transcode cleanup refused unsafe path on $(hostname): '$location'" \
|
|
"Transcode Cleanup" "warning"
|
|
LOCATION_REMOVED=0 LOCATION_FREED="0B" LOCATION_SKIPPED=0 LOCATION_ACTIVE=0 LOCATION_STREAMING=0 LOCATION_TOO_YOUNG=0 LOCATION_FAILED=0
|
|
return
|
|
fi
|
|
|
|
local file_count eligible_count
|
|
file_count=$(find "$location" -type f 2>/dev/null | wc -l)
|
|
eligible_count=$(find "$location" -type f -mmin +"$max_age" 2>/dev/null | wc -l)
|
|
files_too_young=$(( file_count - eligible_count ))
|
|
log "$label: $file_count files ($eligible_count eligible, $files_too_young < ${max_age}min)"
|
|
|
|
# Build in-memory open file map — O(1) lookup per file
|
|
# One lsof call per location — never per file
|
|
# Note: HLS/remux segments (Live TV, Direct Stream) are written atomically and immediately
|
|
# closed — lsof will not detect them. The age check is the effective guard for those files.
|
|
declare -A OPEN_FILES_MAP
|
|
if [[ "$LSOF_AVAILABLE" == true ]]; then
|
|
log "Building open file map for $label..."
|
|
while IFS= read -r open_file; do
|
|
[[ -n "$open_file" ]] && OPEN_FILES_MAP["$open_file"]=1
|
|
done < <(timeout 15 lsof +D "$location" 2>/dev/null | awk 'NR>1 {print $9}' | sort -u)
|
|
files_streaming=${#OPEN_FILES_MAP[@]}
|
|
log "$files_streaming files currently open in $label"
|
|
fi
|
|
|
|
# Process aged files. -printf gets size directly from find's own stat() during the walk
|
|
# instead of a separate stat fork per file (2026-07-17, same fix as the arr cleanup
|
|
# scripts) — mtime isn't needed here since -mmin above already did the age filtering.
|
|
local file_size file
|
|
while read -r file_size file; do
|
|
[[ -z "$file" ]] && continue
|
|
|
|
# O(1) open file check — in-memory map
|
|
if [[ -n "${OPEN_FILES_MAP[$file]:-}" ]]; then
|
|
(( files_active++ ))
|
|
log "Skipping open file: $file"
|
|
continue
|
|
fi
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
log "DRY RUN — would delete: ${file##*/}"
|
|
(( files_skipped++ ))
|
|
else
|
|
if rm -f "$file" 2>/dev/null; then
|
|
(( files_removed++ ))
|
|
bytes_freed=$(( bytes_freed + file_size ))
|
|
log "Deleted: $file"
|
|
else
|
|
warn "Could not delete: $file"
|
|
(( files_failed++ ))
|
|
fi
|
|
fi
|
|
|
|
done < <(find "$location" -type f -mmin +"$max_age" -printf '%s %p\n' 2>/dev/null)
|
|
|
|
# Remove empty directories older than TRANSCODE_ORPHAN_AGE — but NEVER remove
|
|
# transcoding-temp. Age gate avoids deleting a session folder ffmpeg just
|
|
# created but hasn't written its first segment into yet.
|
|
if [[ "$DRY_RUN" == false ]]; then
|
|
find "$location" -mindepth 1 -type d -empty -mmin +"${TRANSCODE_ORPHAN_AGE:-30}" \
|
|
! -name "transcoding-temp" -delete 2>/dev/null
|
|
fi
|
|
|
|
# Format bytes freed
|
|
local freed_human
|
|
freed_human=$(format_bytes "$bytes_freed")
|
|
|
|
log "$label — removed $files_removed ($freed_human) | active(fresh): $files_too_young | streaming(lsof): $files_streaming | protected(old+open): $files_active | skipped: $files_skipped | failed: $files_failed"
|
|
|
|
LOCATION_REMOVED=$files_removed
|
|
LOCATION_FREED=$freed_human
|
|
LOCATION_SKIPPED=$files_skipped
|
|
LOCATION_ACTIVE=$files_active
|
|
LOCATION_STREAMING=$files_streaming
|
|
LOCATION_TOO_YOUNG=$files_too_young
|
|
LOCATION_FAILED=$files_failed
|
|
}
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Transcode Cleanup ━━━
|
|
# ==============================================================================================
|
|
START=$(date +%s)
|
|
TOTAL_REMOVED=0 TOTAL_SKIPPED=0 TOTAL_ACTIVE=0 TOTAL_STREAMING=0 TOTAL_TOO_YOUNG=0 TOTAL_FAILED=0
|
|
RAMDISK_FREED="0B" SSD_FREED="0B"
|
|
|
|
# Cleanup ramdisk
|
|
if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
|
|
cleanup_location "$RAMDISK_PATH" "Ramdisk" "$TRANSCODE_MAX_AGE"
|
|
TOTAL_REMOVED=$(( TOTAL_REMOVED + LOCATION_REMOVED ))
|
|
TOTAL_SKIPPED=$(( TOTAL_SKIPPED + LOCATION_SKIPPED ))
|
|
TOTAL_ACTIVE=$(( TOTAL_ACTIVE + LOCATION_ACTIVE ))
|
|
TOTAL_STREAMING=$(( TOTAL_STREAMING + LOCATION_STREAMING ))
|
|
TOTAL_TOO_YOUNG=$(( TOTAL_TOO_YOUNG + LOCATION_TOO_YOUNG ))
|
|
TOTAL_FAILED=$(( TOTAL_FAILED + LOCATION_FAILED ))
|
|
RAMDISK_FREED=$LOCATION_FREED
|
|
else
|
|
log "Ramdisk not mounted — skipping ramdisk cleanup"
|
|
fi
|
|
|
|
# Cleanup SSD fallback
|
|
if [[ -d "$TRANSCODE_SSD" ]]; then
|
|
cleanup_location "$TRANSCODE_SSD" "SSD fallback" "$TRANSCODE_MAX_AGE"
|
|
TOTAL_REMOVED=$(( TOTAL_REMOVED + LOCATION_REMOVED ))
|
|
TOTAL_SKIPPED=$(( TOTAL_SKIPPED + LOCATION_SKIPPED ))
|
|
TOTAL_ACTIVE=$(( TOTAL_ACTIVE + LOCATION_ACTIVE ))
|
|
TOTAL_STREAMING=$(( TOTAL_STREAMING + LOCATION_STREAMING ))
|
|
TOTAL_TOO_YOUNG=$(( TOTAL_TOO_YOUNG + LOCATION_TOO_YOUNG ))
|
|
TOTAL_FAILED=$(( TOTAL_FAILED + LOCATION_FAILED ))
|
|
SSD_FREED=$LOCATION_FREED
|
|
else
|
|
log "SSD fallback not found — skipping SSD cleanup"
|
|
fi
|
|
|
|
# Post-cleanup — check if ramdisk recovered enough to flip symlink back to ramdisk
|
|
if [[ "$DRY_RUN" == false ]] && mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
|
|
RAMDISK_USED_KB=$(df "$RAMDISK_PATH" --output=used 2>/dev/null | tail -1 | tr -d ' ')
|
|
RAMDISK_USED_GB=$(kb_to_gb "$RAMDISK_USED_KB")
|
|
LOW_RECOVERED=$(awk "BEGIN {print ($RAMDISK_USED_GB < $RAMDISK_LOW_GB) ? 1 : 0}")
|
|
CURRENT_TARGET=$(grep "^current_target=" "$STATE_FILE" 2>/dev/null | cut -d= -f2)
|
|
|
|
if [[ "$LOW_RECOVERED" == "1" && "$CURRENT_TARGET" == "$TRANSCODE_SSD" ]]; then
|
|
echo "Ramdisk has space after cleanup (${RAMDISK_USED_GB}GB < ${RAMDISK_LOW_GB}GB) — triggering manager to flip back"
|
|
bash "$SCRIPT_DIR/transcode_manager.sh" --no-log
|
|
fi
|
|
fi
|
|
|
|
END=$(date +%s)
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Summary ━━━
|
|
# ==============================================================================================
|
|
echo "━━━━━ $ICON_SUMMARY TRANSCODE CLEANUP SUMMARY ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_RAM Ramdisk freed: $RAMDISK_FREED"
|
|
echo "$ICON_DISK SSD freed: $SSD_FREED"
|
|
echo "$ICON_TRASH Removed: $TOTAL_REMOVED files"
|
|
echo "$ICON_RUNNING Active: $TOTAL_TOO_YOUNG files (< ${TRANSCODE_MAX_AGE}min old — Live TV / Direct Stream segments)"
|
|
echo "$ICON_RUNNING Streaming: $TOTAL_STREAMING files (open file handle — long-running transcode)"
|
|
echo "$ICON_SHIELD Protected: $TOTAL_ACTIVE aged files saved by open-file check"
|
|
echo "$ICON_TRASH Skipped: $TOTAL_SKIPPED files"
|
|
[[ "$TOTAL_FAILED" -gt 0 ]] && echo "$ICON_ERROR Failed: $TOTAL_FAILED files could not be deleted"
|
|
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — no files deleted"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
elif [[ "$TOTAL_FAILED" -gt 0 ]]; then
|
|
warn "Status: $TOTAL_FAILED file(s) failed to delete"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 1
|
|
else
|
|
echo "$ICON_DONE Status: done ✅"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
fi |