FailedProxy/Varaverk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fixed transcode cleanup

Browse Source
This commit is contained in:
FailedProxy
2026-04-11 10:37:14 -04:00
parent 02551ae215
commit ee107e5df7
1 changed files with 71 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Transcodes/transcode_cleanup.sh
+71 -68
View File
@@ -3,13 +3,17 @@
# --------------------------------- Transcode Cleanup ------------------------------------------
# -----------------------------------------------------------------------------------------------
# Removes old inactive transcode files from both ramdisk and SSD fallback locations.
# Never deletes files belonging to active sessions or files being actively written.
# Never deletes files that are currently open by any process.
# After cleanup checks if ramdisk usage dropped enough to flip symlink back to ramdisk.
#
# Safety rules — a file is eligible for deletion only if ALL conditions are true:
# 1. Older than TRANSCODE_MAX_AGE minutes (mtime)
# 2. Not currently open by any process (lsof check)
# 3. Not in an orphan grace period shorter than TRANSCODE_ORPHAN_AGE minutes
# 2. Not currently open by any process (single lsof call per location — not per file)
#
# Performance note:
# lsof is called ONCE per location to build an open file list — not once per file.
# This is critical for locations with hundreds or thousands of segment files.
# A per-file lsof approach stalls on busy systems with live TV buffering.
#
# Run every 5 minutes via cron/User Scripts plugin.
# All configuration in Master.conf under Transcode Manager section.
@@ -39,7 +43,7 @@ fi
success "Running as root"
if ! command -v lsof >/dev/null 2>&1; then
warn "lsof not available — active file check will be skipped"
warn "lsof not available — active file check will be skipped, all aged files will be eligible"
LSOF_AVAILABLE=false
else
LSOF_AVAILABLE=true
@@ -63,42 +67,23 @@ fi
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be deleted"
# -----------------------------------------------------------------------------------------------
# FILE SAFETY CHECK
# Returns 0 (safe to delete) if file is old enough and not actively open.
# Returns 1 (keep) if file should be preserved.
# -----------------------------------------------------------------------------------------------
is_safe_to_delete() {
local file="$1" max_age="$2"
# Check age using find — if find returns the file it's old enough
if ! find "$file" -maxdepth 0 -mmin +"$max_age" 2>/dev/null | grep -q .; then
return 1 # Too recent
fi
# Check if file is actively open by any process
if [[ "$LSOF_AVAILABLE" == true ]]; then
if lsof "$file" >/dev/null 2>&1; then
return 1 # File is open — being written or read
fi
fi
return 0 # Safe to delete
}
# -----------------------------------------------------------------------------------------------
# CLEANUP FUNCTION
# Scans a directory and removes eligible files.
# Returns bytes freed.
# Scans a location and removes eligible files.
# Calls lsof ONCE per location to build open file list — never per file.
# -----------------------------------------------------------------------------------------------
cleanup_location() {
local location="$1" label="$2" max_age="$3"
local files_removed=0
local bytes_freed=0
local files_skipped=0
local files_active=0
if [[ ! -d "$location" ]]; then
warn "$label does not exist — skipping"
LOCATION_REMOVED=0
LOCATION_FREED="0B"
LOCATION_SKIPPED=0
return
fi
@@ -106,56 +91,73 @@ cleanup_location() {
file_count=$(find "$location" -type f 2>/dev/null | wc -l)
info "$ICON_TRASH $label: $file_count files to scan (age threshold: ${max_age}min)"
# Build open file list with a single lsof call — timeout prevents stalling
local OPEN_FILES=""
if [[ "$LSOF_AVAILABLE" == true ]]; then
info "Building open file list for $label..."
OPEN_FILES=$(timeout 15 lsof +D "$location" 2>/dev/null | awk 'NR>1 {print $9}' | sort -u)
local open_count
open_count=$(echo "$OPEN_FILES" | grep -c "." 2>/dev/null || echo 0)
info "$open_count files currently open in $label"
fi
# Find files older than max_age and process them
while IFS= read -r file; do
[[ -z "$file" ]] && continue
if is_safe_to_delete "$file" "$max_age"; then
local file_size
file_size=$(stat -c%s "$file" 2>/dev/null || echo 0)
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would delete: $file"
((files_skipped++))
else
if rm -f "$file" 2>/dev/null; then
((files_removed++))
bytes_freed=$((bytes_freed + file_size))
log "Deleted: $file"
else
warn "Could not delete: $file"
((files_skipped++))
fi
fi
else
((files_skipped++))
log "Keeping: $file (active or too recent)"
# Check if file is currently open — fast string match against pre-built list
if [[ "$LSOF_AVAILABLE" == true ]] && echo "$OPEN_FILES" | grep -qF "$file"; then
((files_active++))
log "Skipping open file: $file"
continue
fi
done < <(find "$location" -type f 2>/dev/null)
local file_size
file_size=$(stat -c%s "$file" 2>/dev/null || echo 0)
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would delete: $(basename "$file")"
((files_skipped++))
else
if rm -f "$file" 2>/dev/null; then
((files_removed++))
bytes_freed=$((bytes_freed + file_size))
log "Deleted: $file"
else
warn "Could not delete: $file"
((files_skipped++))
fi
fi
done < <(find "$location" -type f -mmin +"$max_age" 2>/dev/null)
# Remove empty directories left behind
if [[ "$DRY_RUN" == false ]]; then
find "$location" -mindepth 1 -type d -empty -delete 2>/dev/null
fi
# Format bytes freed for display
local freed_human
if (( bytes_freed > 1073741824 )); then
freed_human=$(awk "BEGIN {printf \"%.1fGB\", $bytes_freed / 1073741824}")
elif (( bytes_freed > 1048576 )); then
freed_human=$(awk "BEGIN {printf \"%.1fMB\", $bytes_freed / 1048576}")
else
elif (( bytes_freed > 0 )); then
freed_human="${bytes_freed}B"
else
freed_human="0B"
fi
if [[ "$DRY_RUN" == true ]]; then
info "$label — dry run complete ($files_skipped files scanned)"
info "$label — dry run complete ($file_count files scanned, $files_active active)"
else
success "$label — removed $files_removed files, freed $freed_human, skipped $files_skipped"
success "$label — removed $files_removed files ($freed_human freed), $files_active active, $files_skipped skipped"
fi
# Export for summary
LOCATION_REMOVED=$files_removed
LOCATION_FREED=$freed_human
LOCATION_SKIPPED=$files_skipped
LOCATION_ACTIVE=$files_active
}
# -----------------------------------------------------------------------------------------------
@@ -168,19 +170,21 @@ echo ""
START=$(date +%s)
TOTAL_REMOVED=0
TOTAL_SKIPPED=0
TOTAL_ACTIVE=0
RAMDISK_FREED="0B"
SSD_FREED="0B"
# Cleanup ramdisk
if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
echo "━━━ $ICON_RAM Ramdisk ━━━"
RAMDISK_BEFORE=$(df "$RAMDISK_PATH" | awk 'NR==2 {print $3}')
cleanup_location "$RAMDISK_PATH" "Ramdisk" "$TRANSCODE_MAX_AGE"
TOTAL_REMOVED=$((TOTAL_REMOVED + LOCATION_REMOVED))
TOTAL_SKIPPED=$((TOTAL_SKIPPED + LOCATION_SKIPPED))
TOTAL_ACTIVE=$((TOTAL_ACTIVE + LOCATION_ACTIVE))
RAMDISK_FREED=$LOCATION_FREED
echo ""
else
warn "$ICON_RAM Ramdisk not mounted — skipping ramdisk cleanup"
RAMDISK_FREED="0B"
fi
# Cleanup SSD fallback
@@ -189,27 +193,25 @@ if [[ -d "$TRANSCODE_SSD" ]]; then
cleanup_location "$TRANSCODE_SSD" "SSD fallback" "$TRANSCODE_MAX_AGE"
TOTAL_REMOVED=$((TOTAL_REMOVED + LOCATION_REMOVED))
TOTAL_SKIPPED=$((TOTAL_SKIPPED + LOCATION_SKIPPED))
TOTAL_ACTIVE=$((TOTAL_ACTIVE + LOCATION_ACTIVE))
SSD_FREED=$LOCATION_FREED
echo ""
else
info "$ICON_DISK SSD fallback not found — skipping"
SSD_FREED="0B"
fi
# -----------------------------------------------------------------------------------------------
# Post-cleanup symlink check
# If ramdisk had space freed, check if manager should flip symlink back
# Post-cleanup — check if ramdisk recovered enough to flip symlink back
# -----------------------------------------------------------------------------------------------
if [[ "$DRY_RUN" == false ]] && mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
RAMDISK_USED_KB=$(df "$RAMDISK_PATH" | awk 'NR==2 {print $3}')
LOW_THRESHOLD=$(awk "BEGIN {print $RAMDISK_LOW_GB * 1024 * 1024}")
RAMDISK_USED_KB=$(df "$RAMDISK_PATH" --output=used | tail -1 | tr -d ' ')
RAMDISK_USED_GB=$(awk "BEGIN {printf \"%.2f\", $RAMDISK_USED_KB / 1048576}")
LOW_RECOVERED=$(awk "BEGIN {print ($RAMDISK_USED_GB < $RAMDISK_LOW_GB) ? 1 : 0}")
CURRENT_TARGET=$(grep "^current_target=" "$STATE_FILE" 2>/dev/null | cut -d'=' -f2)
if (( RAMDISK_USED_KB < LOW_THRESHOLD )); then
CURRENT_TARGET=$(grep "^current_target=" "$STATE_FILE" 2>/dev/null | cut -d'=' -f2)
if [[ "$CURRENT_TARGET" == "$TRANSCODE_SSD" ]]; then
info "$ICON_RAM Ramdisk has space after cleanup — triggering manager to flip back"
bash "$SCRIPT_DIR/transcode_manager.sh" --no-log
fi
if [[ "$LOW_RECOVERED" == "1" && "$CURRENT_TARGET" == "$TRANSCODE_SSD" ]]; then
info "$ICON_RAM Ramdisk has space after cleanup — triggering manager to flip back"
bash "$SCRIPT_DIR/transcode_manager.sh" --no-log
fi
fi
@@ -222,7 +224,8 @@ echo "━━━━━ $ICON_SUMMARY TRANSCODE CLEANUP SUMMARY ━━━━━"
echo "$ICON_RAM Ramdisk freed: $RAMDISK_FREED"
echo "$ICON_DISK SSD freed: $SSD_FREED"
echo "$ICON_TRASH Removed: $TOTAL_REMOVED files"
echo "$ICON_TRASH Skipped: $TOTAL_SKIPPED files (active or recent)"
echo "$ICON_RUNNING Active: $TOTAL_ACTIVE files (open — protected)"
echo "$ICON_TRASH Skipped: $TOTAL_SKIPPED files"
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
if [[ "$DRY_RUN" == true ]]; then
echo "$ICON_WARN Status: DRY RUN — no files deleted"