diff --git a/Master.conf b/Master.conf index d520203..2438c7b 100644 --- a/Master.conf +++ b/Master.conf @@ -36,6 +36,9 @@ # MEDIA PERMISSIONS Share list, mode and owner for permissions script # MEDIA CLEANER Anime and media folder lists and file patterns # +# ── TRANSCODES ───────────────────────────────────────────────────────────────────────────── +# TRANSCODE MANAGER Ramdisk and SSD fallback transcode management +# # ── SYSTEM WATCHDOG ──────────────────────────────────────────────────────────────────────── # SYSTEM WATCHDOG System health monitoring — last line of defense # @@ -90,7 +93,6 @@ # ━━━ Daily Sync Shares ━━━ # Shares synced once daily by Orchestrators/daily_sync.sh # No profile needed — all fall through to DEFAULT_RSYNC_OPTS above. -# Add or remove paths here to manage what gets synced. DAILY_SYNC_SHARES=( /mnt/user/Anime_Movies-Old /mnt/user/Anime_Shows-Old @@ -227,6 +229,7 @@ WEEKLY_RESTART_CONTAINERS=( # 8GB=8192 6GB=6144 4GB=4096 2GB=2048 1GB=1024 declare -A WATCHDOG_CONTAINERS=( ["Emby"]=16384 + ["jellyfin_with_request"]=12288 ["LidaTube"]=6144 ["Tdarr"]=6144 ["Code-Server"]=1024 @@ -235,6 +238,7 @@ declare -A WATCHDOG_CONTAINERS=( # Containers to check HTTP responsiveness — omit to skip declare -A WATCHDOG_CONTAINER_URLS=( ["Emby"]="http://localhost:8096" + ["Jellyfin-Gmer4Lfe"]="http://localhost:8095" ) # Containers that should always be running — monitored for unexpected stops @@ -250,7 +254,7 @@ WATCHDOG_REQUIRED_CONTAINERS=( # CPU thresholds — normalised against total core count at runtime SOFT_CPU_THRESHOLD=80 # warn at this % of total system CPU - HARD_CPU_THRESHOLD=90 # strike at this % of total system CPU + HARD_CPU_THRESHOLD=85 # strike at this % of total system CPU CPU_FAIL_LIMIT=2 # consecutive strikes before restart # Memory threshold @@ -265,24 +269,19 @@ WATCHDOG_REQUIRED_CONTAINERS=( # ============================================================================================== # ━━━ Reboot ━━━ -# User warning delay before scheduled reboot (seconds) - REBOOT_SLEEP=300 + REBOOT_SLEEP=300 # user warning delay before scheduled reboot (seconds) # ━━━ Mover ━━━ -# Timeout before stopping the mover (seconds) - MOVER_STOP_TIMEOUT=300 + MOVER_STOP_TIMEOUT=300 # timeout before stopping the mover (seconds) # ━━━ Syslog Filter ━━━ -# Path for the rsyslog Docker noise filter file FILTER_FILE="/etc/rsyslog.d/ignore-docker-veth.conf" # ━━━ PHP-FPM ━━━ -# PHP-FPM config file path and max children value PHP_CONF="/etc/php-fpm.d/www.conf" PHP_MAX_CHILDREN=250 # ━━━ Clear Logs ━━━ -# System log files to clear on each run LOG_FILES=(/var/log/syslog /var/log/messages /var/log/dmesg) # ============================================================================================== @@ -290,7 +289,6 @@ WATCHDOG_REQUIRED_CONTAINERS=( # ============================================================================================== # ━━━ Media Permissions ━━━ -# Mode and owner applied recursively to all listed shares PERMISSIONS_MODE="777" PERMISSIONS_OWNER="nobody:users" @@ -340,7 +338,6 @@ MEDIA_CLEAN_FOLDERS=( /mnt/user/Tv_Shows ) -# Anime file patterns — junk files common in anime downloads ANIME_FILE_PATTERNS=( '*.sfv' '*.md5' '*.sha1' '*.txt' '*.url' '*.lnk' '*.rar' '*.zip' '*.info' '*.torrent' '*.sample*' '*.proof*' @@ -348,7 +345,6 @@ ANIME_FILE_PATTERNS=( '*.log' '*.json' ) -# Media file patterns — includes *.iso and *.lrc not needed in anime MEDIA_FILE_PATTERNS=( '*.sfv' '*.md5' '*.sha1' '*.txt' '*.url' '*.lnk' '*.rar' '*.zip' '*.info' '*.torrent' '*.sample*' '*.proof*' @@ -356,44 +352,70 @@ MEDIA_FILE_PATTERNS=( '*.log' '*.json' '*.iso' '*.lrc' ) +# ============================================================================================== +# ── TRANSCODES ──────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# Session-based storage allocator using filesystem indirection. +# New transcode sessions land wherever TRANSCODE_LINK points. +# Existing sessions are never interrupted — ffmpeg resolves path once at session start. +# +# Flow: +# ramdisk_setup.sh — run once at array start, creates ramdisk and symlink +# transcode_manager.sh — runs every 2-3 min, monitors usage and flips symlink +# transcode_cleanup.sh — runs every 5 min, removes old inactive files +# +# Hysteresis gap between RAMDISK_WARN_GB and RAMDISK_LOW_GB prevents flip-flop +# when usage hovers near the threshold. Gap should be at least 0.5-1GB. + +# ━━━ Transcode Manager ━━━ +# Paths + RAMDISK_PATH="/mnt/ramdisk_transcodes" # tmpfs mount point + RAMDISK_SIZE="8G" # increase if you have RAM headroom + TRANSCODE_LINK="/mnt/user/ram-transcode" # symlink Emby points at + TRANSCODE_SSD="/mnt/cache/Temp_Storage/Emby/Transcodes/" # SSD fallback location + +# Thresholds in GB + RAMDISK_WARN_GB=6.8 # flip symlink to SSD at or above this usage + RAMDISK_LOW_GB=5.5 # flip symlink back to ramdisk when usage drops here + RAMDISK_SSD_MIN_GB=20 # minimum free GB on SSD before allowing flip — abort if below + +# Cleanup settings + TRANSCODE_MAX_AGE=20 # minutes before a file is eligible for cleanup + TRANSCODE_ORPHAN_AGE=30 # minutes before an orphaned file is eligible (slightly longer buffer) + +# Flip frequency monitoring + TRANSCODE_FLIP_WARN=3 # notify if symlink flips this many times in one hour + +# Permissions — should match your Emby container user + TRANSCODE_OWNER="nobody:users" + TRANSCODE_MODE="755" + # ============================================================================================== # ── SYSTEM WATCHDOG ─────────────────────────────────────────────────────────────────────────── # ============================================================================================== # Last line of defense — reboots the system cleanly if it is about to become unstable. -# Designed to run on a cron schedule (recommended every 15-30 minutes). # Works alongside docker_watchdog.sh — containers first, system second. -# -# Strike system — sustained threshold hits trigger reboot, not single spikes. -# Reboot loop protection — shuts down instead of rebooting if limit hit in window. -# Container skip list — persistent, auto-clears when container recovers. +# Thresholds set at "about to fall over" levels — not just high usage. # ━━━ System Watchdog State Files ━━━ -# Strike counts — /tmp resets on reboot, correct for strike tracking SYS_WATCHDOG_STATE_FILE="/tmp/system_watchdog_state.db" -# Persistent container skip list — survives reboots, auto-clears on recovery SYS_WATCHDOG_FAILED_FILE="/boot/config/system_watchdog_failed.db" -# Reboot timestamp log — survives reboots for loop detection SYS_WATCHDOG_REBOOT_LOG="/boot/config/system_watchdog_reboots.db" # ━━━ Strike and Reboot Loop Settings ━━━ -# Consecutive threshold hits before triggering reboot - SYS_WATCHDOG_STRIKE_LIMIT=2 -# Maximum reboots allowed within the window before shutdown instead - SYS_WATCHDOG_REBOOT_LIMIT=3 -# Window in hours — controls both reboot count window AND rolling log purge -# 12 = entries older than 12hrs purge automatically / 24 = entries older than 24hrs purge - SYS_WATCHDOG_REBOOT_WINDOW_HRS=12 + SYS_WATCHDOG_STRIKE_LIMIT=2 # consecutive hits before reboot trigger + SYS_WATCHDOG_REBOOT_LIMIT=3 # max reboots in window before shutdown instead + SYS_WATCHDOG_REBOOT_WINDOW_HRS=12 # window in hours — controls reboot count AND log purge # ━━━ Thresholds ━━━ -# Set at "about to fall over" levels — not just high usage - SYS_WATCHDOG_ROOTFS_PCT=95 # rootfs usage % before strike - SYS_WATCHDOG_LOG_PCT=95 # /var/log usage % before strike - SYS_WATCHDOG_MEM_GB=4 # free RAM in GB below which strikes + SYS_WATCHDOG_ROOTFS_PCT=95 # rootfs % before strike + SYS_WATCHDOG_LOG_PCT=95 # /var/log % before strike + SYS_WATCHDOG_MEM_GB=4 # free RAM GB below which strikes (128GB system) SYS_WATCHDOG_ARC_PINNED_PCT=98 # ZFS ARC % of max before reclaim attempt SYS_WATCHDOG_ARC_RELEASE_PCT=95 # ZFS ARC % after reclaim that still triggers - SYS_WATCHDOG_LOAD_MULTIPLIER=32 # strike if load avg > cores x this value - SYS_WATCHDOG_ZOMBIE_LIMIT=50 # strike if zombie process count exceeds this - SYS_WATCHDOG_CPU_TEMP_MAX=95 # degrees C — tjmax varies by CPU + SYS_WATCHDOG_LOAD_MULTIPLIER=3 # strike if load avg > cores x this value + SYS_WATCHDOG_ZOMBIE_LIMIT=50 # strike if zombie count exceeds this + SYS_WATCHDOG_CPU_TEMP_MAX=95 # degrees C — adjust for your CPU tjmax # ━━━ Check Toggles ━━━ # true = run this check / false = skip entirely @@ -402,17 +424,17 @@ MEDIA_FILE_PATTERNS=( SYS_WATCHDOG_CHECK_RAM=true SYS_WATCHDOG_CHECK_ARC=true SYS_WATCHDOG_CHECK_CPU_TEMP=true - SYS_WATCHDOG_CHECK_LOAD=true + SYS_WATCHDOG_CHECK_LOAD=false SYS_WATCHDOG_CHECK_ZOMBIES=true SYS_WATCHDOG_CHECK_CONTAINERS=true SYS_WATCHDOG_CHECK_DOCKER_DAEMON=true # ━━━ Abort Toggles ━━━ -# true = abort reboot if condition is active / false = reboot anyway -# Default true = conservative / set false to reboot regardless +# true = abort reboot if condition active / false = reboot anyway +# Default true = conservative — set false only when you want "reboot no matter what" SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY=true - SYS_WATCHDOG_ABORT_ON_PARITY=false - SYS_WATCHDOG_ABORT_ON_MOVER=false + SYS_WATCHDOG_ABORT_ON_PARITY=true + SYS_WATCHDOG_ABORT_ON_MOVER=true # ============================================================================================== # ──────────────────────── End Of User Variables ─────────────────────────────────────────────── diff --git a/Transcodes/ramdisk_setup.sh b/Transcodes/ramdisk_setup.sh new file mode 100644 index 0000000..61d1608 --- /dev/null +++ b/Transcodes/ramdisk_setup.sh @@ -0,0 +1,235 @@ +#!/bin/bash +# ----------------------------------------------------------------------------------------------- +# --------------------------------- Ramdisk Setup Script --------------------------------------- +# ----------------------------------------------------------------------------------------------- +# Creates a tmpfs ramdisk for Emby transcodes and points the transcode symlink at it. +# Run once at array start via User Scripts plugin — scheduled as "At Startup of Array". +# If ramdisk already mounted reports status and exits cleanly without remounting. +# +# Creates: +# RAMDISK_PATH — tmpfs mount point (fast, in-memory transcode location) +# TRANSCODE_SSD — SSD fallback directory (created if missing) +# TRANSCODE_LINK — symlink pointing at ramdisk by default +# +# All configuration in Master.conf under Transcode Manager section. +# Supports --dry-run to preview what would be created without making changes. +# ----------------------------------------------------------------------------------------------- + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "$SCRIPT_DIR/../Master.conf" +source "$SCRIPT_DIR/../common.sh" + +parse_args "$@" + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_GEAR Setup ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_GEAR Setup ━━━" + +if [[ "$EUID" -ne 0 ]]; then + error "Must be run as root" + exit 1 +fi + +success "Running as root" + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_SUMMARY Status ━━━ +# ----------------------------------------------------------------------------------------------- +if [[ "$SHOW_STATUS" == true ]]; then + echo "" + echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" + echo "$ICON_RAM Ramdisk path: $RAMDISK_PATH" + echo "$ICON_RAM Ramdisk size: $RAMDISK_SIZE" + echo "$ICON_LINK Transcode link: $TRANSCODE_LINK" + echo "$ICON_DISK SSD fallback: $TRANSCODE_SSD" + echo "$ICON_GEAR Owner: $TRANSCODE_OWNER" + echo "$ICON_GEAR Mode: $TRANSCODE_MODE" + echo "$ICON_GEAR Dry Run: $DRY_RUN" + + # Show current mount state + if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then + CURRENT_USAGE=$(df -BG "$RAMDISK_PATH" | awk 'NR==2 {print $3}') + CURRENT_AVAIL=$(df -BG "$RAMDISK_PATH" | awk 'NR==2 {print $4}') + echo "$ICON_RAM Current usage: $CURRENT_USAGE used / $CURRENT_AVAIL available" + else + echo "$ICON_RAM Ramdisk: not mounted" + fi + + if [[ -L "$TRANSCODE_LINK" ]]; then + echo "$ICON_LINK Current target: $(readlink "$TRANSCODE_LINK")" + else + echo "$ICON_LINK Symlink: not set" + fi + echo "━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made" + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_RAM Ramdisk Setup ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_RAM Ramdisk Setup ━━━" +echo "$ICON_RAM Path: $RAMDISK_PATH" +echo "$ICON_RAM Size: $RAMDISK_SIZE" +echo "" + +START=$(date +%s) +SETUP_SUCCESS=true + +# Check if ramdisk already mounted +if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then + CURRENT_USAGE=$(df -BG "$RAMDISK_PATH" | awk 'NR==2 {print $3}') + CURRENT_AVAIL=$(df -BG "$RAMDISK_PATH" | awk 'NR==2 {print $4}') + info "$ICON_RAM Ramdisk already mounted — $CURRENT_USAGE used / $CURRENT_AVAIL available" + info "Skipping mount — verifying symlink and permissions" +else + # Create mount point + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would create $RAMDISK_PATH" + warn "DRY RUN — would mount tmpfs ${RAMDISK_SIZE} at $RAMDISK_PATH" + else + info "Creating ramdisk mount point: $RAMDISK_PATH" + mkdir -p "$RAMDISK_PATH" || { + error "Failed to create $RAMDISK_PATH" + notify "Ramdisk setup failed on $(hostname) — could not create mount point" "Ramdisk Setup" "warning" + exit 1 + } + + info "$ICON_RAM Mounting tmpfs ${RAMDISK_SIZE} at $RAMDISK_PATH..." + if mount -t tmpfs -o size="$RAMDISK_SIZE" tmpfs "$RAMDISK_PATH"; then + success "Ramdisk mounted — ${RAMDISK_SIZE} at $RAMDISK_PATH" + else + error "Failed to mount ramdisk at $RAMDISK_PATH" + notify "Ramdisk setup failed on $(hostname) — mount failed" "Ramdisk Setup" "warning" + exit 1 + fi + fi +fi + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_DISK SSD Fallback ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_DISK SSD Fallback ━━━" + +if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would create SSD fallback: $TRANSCODE_SSD" +else + if [[ -d "$TRANSCODE_SSD" ]]; then + info "$ICON_DISK SSD fallback already exists: $TRANSCODE_SSD" + else + info "Creating SSD fallback directory: $TRANSCODE_SSD" + if mkdir -p "$TRANSCODE_SSD"; then + success "SSD fallback created: $TRANSCODE_SSD" + else + error "Failed to create SSD fallback: $TRANSCODE_SSD" + SETUP_SUCCESS=false + fi + fi +fi + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_LINK Symlink ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_LINK Symlink ━━━" + +if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would set $TRANSCODE_LINK → $RAMDISK_PATH" +else + # Remove existing symlink or directory at link path + if [[ -L "$TRANSCODE_LINK" ]]; then + CURRENT_TARGET=$(readlink "$TRANSCODE_LINK") + if [[ "$CURRENT_TARGET" == "$RAMDISK_PATH" ]]; then + info "$ICON_LINK Symlink already points to ramdisk — no change needed" + else + info "$ICON_LINK Updating symlink: $CURRENT_TARGET → $RAMDISK_PATH" + ln -sfn "$RAMDISK_PATH" "$TRANSCODE_LINK" || { + error "Failed to update symlink" + SETUP_SUCCESS=false + } + fi + elif [[ -e "$TRANSCODE_LINK" ]]; then + warn "$TRANSCODE_LINK exists but is not a symlink — removing and replacing" + rm -rf "$TRANSCODE_LINK" + ln -sfn "$RAMDISK_PATH" "$TRANSCODE_LINK" || { + error "Failed to create symlink" + SETUP_SUCCESS=false + } + else + info "Creating symlink: $TRANSCODE_LINK → $RAMDISK_PATH" + # Ensure parent directory exists + mkdir -p "$(dirname "$TRANSCODE_LINK")" + ln -sfn "$RAMDISK_PATH" "$TRANSCODE_LINK" || { + error "Failed to create symlink" + SETUP_SUCCESS=false + } + fi + + if [[ "$SETUP_SUCCESS" == true ]]; then + success "$ICON_LINK $TRANSCODE_LINK → $RAMDISK_PATH" + fi +fi + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_GEAR Permissions ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_GEAR Permissions ━━━" + +if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would apply $TRANSCODE_MODE $TRANSCODE_OWNER to $RAMDISK_PATH and $TRANSCODE_SSD" +else + for path in "$RAMDISK_PATH" "$TRANSCODE_SSD"; do + if [[ -d "$path" ]]; then + chmod "$TRANSCODE_MODE" "$path" + chown "$TRANSCODE_OWNER" "$path" + success "Permissions set: $path" + fi + done +fi + +# ----------------------------------------------------------------------------------------------- +# Initialise state file +# ----------------------------------------------------------------------------------------------- +if [[ "$DRY_RUN" == false ]]; then + STATE_FILE="/tmp/transcode_state.db" + NOW=$(date +%s) + cat > "$STATE_FILE" </dev/null 2>&1; then + warn "lsof not available — active file check will be skipped" + LSOF_AVAILABLE=false +else + LSOF_AVAILABLE=true +fi + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_SUMMARY Status ━━━ +# ----------------------------------------------------------------------------------------------- +if [[ "$SHOW_STATUS" == true ]]; then + echo "" + echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" + echo "$ICON_RAM Ramdisk: $RAMDISK_PATH" + echo "$ICON_DISK SSD fallback: $TRANSCODE_SSD" + echo "$ICON_TRASH Max age: ${TRANSCODE_MAX_AGE} minutes" + echo "$ICON_TRASH Orphan age: ${TRANSCODE_ORPHAN_AGE} minutes" + echo "$ICON_GEAR lsof check: $LSOF_AVAILABLE" + echo "$ICON_GEAR Dry Run: $DRY_RUN" + echo "━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be deleted" + +# ----------------------------------------------------------------------------------------------- +# FILE SAFETY CHECK +# Returns 0 (safe to delete) if file is old enough and not actively open. +# Returns 1 (keep) if file should be preserved. +# ----------------------------------------------------------------------------------------------- +is_safe_to_delete() { + local file="$1" max_age="$2" + + # Check age using find — if find returns the file it's old enough + if ! find "$file" -maxdepth 0 -mmin +"$max_age" 2>/dev/null | grep -q .; then + return 1 # Too recent + fi + + # Check if file is actively open by any process + if [[ "$LSOF_AVAILABLE" == true ]]; then + if lsof "$file" >/dev/null 2>&1; then + return 1 # File is open — being written or read + fi + fi + + return 0 # Safe to delete +} + +# ----------------------------------------------------------------------------------------------- +# CLEANUP FUNCTION +# Scans a directory and removes eligible files. +# Returns bytes freed. +# ----------------------------------------------------------------------------------------------- +cleanup_location() { + local location="$1" label="$2" max_age="$3" + local files_removed=0 + local bytes_freed=0 + local files_skipped=0 + + if [[ ! -d "$location" ]]; then + warn "$label does not exist — skipping" + return + fi + + local file_count + file_count=$(find "$location" -type f 2>/dev/null | wc -l) + info "$ICON_TRASH $label: $file_count files to scan (age threshold: ${max_age}min)" + + while IFS= read -r file; do + [[ -z "$file" ]] && continue + + if is_safe_to_delete "$file" "$max_age"; then + local file_size + file_size=$(stat -c%s "$file" 2>/dev/null || echo 0) + + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would delete: $file" + ((files_skipped++)) + else + if rm -f "$file" 2>/dev/null; then + ((files_removed++)) + bytes_freed=$((bytes_freed + file_size)) + log "Deleted: $file" + else + warn "Could not delete: $file" + ((files_skipped++)) + fi + fi + else + ((files_skipped++)) + log "Keeping: $file (active or too recent)" + fi + done < <(find "$location" -type f 2>/dev/null) + + # Remove empty directories left behind + if [[ "$DRY_RUN" == false ]]; then + find "$location" -mindepth 1 -type d -empty -delete 2>/dev/null + fi + + local freed_human + if (( bytes_freed > 1073741824 )); then + freed_human=$(awk "BEGIN {printf \"%.1fGB\", $bytes_freed / 1073741824}") + elif (( bytes_freed > 1048576 )); then + freed_human=$(awk "BEGIN {printf \"%.1fMB\", $bytes_freed / 1048576}") + else + freed_human="${bytes_freed}B" + fi + + if [[ "$DRY_RUN" == true ]]; then + info "$label — dry run complete ($files_skipped files scanned)" + else + success "$label — removed $files_removed files, freed $freed_human, skipped $files_skipped" + fi + + # Export for summary + LOCATION_REMOVED=$files_removed + LOCATION_FREED=$freed_human + LOCATION_SKIPPED=$files_skipped +} + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_TRASH Transcode Cleanup ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_TRASH Transcode Cleanup — $(date '+%Y-%m-%d %H:%M:%S') ━━━" +echo "" + +START=$(date +%s) +TOTAL_REMOVED=0 +TOTAL_SKIPPED=0 + +# Cleanup ramdisk +if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then + echo "━━━ $ICON_RAM Ramdisk ━━━" + RAMDISK_BEFORE=$(df "$RAMDISK_PATH" | awk 'NR==2 {print $3}') + cleanup_location "$RAMDISK_PATH" "Ramdisk" "$TRANSCODE_MAX_AGE" + TOTAL_REMOVED=$((TOTAL_REMOVED + LOCATION_REMOVED)) + TOTAL_SKIPPED=$((TOTAL_SKIPPED + LOCATION_SKIPPED)) + RAMDISK_FREED=$LOCATION_FREED + echo "" +else + warn "$ICON_RAM Ramdisk not mounted — skipping ramdisk cleanup" + RAMDISK_FREED="0B" +fi + +# Cleanup SSD fallback +if [[ -d "$TRANSCODE_SSD" ]]; then + echo "━━━ $ICON_DISK SSD Fallback ━━━" + cleanup_location "$TRANSCODE_SSD" "SSD fallback" "$TRANSCODE_MAX_AGE" + TOTAL_REMOVED=$((TOTAL_REMOVED + LOCATION_REMOVED)) + TOTAL_SKIPPED=$((TOTAL_SKIPPED + LOCATION_SKIPPED)) + SSD_FREED=$LOCATION_FREED + echo "" +else + info "$ICON_DISK SSD fallback not found — skipping" + SSD_FREED="0B" +fi + +# ----------------------------------------------------------------------------------------------- +# Post-cleanup symlink check +# If ramdisk had space freed, check if manager should flip symlink back +# ----------------------------------------------------------------------------------------------- +if [[ "$DRY_RUN" == false ]] && mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then + RAMDISK_USED_KB=$(df "$RAMDISK_PATH" | awk 'NR==2 {print $3}') + LOW_THRESHOLD=$(awk "BEGIN {print $RAMDISK_LOW_GB * 1024 * 1024}") + + if (( RAMDISK_USED_KB < LOW_THRESHOLD )); then + CURRENT_TARGET=$(grep "^current_target=" "$STATE_FILE" 2>/dev/null | cut -d'=' -f2) + if [[ "$CURRENT_TARGET" == "$TRANSCODE_SSD" ]]; then + info "$ICON_RAM Ramdisk has space after cleanup — triggering manager to flip back" + bash "$SCRIPT_DIR/transcode_manager.sh" --no-log + fi + fi +fi + +END=$(date +%s) + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_SUMMARY Summary ━━━ +# ----------------------------------------------------------------------------------------------- +echo "━━━━━ $ICON_SUMMARY TRANSCODE CLEANUP SUMMARY ━━━━━" +echo "$ICON_RAM Ramdisk freed: $RAMDISK_FREED" +echo "$ICON_DISK SSD freed: $SSD_FREED" +echo "$ICON_TRASH Removed: $TOTAL_REMOVED files" +echo "$ICON_TRASH Skipped: $TOTAL_SKIPPED files (active or recent)" +echo "$ICON_TIME Duration: $(format_duration $((END - START)))" +if [[ "$DRY_RUN" == true ]]; then + echo "$ICON_WARN Status: DRY RUN — no files deleted" +else + echo "$ICON_DONE Status: $ICON_SUCCESS DONE" + if [[ "$TOTAL_REMOVED" -gt 0 ]]; then + notify "Transcode cleanup on $(hostname) — removed $TOTAL_REMOVED files (RAM: $RAMDISK_FREED SSD: $SSD_FREED)" "Transcode Cleanup" "normal" + fi +fi +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" \ No newline at end of file diff --git a/Transcodes/transcode_manager.sh b/Transcodes/transcode_manager.sh new file mode 100644 index 0000000..68bc9dd --- /dev/null +++ b/Transcodes/transcode_manager.sh @@ -0,0 +1,298 @@ +#!/bin/bash +# ----------------------------------------------------------------------------------------------- +# --------------------------------- Transcode Manager ------------------------------------------ +# ----------------------------------------------------------------------------------------------- +# Monitors ramdisk usage and manages the transcode symlink direction. +# New transcode sessions land wherever the symlink points — existing sessions are unaffected. +# Run every 2-3 minutes via cron/User Scripts plugin. +# +# Logic: +# Ramdisk not found → ensure symlink points to SSD, notify +# Usage < RAMDISK_LOW_GB → ensure symlink points to ramdisk +# Usage >= RAMDISK_WARN_GB → flip symlink to SSD (after checking SSD has space) +# Only notifies on STATE CHANGE — silent when nothing changes +# +# Hysteresis gap between RAMDISK_WARN_GB and RAMDISK_LOW_GB prevents flip-flop +# when usage hovers near the threshold. +# +# State tracked in /tmp/transcode_state.db — resets on reboot (correct, ramdisk also resets) +# All configuration in Master.conf under Transcode Manager section. +# ----------------------------------------------------------------------------------------------- + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "$SCRIPT_DIR/../Master.conf" +source "$SCRIPT_DIR/../common.sh" + +parse_args "$@" + +STATE_FILE="/tmp/transcode_state.db" + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_GEAR Setup ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_GEAR Setup ━━━" + +if [[ "$EUID" -ne 0 ]]; then + error "Must be run as root" + exit 1 +fi + +success "Running as root" + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_SUMMARY Status ━━━ +# ----------------------------------------------------------------------------------------------- +if [[ "$SHOW_STATUS" == true ]]; then + echo "" + echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" + echo "$ICON_RAM Ramdisk path: $RAMDISK_PATH" + echo "$ICON_RAM Warn at: ${RAMDISK_WARN_GB}GB" + echo "$ICON_RAM Back to RAM at: ${RAMDISK_LOW_GB}GB" + echo "$ICON_DISK SSD fallback: $TRANSCODE_SSD" + echo "$ICON_DISK SSD min free: ${RAMDISK_SSD_MIN_GB}GB" + echo "$ICON_LINK Symlink: $TRANSCODE_LINK" + echo "$ICON_NOTIFY Flip warn: $TRANSCODE_FLIP_WARN per hour" + echo "$ICON_GEAR Dry Run: $DRY_RUN" + + if [[ -f "$STATE_FILE" ]]; then + echo "" + echo "━━━ Current State ━━━" + cat "$STATE_FILE" + fi + echo "━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no symlink changes will be made" + +# ----------------------------------------------------------------------------------------------- +# STATE HELPERS +# ----------------------------------------------------------------------------------------------- + +get_state() { + local key="$1" + grep -E "^${key}=" "$STATE_FILE" 2>/dev/null | cut -d'=' -f2 +} + +set_state() { + local key="$1" value="$2" + if [[ -f "$STATE_FILE" ]]; then + grep -vE "^${key}=" "$STATE_FILE" > "${STATE_FILE}.tmp" + echo "${key}=${value}" >> "${STATE_FILE}.tmp" + mv "${STATE_FILE}.tmp" "$STATE_FILE" + else + echo "${key}=${value}" > "$STATE_FILE" + fi +} + +# Initialise state file if missing — ramdisk_setup.sh creates it but just in case +init_state() { + if [[ ! -f "$STATE_FILE" ]]; then + NOW=$(date +%s) + cat > "$STATE_FILE" <= 3600 )); then + FLIP_COUNT=0 + set_state "flip_hour_start" "$NOW" + fi + + ((FLIP_COUNT++)) + set_state "flip_count_hour" "$FLIP_COUNT" + set_state "last_flip_time" "$NOW" + + log "Symlink flip count this hour: $FLIP_COUNT" + + if (( FLIP_COUNT >= TRANSCODE_FLIP_WARN )); then + warn "$ICON_LINK Symlink has flipped $FLIP_COUNT times this hour — consider increasing ramdisk size" + notify "Transcode symlink flipping frequently on $(hostname) — $FLIP_COUNT flips in 1hr — consider increasing RAMDISK_SIZE" "Transcode Manager" "warning" + fi +} + +# ----------------------------------------------------------------------------------------------- +# SYMLINK MANAGEMENT +# Changes the transcode symlink target. +# Only flips if current target differs from desired target. +# Tracks flip for frequency monitoring. +# ----------------------------------------------------------------------------------------------- +set_symlink() { + local target="$1" + local current_target + current_target=$(get_state "current_target") + + if [[ "$current_target" == "$target" ]]; then + log "Symlink already points to $target — no change" + return 0 + fi + + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would flip symlink: $TRANSCODE_LINK → $target" + return 0 + fi + + info "$ICON_LINK Flipping symlink: $TRANSCODE_LINK → $target" + + if ln -sfn "$target" "$TRANSCODE_LINK"; then + success "$ICON_LINK Symlink updated → $target" + set_state "current_target" "$target" + track_flip + + if [[ "$target" == "$RAMDISK_PATH" ]]; then + notify "Transcode symlink flipped to RAMDISK on $(hostname) — ramdisk has space" "Transcode Manager" "normal" + else + notify "Transcode symlink flipped to SSD on $(hostname) — ramdisk usage high" "Transcode Manager" "warning" + fi + else + error "Failed to update symlink" + notify "Transcode symlink flip failed on $(hostname)" "Transcode Manager" "warning" + return 1 + fi +} + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_RAM Transcode Manager ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_RAM Transcode Manager — $(date '+%Y-%m-%d %H:%M:%S') ━━━" +echo "" + +init_state +START=$(date +%s) + +# ----------------------------------------------------------------------------------------------- +# Symlink integrity check +# ----------------------------------------------------------------------------------------------- +info "$ICON_LINK Checking symlink integrity..." + +if [[ ! -L "$TRANSCODE_LINK" ]]; then + error "$ICON_LINK Symlink missing: $TRANSCODE_LINK" + notify "Transcode symlink missing on $(hostname) — attempting recovery" "Transcode Manager" "warning" + + # Attempt recovery — point at whichever location exists + if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then + ln -sfn "$RAMDISK_PATH" "$TRANSCODE_LINK" && success "Symlink recovered → ramdisk" + else + ln -sfn "$TRANSCODE_SSD" "$TRANSCODE_LINK" && success "Symlink recovered → SSD" + fi +fi + +LINK_TARGET=$(readlink "$TRANSCODE_LINK" 2>/dev/null) +if [[ ! -d "$LINK_TARGET" ]]; then + error "$ICON_LINK Symlink target does not exist: $LINK_TARGET" + notify "Transcode symlink target missing on $(hostname): $LINK_TARGET" "Transcode Manager" "warning" +fi + +info "$ICON_LINK Current target: $LINK_TARGET" + +# ----------------------------------------------------------------------------------------------- +# Ramdisk mount check +# ----------------------------------------------------------------------------------------------- +info "$ICON_RAM Checking ramdisk..." + +RAMDISK_MOUNTED=true +if ! mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then + warn "$ICON_RAM Ramdisk not mounted at $RAMDISK_PATH — falling back to SSD" + RAMDISK_MOUNTED=false + set_symlink "$TRANSCODE_SSD" + notify "Ramdisk not found on $(hostname) — transcode symlink set to SSD fallback" "Transcode Manager" "warning" +fi + +if [[ "$RAMDISK_MOUNTED" == true ]]; then + # Verify ramdisk size is correct + MOUNTED_SIZE=$(df -BG "$RAMDISK_PATH" | awk 'NR==2 {print $2}' | tr -d 'G') + EXPECTED_SIZE=$(echo "$RAMDISK_SIZE" | tr -d 'G') + if [[ "$MOUNTED_SIZE" != "$EXPECTED_SIZE" ]]; then + warn "$ICON_RAM Ramdisk size mismatch — expected ${RAMDISK_SIZE} got ${MOUNTED_SIZE}G" + else + success "$ICON_RAM Ramdisk mounted and correct size (${RAMDISK_SIZE})" + fi + + # Get current usage + RAMDISK_USED_KB=$(df "$RAMDISK_PATH" | awk 'NR==2 {print $3}') + RAMDISK_AVAIL_KB=$(df "$RAMDISK_PATH" | awk 'NR==2 {print $4}') + RAMDISK_USED_GB=$(awk "BEGIN {printf \"%.1f\", $RAMDISK_USED_KB / 1024 / 1024}") + RAMDISK_AVAIL_GB=$(awk "BEGIN {printf \"%.1f\", $RAMDISK_AVAIL_KB / 1024 / 1024}") + + info "$ICON_RAM Ramdisk usage: ${RAMDISK_USED_GB}GB used / ${RAMDISK_AVAIL_GB}GB available" + + # Count sessions in ramdisk and SSD + RAMDISK_FILES=$(find "$RAMDISK_PATH" -type f 2>/dev/null | wc -l) + SSD_FILES=$(find "$TRANSCODE_SSD" -type f 2>/dev/null | wc -l) + RAMDISK_SIZE_HUMAN=$(du -sh "$RAMDISK_PATH" 2>/dev/null | cut -f1) + SSD_SIZE_HUMAN=$(du -sh "$TRANSCODE_SSD" 2>/dev/null | cut -f1) + + info "$ICON_RAM Ramdisk files: $RAMDISK_FILES ($RAMDISK_SIZE_HUMAN)" + info "$ICON_DISK SSD files: $SSD_FILES ($SSD_SIZE_HUMAN)" + + # ----------------------------------------------------------------------------------------------- + # Symlink direction decision + # ----------------------------------------------------------------------------------------------- + WARN_THRESHOLD=$(awk "BEGIN {print $RAMDISK_WARN_GB * 1024 * 1024}") + LOW_THRESHOLD=$(awk "BEGIN {print $RAMDISK_LOW_GB * 1024 * 1024}") + + if (( RAMDISK_USED_KB >= WARN_THRESHOLD )); then + # Ramdisk is getting full — check SSD has space before flipping + info "$ICON_DISK Checking SSD fallback space..." + SSD_AVAIL_KB=$(df "$TRANSCODE_SSD" | awk 'NR==2 {print $4}') + SSD_AVAIL_GB=$(awk "BEGIN {printf \"%.1f\", $SSD_AVAIL_KB / 1024 / 1024}") + SSD_MIN_KB=$(awk "BEGIN {print $RAMDISK_SSD_MIN_GB * 1024 * 1024}") + + if (( SSD_AVAIL_KB < SSD_MIN_KB )); then + error "$ICON_DISK SSD fallback only has ${SSD_AVAIL_GB}GB free — minimum is ${RAMDISK_SSD_MIN_GB}GB" + notify "Transcode SSD fallback critically low on $(hostname) — ${SSD_AVAIL_GB}GB free" "Transcode Manager" "warning" + else + info "$ICON_DISK SSD has ${SSD_AVAIL_GB}GB free — flipping to SSD" + set_symlink "$TRANSCODE_SSD" + fi + + elif (( RAMDISK_USED_KB < LOW_THRESHOLD )); then + # Ramdisk has recovered — flip back + set_symlink "$RAMDISK_PATH" + + else + # In hysteresis gap — hold current state + CURRENT=$(get_state "current_target") + log "Usage ${RAMDISK_USED_GB}GB in hysteresis gap (${RAMDISK_LOW_GB}-${RAMDISK_WARN_GB}GB) — holding current target: $CURRENT" + info "$ICON_RAM Usage in hysteresis gap — holding current symlink target" + fi +fi + +END=$(date +%s) + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_SUMMARY Summary ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━━━ $ICON_SUMMARY TRANSCODE MANAGER SUMMARY ━━━━━" +echo "$ICON_RAM Ramdisk: $([[ "$RAMDISK_MOUNTED" == true ]] && echo "${RAMDISK_USED_GB}GB used" || echo "NOT MOUNTED")" +echo "$ICON_LINK Symlink: $TRANSCODE_LINK → $(readlink "$TRANSCODE_LINK" 2>/dev/null || echo "BROKEN")" +echo "$ICON_LINK Flip count: $(get_state "flip_count_hour") this hour (warn at $TRANSCODE_FLIP_WARN)" +echo "$ICON_TIME Duration: $(format_duration $((END - START)))" +echo "$ICON_DONE Status: $ICON_SUCCESS DONE" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" \ No newline at end of file