313 lines
13 KiB
Bash
313 lines
13 KiB
Bash
#!/bin/bash
|
||
# -----------------------------------------------------------------------------------------------
|
||
# --------------------------------- Transcode Manager ------------------------------------------
|
||
# -----------------------------------------------------------------------------------------------
|
||
# Monitors ramdisk usage and manages the transcode symlink direction.
|
||
# New transcode sessions land wherever the symlink points — existing sessions are unaffected.
|
||
# Run every 2-3 minutes via cron/User Scripts plugin.
|
||
#
|
||
# Logic:
|
||
# Ramdisk not found → ensure symlink points to SSD, notify
|
||
# Usage < RAMDISK_LOW_GB → ensure symlink points to ramdisk
|
||
# Usage >= RAMDISK_WARN_GB → flip symlink to SSD (after checking SSD has space)
|
||
# Only notifies on STATE CHANGE — silent when nothing changes
|
||
#
|
||
# Hysteresis gap between RAMDISK_WARN_GB and RAMDISK_LOW_GB prevents flip-flop
|
||
# when usage hovers near the threshold.
|
||
#
|
||
# State tracked in /tmp/transcode_state.db — resets on reboot (correct, ramdisk also resets)
|
||
# All configuration in Master.conf under Transcode Manager section.
|
||
# -----------------------------------------------------------------------------------------------
|
||
|
||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||
|
||
source "$SCRIPT_DIR/../Master.conf"
|
||
source "$SCRIPT_DIR/../common.sh"
|
||
|
||
parse_args "$@"
|
||
|
||
STATE_FILE="/tmp/transcode_state.db"
|
||
|
||
# -----------------------------------------------------------------------------------------------
|
||
# ━━━ $ICON_GEAR Setup ━━━
|
||
# -----------------------------------------------------------------------------------------------
|
||
echo ""
|
||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||
|
||
if [[ "$EUID" -ne 0 ]]; then
|
||
error "Must be run as root"
|
||
exit 1
|
||
fi
|
||
|
||
success "Running as root"
|
||
|
||
# -----------------------------------------------------------------------------------------------
|
||
# ━━━ $ICON_SUMMARY Status ━━━
|
||
# -----------------------------------------------------------------------------------------------
|
||
if [[ "$SHOW_STATUS" == true ]]; then
|
||
echo ""
|
||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||
echo "$ICON_RAM Ramdisk path: $RAMDISK_PATH"
|
||
echo "$ICON_RAM Warn at: ${RAMDISK_WARN_GB}GB"
|
||
echo "$ICON_RAM Back to RAM at: ${RAMDISK_LOW_GB}GB"
|
||
echo "$ICON_DISK SSD fallback: $TRANSCODE_SSD"
|
||
echo "$ICON_DISK SSD min free: ${RAMDISK_SSD_MIN_GB}GB"
|
||
echo "$ICON_LINK Symlink: $TRANSCODE_LINK"
|
||
echo "$ICON_NOTIFY Flip warn: $TRANSCODE_FLIP_WARN per hour"
|
||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||
|
||
if [[ -f "$STATE_FILE" ]]; then
|
||
echo ""
|
||
echo "━━━ Current State ━━━"
|
||
cat "$STATE_FILE"
|
||
fi
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||
exit 0
|
||
fi
|
||
|
||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no symlink changes will be made"
|
||
|
||
# -----------------------------------------------------------------------------------------------
|
||
# STATE HELPERS
|
||
# -----------------------------------------------------------------------------------------------
|
||
|
||
get_state() {
|
||
local key="$1"
|
||
grep -E "^${key}=" "$STATE_FILE" 2>/dev/null | cut -d'=' -f2
|
||
}
|
||
|
||
set_state() {
|
||
local key="$1" value="$2"
|
||
if [[ -f "$STATE_FILE" ]]; then
|
||
grep -vE "^${key}=" "$STATE_FILE" > "${STATE_FILE}.tmp"
|
||
echo "${key}=${value}" >> "${STATE_FILE}.tmp"
|
||
mv "${STATE_FILE}.tmp" "$STATE_FILE"
|
||
else
|
||
echo "${key}=${value}" > "$STATE_FILE"
|
||
fi
|
||
}
|
||
|
||
init_state() {
|
||
if [[ ! -f "$STATE_FILE" ]]; then
|
||
local NOW
|
||
NOW=$(date +%s)
|
||
cat > "$STATE_FILE" <<EOF
|
||
current_target=$RAMDISK_PATH
|
||
last_flip_time=$NOW
|
||
flip_count_hour=0
|
||
flip_hour_start=$NOW
|
||
EOF
|
||
log "State file initialised"
|
||
fi
|
||
}
|
||
|
||
# -----------------------------------------------------------------------------------------------
|
||
# FLIP TRACKING
|
||
# -----------------------------------------------------------------------------------------------
|
||
track_flip() {
|
||
local NOW
|
||
NOW=$(date +%s)
|
||
local HOUR_START
|
||
HOUR_START=$(get_state "flip_hour_start")
|
||
local FLIP_COUNT
|
||
FLIP_COUNT=$(get_state "flip_count_hour")
|
||
[[ -z "$FLIP_COUNT" ]] && FLIP_COUNT=0
|
||
[[ -z "$HOUR_START" ]] && HOUR_START=$NOW
|
||
|
||
if (( NOW - HOUR_START >= 3600 )); then
|
||
FLIP_COUNT=0
|
||
set_state "flip_hour_start" "$NOW"
|
||
fi
|
||
|
||
((FLIP_COUNT++))
|
||
set_state "flip_count_hour" "$FLIP_COUNT"
|
||
set_state "last_flip_time" "$NOW"
|
||
|
||
log "Symlink flip count this hour: $FLIP_COUNT"
|
||
|
||
if (( FLIP_COUNT >= TRANSCODE_FLIP_WARN )); then
|
||
warn "$ICON_LINK Symlink has flipped $FLIP_COUNT times this hour — consider increasing ramdisk size"
|
||
notify "Transcode symlink flipping frequently on $(hostname) — $FLIP_COUNT flips in 1hr — consider increasing RAMDISK_SIZE" "Transcode Manager" "warning"
|
||
fi
|
||
}
|
||
|
||
# -----------------------------------------------------------------------------------------------
|
||
# SYMLINK MANAGEMENT
|
||
# Sets the symlink to the desired target.
|
||
# Only flips if current target differs — tracks flip for frequency monitoring.
|
||
# Force mode skips the state check and always sets the symlink (used for recovery).
|
||
# -----------------------------------------------------------------------------------------------
|
||
set_symlink() {
|
||
local target="$1" force="${2:-false}"
|
||
local current_target
|
||
current_target=$(get_state "current_target")
|
||
|
||
if [[ "$force" == false && "$current_target" == "$target" ]]; then
|
||
log "Symlink already points to $target — no change"
|
||
return 0
|
||
fi
|
||
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would set symlink: $TRANSCODE_LINK → $target"
|
||
return 0
|
||
fi
|
||
|
||
info "$ICON_LINK Setting symlink: $TRANSCODE_LINK → $target"
|
||
|
||
# Ensure parent directory exists
|
||
mkdir -p "$(dirname "$TRANSCODE_LINK")"
|
||
|
||
if ln -sfn "$target" "$TRANSCODE_LINK"; then
|
||
success "$ICON_LINK Symlink set → $target"
|
||
set_state "current_target" "$target"
|
||
|
||
# Only track flip and notify if this was a real direction change
|
||
if [[ "$current_target" != "$target" ]]; then
|
||
track_flip
|
||
if [[ "$target" == "$RAMDISK_PATH" ]]; then
|
||
notify "Transcode symlink flipped to RAMDISK on $(hostname) — ramdisk has space" "Transcode Manager" "normal"
|
||
else
|
||
notify "Transcode symlink flipped to SSD on $(hostname) — ramdisk usage high" "Transcode Manager" "warning"
|
||
fi
|
||
fi
|
||
else
|
||
error "Failed to set symlink: $TRANSCODE_LINK → $target"
|
||
notify "Transcode symlink failed to set on $(hostname)" "Transcode Manager" "warning"
|
||
return 1
|
||
fi
|
||
}
|
||
|
||
# -----------------------------------------------------------------------------------------------
|
||
# ━━━ $ICON_RAM Transcode Manager ━━━
|
||
# -----------------------------------------------------------------------------------------------
|
||
echo ""
|
||
echo "━━━ $ICON_RAM Transcode Manager — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||
echo ""
|
||
|
||
init_state
|
||
START=$(date +%s)
|
||
|
||
# -----------------------------------------------------------------------------------------------
|
||
# Symlink integrity check
|
||
# If symlink is missing or target doesn't exist — force correct it immediately
|
||
# -----------------------------------------------------------------------------------------------
|
||
info "$ICON_LINK Checking symlink integrity..."
|
||
|
||
SYMLINK_OK=true
|
||
|
||
if [[ ! -L "$TRANSCODE_LINK" ]]; then
|
||
error "$ICON_LINK Symlink missing: $TRANSCODE_LINK — recovering"
|
||
SYMLINK_OK=false
|
||
else
|
||
LINK_TARGET=$(readlink "$TRANSCODE_LINK" 2>/dev/null)
|
||
if [[ ! -d "$LINK_TARGET" ]]; then
|
||
error "$ICON_LINK Symlink target does not exist: $LINK_TARGET — recovering"
|
||
SYMLINK_OK=false
|
||
else
|
||
info "$ICON_LINK Symlink valid: $TRANSCODE_LINK → $LINK_TARGET"
|
||
fi
|
||
fi
|
||
|
||
# Force recovery — point at ramdisk if mounted, SSD otherwise
|
||
if [[ "$SYMLINK_OK" == false ]]; then
|
||
if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
|
||
warn "Recovering symlink → ramdisk"
|
||
set_symlink "$RAMDISK_PATH" force
|
||
else
|
||
warn "Ramdisk not mounted — recovering symlink → SSD"
|
||
set_symlink "$TRANSCODE_SSD" force
|
||
fi
|
||
fi
|
||
|
||
# -----------------------------------------------------------------------------------------------
|
||
# Ramdisk mount check
|
||
# -----------------------------------------------------------------------------------------------
|
||
info "$ICON_RAM Checking ramdisk..."
|
||
|
||
RAMDISK_MOUNTED=true
|
||
if ! mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
|
||
warn "$ICON_RAM Ramdisk not mounted at $RAMDISK_PATH — falling back to SSD"
|
||
RAMDISK_MOUNTED=false
|
||
set_symlink "$TRANSCODE_SSD"
|
||
notify "Ramdisk not found on $(hostname) — transcode symlink set to SSD fallback" "Transcode Manager" "warning"
|
||
fi
|
||
|
||
if [[ "$RAMDISK_MOUNTED" == true ]]; then
|
||
# Verify ramdisk size matches expected
|
||
MOUNTED_SIZE=$(df -BG "$RAMDISK_PATH" | awk 'NR==2 {print $2}' | tr -d 'G')
|
||
EXPECTED_SIZE=$(echo "$RAMDISK_SIZE" | tr -d 'G')
|
||
if [[ "$MOUNTED_SIZE" != "$EXPECTED_SIZE" ]]; then
|
||
warn "$ICON_RAM Ramdisk size mismatch — expected ${RAMDISK_SIZE} got ${MOUNTED_SIZE}G"
|
||
else
|
||
success "$ICON_RAM Ramdisk mounted and correct size (${RAMDISK_SIZE})"
|
||
fi
|
||
|
||
# Get current usage in KB — use raw bytes to avoid scientific notation issues
|
||
RAMDISK_USED_KB=$(df "$RAMDISK_PATH" --output=used | tail -1 | tr -d ' ')
|
||
RAMDISK_AVAIL_KB=$(df "$RAMDISK_PATH" --output=avail | tail -1 | tr -d ' ')
|
||
|
||
# Convert to GB for display using awk — avoids bash integer/float issues
|
||
RAMDISK_USED_GB=$(awk "BEGIN {printf \"%.2f\", $RAMDISK_USED_KB / 1048576}")
|
||
RAMDISK_AVAIL_GB=$(awk "BEGIN {printf \"%.2f\", $RAMDISK_AVAIL_KB / 1048576}")
|
||
|
||
info "$ICON_RAM Ramdisk usage: ${RAMDISK_USED_GB}GB used / ${RAMDISK_AVAIL_GB}GB available"
|
||
|
||
# File counts and sizes
|
||
RAMDISK_FILES=$(find "$RAMDISK_PATH" -type f 2>/dev/null | wc -l)
|
||
SSD_FILES=$(find "$TRANSCODE_SSD" -type f 2>/dev/null | wc -l)
|
||
RAMDISK_SIZE_HUMAN=$(du -sh "$RAMDISK_PATH" 2>/dev/null | cut -f1)
|
||
SSD_SIZE_HUMAN=$(du -sh "$TRANSCODE_SSD" 2>/dev/null | cut -f1)
|
||
|
||
info "$ICON_RAM Ramdisk files: $RAMDISK_FILES ($RAMDISK_SIZE_HUMAN)"
|
||
info "$ICON_DISK SSD files: $SSD_FILES ($SSD_SIZE_HUMAN)"
|
||
|
||
# -----------------------------------------------------------------------------------------------
|
||
# Symlink direction decision — all comparisons via awk to handle float thresholds
|
||
# -----------------------------------------------------------------------------------------------
|
||
WARN_EXCEEDED=$(awk "BEGIN {print ($RAMDISK_USED_GB >= $RAMDISK_WARN_GB) ? 1 : 0}")
|
||
LOW_RECOVERED=$(awk "BEGIN {print ($RAMDISK_USED_GB < $RAMDISK_LOW_GB) ? 1 : 0}")
|
||
|
||
if [[ "$WARN_EXCEEDED" == "1" ]]; then
|
||
# Ramdisk is getting full — check SSD has enough space before flipping
|
||
info "$ICON_DISK Checking SSD fallback space..."
|
||
SSD_AVAIL_KB=$(df "$TRANSCODE_SSD" --output=avail | tail -1 | tr -d ' ')
|
||
SSD_AVAIL_GB=$(awk "BEGIN {printf \"%.2f\", $SSD_AVAIL_KB / 1048576}")
|
||
SSD_SUFFICIENT=$(awk "BEGIN {print ($SSD_AVAIL_GB >= $RAMDISK_SSD_MIN_GB) ? 1 : 0}")
|
||
|
||
if [[ "$SSD_SUFFICIENT" == "0" ]]; then
|
||
error "$ICON_DISK SSD fallback only has ${SSD_AVAIL_GB}GB free — minimum is ${RAMDISK_SSD_MIN_GB}GB"
|
||
notify "Transcode SSD fallback critically low on $(hostname) — ${SSD_AVAIL_GB}GB free" "Transcode Manager" "warning"
|
||
else
|
||
info "$ICON_DISK SSD has ${SSD_AVAIL_GB}GB free — flipping to SSD"
|
||
set_symlink "$TRANSCODE_SSD"
|
||
fi
|
||
|
||
elif [[ "$LOW_RECOVERED" == "1" ]]; then
|
||
# Ramdisk has recovered — flip back
|
||
set_symlink "$RAMDISK_PATH"
|
||
|
||
else
|
||
# In hysteresis gap — hold current state
|
||
CURRENT=$(get_state "current_target")
|
||
log "Usage ${RAMDISK_USED_GB}GB in hysteresis gap (${RAMDISK_LOW_GB}–${RAMDISK_WARN_GB}GB) — holding: $CURRENT"
|
||
info "$ICON_RAM Usage in hysteresis gap — holding current symlink"
|
||
fi
|
||
fi
|
||
|
||
END=$(date +%s)
|
||
|
||
# -----------------------------------------------------------------------------------------------
|
||
# ━━━ $ICON_SUMMARY Summary ━━━
|
||
# -----------------------------------------------------------------------------------------------
|
||
echo ""
|
||
echo "━━━━━ $ICON_SUMMARY TRANSCODE MANAGER SUMMARY ━━━━━"
|
||
if [[ "$RAMDISK_MOUNTED" == true ]]; then
|
||
echo "$ICON_RAM Ramdisk: ${RAMDISK_USED_GB}GB used / ${RAMDISK_AVAIL_GB}GB available"
|
||
else
|
||
echo "$ICON_RAM Ramdisk: NOT MOUNTED — using SSD"
|
||
fi
|
||
echo "$ICON_LINK Symlink: $TRANSCODE_LINK → $(readlink "$TRANSCODE_LINK" 2>/dev/null || echo "BROKEN")"
|
||
echo "$ICON_LINK Flips: $(get_state "flip_count_hour") this hour (warn at $TRANSCODE_FLIP_WARN)"
|
||
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
||
echo "$ICON_DONE Status: $ICON_SUCCESS DONE"
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" |