audit echo vs log across all scripts — outcomes always visible, verbose for per-item loops
This commit is contained in:
Gmer4Lfe
+331
@@ -0,0 +1,331 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# =========================== Intermediate Sync Maintenance ====================================
|
||||
# ==============================================================================================
|
||||
# 4-hour orchestrator — arr library reconciliation, artwork fetching, and optional rsync.
|
||||
# Schedule: 0 */4 * * * (every 4 hours)
|
||||
#
|
||||
# ── EXECUTION ORDER ───────────────────────────────────────────────────────────────────────────
|
||||
# 1. conf_sync.sh --pull-only — refresh partner conf cache in RAM (/tmp/.vv/)
|
||||
# 2. arr_sync.sh — sync Lidarr/Sonarr/Radarr libraries across all nodes
|
||||
# 3. Rsync window (optional) — INTERMEDIATE_SYNC_SHARES, if any configured
|
||||
# 4. INTERMEDIATE_MAINTENANCE_SCRIPTS — artwork fetch and any future 4-hour jobs
|
||||
#
|
||||
# ── WHY A SEPARATE ORCHESTRATOR ───────────────────────────────────────────────────────────────
|
||||
# arr libraries need to converge more frequently than once a day. If a remote node adds
|
||||
# something at 2am, the next daily window is 23 hours away — remote arrs search for content
|
||||
# they don't know is already owned. Running every 4 hours closes that gap.
|
||||
#
|
||||
# lidarr_missing_art.sh is idempotent — skips existing files, runs fast after initial fill.
|
||||
# Pairing it here means artwork catches up within 4 hours of a new album landing.
|
||||
#
|
||||
# Rsync is optional — INTERMEDIATE_SYNC_SHARES empty by default. Add shares to the config
|
||||
# if a subset of data needs mid-day propagation (e.g. watch state, metadata). Full media
|
||||
# share sync stays in the daily window.
|
||||
#
|
||||
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
|
||||
# detect_hosts() sets MY_ID and aliases HOST*_INTERMEDIATE_SYNC_SHARES → INTERMEDIATE_SYNC_SHARES.
|
||||
# Each server can have a different set of mid-day shares — configure in host*.conf.
|
||||
# Each script in INTERMEDIATE_MAINTENANCE_SCRIPTS handles its own host logic.
|
||||
#
|
||||
# ── DRIVE TEMP HANDLING ───────────────────────────────────────────────────────────────────────
|
||||
# Same as daily_sync_maintenance.sh:
|
||||
# exit 1 = temp WARN — skip this share, continue to next
|
||||
# exit 2 = temp CRITICAL — abort ALL remaining syncs in this window
|
||||
#
|
||||
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
|
||||
# Root check — scripts called here require root
|
||||
# acquire_lock — prevents concurrent intermediate windows
|
||||
# check_connectivity — verified before any rsync (skipped if no shares)
|
||||
# check_remote_rootfs — aborts rsync if remote rootfs nearly full
|
||||
# Non-fatal jobs — a failed arr_sync warns but does not block rsync or artwork fetch
|
||||
# Silent on success — runs 4x/day, only failures warrant notification
|
||||
#
|
||||
# ── CONFIGURATION ─────────────────────────────────────────────────────────────────────────────
|
||||
# host*.conf: HOST*_INTERMEDIATE_SYNC_SHARES — shares synced mid-day (empty = rsync skipped)
|
||||
# master.conf: INTERMEDIATE_RSYNC_ENABLED — enable/disable rsync section (default: true)
|
||||
# master.conf: INTERMEDIATE_MAINTENANCE_SCRIPTS — jobs run after rsync
|
||||
# master.conf: ARR_SYNC_ENABLED — toggle inside arr_sync.sh
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# intermediate_sync_maintenance.sh — normal run
|
||||
# intermediate_sync_maintenance.sh --dry-run — preview without changes
|
||||
# intermediate_sync_maintenance.sh --log — verbose per-job output
|
||||
# intermediate_sync_maintenance.sh --status — show configured shares/jobs and exit
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
RSYNC_SCRIPT="$SCRIPT_DIR/../Rsync/rsync.sh"
|
||||
SCRIPTS_ROOT="$SCRIPT_DIR/.."
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
acquire_lock
|
||||
|
||||
detect_hosts
|
||||
resolve_remote_ip
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
|
||||
# ── Helper — run a maintenance job, track pass/fail ───────────────────────────────────────────
|
||||
run_job() {
|
||||
local script_entry="$1"
|
||||
local extra_dry=""
|
||||
[[ "$DRY_RUN" == true ]] && extra_dry="--dry-run"
|
||||
|
||||
read -r -a script_args <<< "$script_entry"
|
||||
local script_path="$SCRIPTS_ROOT/${script_args[0]}"
|
||||
local script_name
|
||||
script_name=$(basename "${script_args[0]}")
|
||||
local extra_args=("${script_args[@]:1}")
|
||||
|
||||
if [[ ! -f "$script_path" ]]; then
|
||||
error "$script_name — not found at $script_path"
|
||||
JOB_FAIL+=("$script_name")
|
||||
return 1
|
||||
fi
|
||||
|
||||
log "Running: $script_name ${extra_args[*]}"
|
||||
# shellcheck disable=SC2086
|
||||
if bash "$script_path" "${extra_args[@]}" $extra_dry; then
|
||||
log "$script_name — done ✅"
|
||||
JOB_PASS+=("$script_name ${extra_args[*]}")
|
||||
else
|
||||
warn "$script_name — failed (exit $?)"
|
||||
JOB_FAIL+=("$script_name ${extra_args[*]}")
|
||||
fi
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY INTERMEDIATE SYNC STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_NET Remote: $REMOTE_ID ($REMOTE_SERVER_NAME)"
|
||||
echo "$ICON_SYNC Rsync enabled: ${RSYNC_ENABLED:-true}"
|
||||
echo "$ICON_SYNC Interm. enabled: ${INTERMEDIATE_RSYNC_ENABLED:-true}"
|
||||
echo "$ICON_GEAR Arr sync: ${ARR_SYNC_ENABLED:-true}"
|
||||
echo ""
|
||||
echo "━━━ Intermediate Sync Shares ━━━"
|
||||
if [[ ${#INTERMEDIATE_SYNC_SHARES[@]} -eq 0 ]]; then
|
||||
echo " None configured — add to INTERMEDIATE_SYNC_SHARES in master.conf to enable"
|
||||
else
|
||||
for share in "${INTERMEDIATE_SYNC_SHARES[@]}"; do
|
||||
[[ -n "$share" ]] && echo " $ICON_SYNC $(basename "$share") ($share)"
|
||||
done
|
||||
fi
|
||||
echo ""
|
||||
echo "━━━ Intermediate Maintenance Scripts ━━━"
|
||||
if [[ ${#INTERMEDIATE_MAINTENANCE_SCRIPTS[@]} -eq 0 ]]; then
|
||||
echo " None configured"
|
||||
else
|
||||
for entry in "${INTERMEDIATE_MAINTENANCE_SCRIPTS[@]}"; do
|
||||
[[ -n "$entry" ]] && echo " $ICON_GEAR ${entry##*/}"
|
||||
done
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
WINDOW_START=$(date +%s)
|
||||
JOB_PASS=()
|
||||
JOB_FAIL=()
|
||||
PASS=()
|
||||
FAIL=()
|
||||
SHARE_TIMES=()
|
||||
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Intermediate Sync — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Conf Pull ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Conf Pull ━━━"
|
||||
|
||||
CONF_SYNC_SCRIPT="$SCRIPTS_ROOT/System_Essentials/conf_sync.sh"
|
||||
if [[ ! -f "$CONF_SYNC_SCRIPT" ]]; then
|
||||
warn "conf_sync.sh not found — skipping partner conf refresh"
|
||||
else
|
||||
_conf_args=("--pull-only")
|
||||
[[ "$DRY_RUN" == true ]] && _conf_args+=("--dry-run")
|
||||
if bash "$CONF_SYNC_SCRIPT" "${_conf_args[@]}"; then
|
||||
log "Partner conf cache refreshed ✅"
|
||||
JOB_PASS+=("conf_sync.sh --pull-only")
|
||||
else
|
||||
warn "Partner conf pull failed — cache may be stale"
|
||||
JOB_FAIL+=("conf_sync.sh --pull-only")
|
||||
fi
|
||||
unset _conf_args
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Arr Sync ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Arr Sync ━━━"
|
||||
|
||||
ARR_SYNC_SCRIPT="$SCRIPTS_ROOT/Media/arr_sync.sh"
|
||||
if [[ "${ARR_SYNC_ENABLED:-true}" != "true" ]]; then
|
||||
echo "ARR_SYNC_ENABLED=false — skipping"
|
||||
elif [[ ! -f "$ARR_SYNC_SCRIPT" ]]; then
|
||||
warn "arr_sync.sh not found at $ARR_SYNC_SCRIPT — skipping"
|
||||
JOB_FAIL+=("arr_sync.sh")
|
||||
else
|
||||
_arr_sync_args=()
|
||||
[[ "$DRY_RUN" == true ]] && _arr_sync_args+=("--dry-run")
|
||||
if bash "$ARR_SYNC_SCRIPT" "${_arr_sync_args[@]}"; then
|
||||
echo "Arr sync complete ✅"
|
||||
JOB_PASS+=("arr_sync.sh")
|
||||
else
|
||||
warn "Arr sync completed with errors — continuing"
|
||||
JOB_FAIL+=("arr_sync.sh")
|
||||
fi
|
||||
unset _arr_sync_args
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Rsync (optional) ━━━
|
||||
# ==============================================================================================
|
||||
SHARE_COUNT=${#INTERMEDIATE_SYNC_SHARES[@]}
|
||||
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Mid-day Share Sync — $SHARE_COUNT share(s) ━━━"
|
||||
|
||||
TOTAL_START=$(date +%s)
|
||||
SHARE_INDEX=0
|
||||
ABORT_ALL_SYNCS=false
|
||||
|
||||
if [[ "$SHARE_COUNT" -eq 0 ]]; then
|
||||
echo "No INTERMEDIATE_SYNC_SHARES configured — skipping"
|
||||
echo "Add shares to INTERMEDIATE_SYNC_SHARES in master.conf to enable mid-day sync"
|
||||
elif ! check_rsync_enabled "INTERMEDIATE"; then
|
||||
warn "Intermediate rsync disabled — skipping all $SHARE_COUNT share sync(s)"
|
||||
else
|
||||
check_connectivity
|
||||
check_remote_rootfs
|
||||
|
||||
RSYNC_DRY=""
|
||||
[[ "$DRY_RUN" == true ]] && RSYNC_DRY="--dry-run"
|
||||
|
||||
for SHARE in "${INTERMEDIATE_SYNC_SHARES[@]}"; do
|
||||
(( SHARE_INDEX++ ))
|
||||
SHARE_NAME=$(basename "$SHARE")
|
||||
SHARE_START=$(date +%s)
|
||||
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Share $SHARE_INDEX/$SHARE_COUNT: $SHARE_NAME ━━━"
|
||||
|
||||
if [[ "$ABORT_ALL_SYNCS" == true ]]; then
|
||||
warn "$SHARE_NAME — skipped (drive temps CRITICAL earlier in window)"
|
||||
FAIL+=("$SHARE_NAME:temp-critical")
|
||||
continue
|
||||
fi
|
||||
|
||||
bash "$RSYNC_SCRIPT" "$SHARE" $RSYNC_DRY
|
||||
RSYNC_EXIT=$?
|
||||
|
||||
SHARE_TIMES+=("$SHARE_NAME:$(( $(date +%s) - SHARE_START ))")
|
||||
|
||||
case "$RSYNC_EXIT" in
|
||||
0)
|
||||
PASS+=("$SHARE_NAME")
|
||||
log "$SHARE_NAME — done ✅"
|
||||
;;
|
||||
1)
|
||||
FAIL+=("$SHARE_NAME:temp-warn")
|
||||
warn "$SHARE_NAME skipped — drive temps too high"
|
||||
;;
|
||||
2)
|
||||
FAIL+=("$SHARE_NAME:temp-critical")
|
||||
ABORT_ALL_SYNCS=true
|
||||
error "$SHARE_NAME aborted — drive temps CRITICAL, stopping all remaining syncs"
|
||||
notify "Intermediate sync aborted on $(hostname) ($MY_ID) — drive temps CRITICAL during $SHARE_NAME" \
|
||||
"Intermediate Sync" "warning"
|
||||
;;
|
||||
*)
|
||||
FAIL+=("$SHARE_NAME")
|
||||
error "$SHARE_NAME failed (exit $RSYNC_EXIT) — continuing to next share"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
fi
|
||||
|
||||
TOTAL_END=$(date +%s)
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Maintenance Jobs ━━━
|
||||
# ==============================================================================================
|
||||
if [[ ${#INTERMEDIATE_MAINTENANCE_SCRIPTS[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Maintenance Jobs ━━━"
|
||||
for script_entry in "${INTERMEDIATE_MAINTENANCE_SCRIPTS[@]}"; do
|
||||
[[ -z "$script_entry" ]] && continue
|
||||
echo ""
|
||||
run_job "$script_entry"
|
||||
done
|
||||
fi
|
||||
|
||||
WINDOW_END=$(date +%s)
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY INTERMEDIATE SYNC SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_TIME Window: $(date -d @"$WINDOW_START" '+%Y-%m-%d %H:%M:%S') → $(date -d @"$WINDOW_END" '+%H:%M:%S')"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( WINDOW_END - WINDOW_START )))"
|
||||
echo ""
|
||||
|
||||
if [[ "$SHARE_COUNT" -gt 0 ]]; then
|
||||
echo "$ICON_SYNC Shares ($SHARE_COUNT):"
|
||||
for entry in "${SHARE_TIMES[@]}"; do
|
||||
sname="${entry%%:*}"
|
||||
sdur="${entry##*:}"
|
||||
if printf '%s\n' "${FAIL[@]}" | grep -q "^${sname}"; then
|
||||
echo " $ICON_ERROR $sname — $(format_duration "$sdur")"
|
||||
else
|
||||
echo " $ICON_DONE $sname — $(format_duration "$sdur")"
|
||||
fi
|
||||
done
|
||||
echo " Passed: ${#PASS[@]}/$SHARE_COUNT Failed: ${#FAIL[@]}/$SHARE_COUNT"
|
||||
echo ""
|
||||
fi
|
||||
|
||||
if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then
|
||||
echo "$ICON_GEAR Jobs:"
|
||||
for job in "${JOB_PASS[@]}"; do echo " $ICON_DONE $job"; done
|
||||
for job in "${JOB_FAIL[@]}"; do echo " $ICON_ERROR $job"; done
|
||||
echo ""
|
||||
fi
|
||||
|
||||
TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} ))
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no changes made"
|
||||
elif [[ "$TOTAL_FAIL" -eq 0 ]]; then
|
||||
echo "$ICON_DONE Status: all complete ✅ — ${#JOB_PASS[@]} job(s) run, ${#PASS[@]}/$SHARE_COUNT share(s) synced"
|
||||
else
|
||||
warn "Status: $TOTAL_FAIL failure(s)"
|
||||
notify "Intermediate sync failed on $(hostname) ($MY_ID) — shares: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \
|
||||
"Intermediate Sync" "warning"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ "$TOTAL_FAIL" -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
+1470
File diff suppressed because it is too large
Load Diff
+653
@@ -0,0 +1,653 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ================================= Transcode Manager ==========================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Monitors ramdisk usage and manages the transcode symlink direction. Called by
|
||||
# transcode_management.sh (Orchestrators/) every 7 minutes — always after
|
||||
# transcode_cleanup.sh runs first. Must be fast, non-blocking, and silent when
|
||||
# nothing has changed.
|
||||
#
|
||||
# Emby's transcode path points at TRANSCODE_LINK (a symlink). ffmpeg resolves
|
||||
# the symlink once at session start and holds a direct reference — existing
|
||||
# sessions are completely unaffected by symlink flips. Only new sessions care
|
||||
# where the symlink currently points.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Three Modes (TRANSCODE_MANAGER_MODE)
|
||||
# smart — auto-flips between ramdisk and SSD based on usage thresholds (default)
|
||||
# ramdisk above RAMDISK_WARN_GB → flip to SSD
|
||||
# ramdisk below RAMDISK_LOW_GB → flip back to ramdisk
|
||||
# ramdisk — always uses ramdisk, warns if above threshold, never flips
|
||||
# ssd — always uses SSD, never uses ramdisk (use during drain or maintenance)
|
||||
#
|
||||
# Safety Checks — Every Run Regardless of Mode
|
||||
# Symlink missing/broken → recreate pointing at ramdisk, notify
|
||||
# Ramdisk disappeared → flip to SSD immediately, notify warning
|
||||
# SSD path missing → disable SSD fallback (error if mode=ssd)
|
||||
# transcoding-temp missing → recreate on ramdisk silently
|
||||
# Permissions drift → fix silently every run
|
||||
# Emby not running → skip threshold checks, verify symlink only
|
||||
#
|
||||
# Session Display
|
||||
# Shows active streams from all configured TRANSCODE_SERVERS with user, title,
|
||||
# type (Live TV / TV Show / Movie), and play method (Transcode / Direct).
|
||||
# Split state shown when sessions exist on both ramdisk and SSD — normal during
|
||||
# a flip while ramdisk sessions drain.
|
||||
#
|
||||
# Daily Log
|
||||
# Appends one entry per run to TRANSCODE_DAILY_LOG, read by weekly_health_digest.sh.
|
||||
# Format: DATE|RAMDISK_USED_GB|FLIP_COUNT|RAM_SESSION_COUNT|SSD_SESSION_COUNT|FILES_CLEANED
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Wait Lock
|
||||
# acquire_lock "wait" — waits if the previous run is still active. The 7-minute
|
||||
# interval can overlap on a system under heavy load.
|
||||
#
|
||||
# Docker Timeout
|
||||
# DOCKER_TIMEOUT caps all docker calls against a hung daemon.
|
||||
#
|
||||
# Notification Validated
|
||||
# platform_require_cmd confirms the notify script is present before use.
|
||||
#
|
||||
# Silent by Default
|
||||
# Runs every 7 minutes — only speaks when something changes or needs attention.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# host*.conf
|
||||
#
|
||||
# HOST*_RAMDISK_PATH / HOST*_TRANSCODE_SSD
|
||||
# Ramdisk mount point and SSD fallback path.
|
||||
# Aliased by detect_hosts().
|
||||
#
|
||||
# HOST*_RAMDISK_WARN_GB / HOST*_RAMDISK_LOW_GB / HOST*_RAMDISK_SIZE
|
||||
# Thresholds and ceiling. Change all three together.
|
||||
# Aliased by detect_hosts().
|
||||
#
|
||||
# HOST*_TRANSCODE_SERVERS
|
||||
# Array of media server definitions: "ContainerName|URL|APIKey|Type"
|
||||
# Type: emby | jellyfin | plex
|
||||
# Aliased by detect_hosts() → TRANSCODE_SERVERS.
|
||||
#
|
||||
# master.conf
|
||||
#
|
||||
# TRANSCODE_MANAGER_MODE
|
||||
# smart | ramdisk | ssd. (default: smart)
|
||||
#
|
||||
# TRANSCODE_CHECK_EMBY
|
||||
# Skip threshold checks when Emby not running — prevents unnecessary flips
|
||||
# overnight when no sessions are active. (default: true)
|
||||
#
|
||||
# TRANSCODE_FLIP_WARN
|
||||
# Notify if symlink flips this many times in one hour — indicates ramdisk
|
||||
# is undersized for the load. (default: 3)
|
||||
#
|
||||
# ==============================================================================================
|
||||
# STATE FILES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# /tmp/transcode_state.db — current symlink target, flip count, last flip time
|
||||
# Lives in /tmp (ephemeral — resets correctly on reboot)
|
||||
# TRANSCODE_DAILY_LOG — per-run append, read by weekly_health_digest.sh
|
||||
# Trimmed to TRANSCODE_LOG_RETENTION days on each write
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# transcode_manager.sh
|
||||
# Check usage, flip if needed, run safety checks, display active sessions.
|
||||
#
|
||||
# transcode_manager.sh --dry-run
|
||||
# Show current usage and what flip decision would be made. No changes.
|
||||
#
|
||||
# transcode_manager.sh --status
|
||||
# Show current symlink target, ramdisk usage, session counts, and flip history.
|
||||
#
|
||||
# transcode_manager.sh --log
|
||||
# Verbose output including per-check results and session detail.
|
||||
#
|
||||
# transcode_manager.sh --no-log
|
||||
# Suppress daily log write. Used internally when called by transcode_cleanup.sh.
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
# Handle --no-log flag before parse_args
|
||||
NO_LOG=false
|
||||
FILTERED_ARGS=()
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--no-log) NO_LOG=true ;;
|
||||
*) FILTERED_ARGS+=("$arg") ;;
|
||||
esac
|
||||
done
|
||||
|
||||
parse_args "${FILTERED_ARGS[@]}"
|
||||
|
||||
DOCKER_TIMEOUT=15
|
||||
STATE_FILE="${TRANSCODE_STATE_FILE:-${STATE_DIR:-/tmp}/transcode_state.db}"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
acquire_lock "wait"
|
||||
|
||||
if ! command -v docker &>/dev/null; then
|
||||
error "Docker command not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# detect_hosts() sets MY_ID and aliases all HOST*_TRANSCODE_* vars
|
||||
detect_hosts
|
||||
|
||||
# Primary container — first entry in TRANSCODE_SERVERS
|
||||
TRANSCODE_EMBY_CONTAINER="Emby"
|
||||
if [[ "${#TRANSCODE_SERVERS[@]}" -gt 0 ]]; then
|
||||
IFS='|' read -r PRIMARY_CONTAINER _ _ _ <<< "${TRANSCODE_SERVERS[0]}"
|
||||
TRANSCODE_EMBY_CONTAINER="${PRIMARY_CONTAINER:-Emby}"
|
||||
fi
|
||||
|
||||
case "$TRANSCODE_MANAGER_MODE" in
|
||||
smart|ramdisk|ssd) log "Mode: $TRANSCODE_MANAGER_MODE" ;;
|
||||
*)
|
||||
warn "Unknown TRANSCODE_MANAGER_MODE: $TRANSCODE_MANAGER_MODE — defaulting to smart"
|
||||
TRANSCODE_MANAGER_MODE="smart"
|
||||
;;
|
||||
esac
|
||||
|
||||
log "$ICON_GEAR Config: ramdisk=$RAMDISK_PATH size=$RAMDISK_SIZE warn-at=${RAMDISK_WARN_GB}GB flip-back-at=${RAMDISK_LOW_GB}GB ssd-min-free=${RAMDISK_SSD_MIN_GB}GB flip-warn=${TRANSCODE_FLIP_WARN}/hr"
|
||||
log "$ICON_DISK SSD: $TRANSCODE_SSD"
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY TRANSCODE MANAGER STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_GEAR Mode: $TRANSCODE_MANAGER_MODE"
|
||||
echo "$ICON_RAM Ramdisk: $RAMDISK_PATH ($RAMDISK_SIZE)"
|
||||
echo "$ICON_LINK Symlink: $TRANSCODE_LINK"
|
||||
echo "$ICON_DISK SSD: $TRANSCODE_SSD"
|
||||
echo "$ICON_RAM Warn at: ${RAMDISK_WARN_GB}GB"
|
||||
echo "$ICON_RAM Low at: ${RAMDISK_LOW_GB}GB"
|
||||
echo "$ICON_DISK SSD min free: ${RAMDISK_SSD_MIN_GB}GB"
|
||||
echo "$ICON_RAM Flip warn: $TRANSCODE_FLIP_WARN per hour"
|
||||
echo "$ICON_GEAR Check Emby: $TRANSCODE_CHECK_EMBY ($TRANSCODE_EMBY_CONTAINER)"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
CURRENT_TARGET=$(readlink "$TRANSCODE_LINK" 2>/dev/null || echo "missing")
|
||||
echo "$ICON_LINK Symlink now: $TRANSCODE_LINK → $CURRENT_TARGET"
|
||||
if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
|
||||
USED_GB=$(df "$RAMDISK_PATH" --output=used 2>/dev/null | \
|
||||
tail -1 | tr -d ' ' | awk '{printf "%.2f", $1/1048576}')
|
||||
echo "$ICON_RAM Ramdisk now: ${USED_GB}GB used"
|
||||
else
|
||||
echo "$ICON_RAM Ramdisk: NOT MOUNTED"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ── HELPER FUNCTIONS ──────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
flip_symlink() {
|
||||
local target="$1" reason="$2"
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would flip symlink to: $target ($reason)"
|
||||
return 0
|
||||
fi
|
||||
ln -sfn "$target" "$TRANSCODE_LINK"
|
||||
warn "$ICON_LINK Symlink flipped to: $target ($reason)"
|
||||
}
|
||||
|
||||
fix_permissions() {
|
||||
local path="$1"
|
||||
[[ ! -d "$path" ]] && return
|
||||
chown -R "$TRANSCODE_OWNER" "$path" 2>/dev/null
|
||||
chmod -R "$TRANSCODE_CHMOD" "$path" 2>/dev/null
|
||||
log "Permissions fixed on $path"
|
||||
}
|
||||
|
||||
get_ramdisk_used_gb() {
|
||||
df "$RAMDISK_PATH" --output=used 2>/dev/null | \
|
||||
tail -1 | tr -d ' ' | awk '{printf "%.2f", $1/1048576}'
|
||||
}
|
||||
|
||||
get_ramdisk_avail_gb() {
|
||||
df "$RAMDISK_PATH" --output=avail 2>/dev/null | \
|
||||
tail -1 | tr -d ' ' | awk '{printf "%.2f", $1/1048576}'
|
||||
}
|
||||
|
||||
get_ssd_free_gb() {
|
||||
df "$TRANSCODE_SSD" --output=avail 2>/dev/null | \
|
||||
tail -1 | tr -d ' ' | awk '{printf "%.2f", $1/1048576}'
|
||||
}
|
||||
|
||||
get_flip_count() {
|
||||
local state_file="${STATE_DIR:-/tmp}/transcode_flip_state.db"
|
||||
local current_hour
|
||||
current_hour=$(date '+%Y-%m-%d-%H')
|
||||
[[ ! -f "$state_file" ]] && echo "0" && return
|
||||
local stored_hour stored_count
|
||||
stored_hour=$(awk -F'|' 'NR==1{print $1}' "$state_file" 2>/dev/null)
|
||||
stored_count=$(awk -F'|' 'NR==1{print $2}' "$state_file" 2>/dev/null)
|
||||
[[ "$stored_hour" == "$current_hour" ]] && echo "${stored_count:-0}" || echo "0"
|
||||
}
|
||||
|
||||
increment_flip_count() {
|
||||
local state_file="${STATE_DIR:-/tmp}/transcode_flip_state.db"
|
||||
local current_hour
|
||||
current_hour=$(date '+%Y-%m-%d-%H')
|
||||
local current_count
|
||||
current_count=$(get_flip_count)
|
||||
current_count=$(( current_count + 1 ))
|
||||
echo "${current_hour}|${current_count}" > "$state_file"
|
||||
echo "$current_count"
|
||||
}
|
||||
|
||||
get_media_type_label() {
|
||||
case "$1" in
|
||||
LiveTv|TvChannel) echo "Live TV" ;;
|
||||
Episode) echo "TV Show" ;;
|
||||
Movie) echo "Movie" ;;
|
||||
Audio) echo "Music" ;;
|
||||
MusicVideo) echo "Music Video" ;;
|
||||
*) echo "$1" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
get_play_method_label() {
|
||||
case "$1" in
|
||||
Transcode) echo "Transcode" ;;
|
||||
DirectStream) echo "Direct Stream" ;;
|
||||
DirectPlay) echo "Direct Play" ;;
|
||||
*) echo "$1" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Transcode Manager ━━━
|
||||
# ==============================================================================================
|
||||
START=$(date +%s)
|
||||
RAMDISK_HEALTHY=true
|
||||
SSD_HEALTHY=true
|
||||
EMBY_RUNNING=true
|
||||
SOMETHING_HAPPENED=false # controls whether summary is printed
|
||||
|
||||
# ── Check 1 — Emby running ───────────────────────────────────────────────────────────────────
|
||||
if [[ "$TRANSCODE_CHECK_EMBY" == true ]]; then
|
||||
log "Checking $TRANSCODE_EMBY_CONTAINER..."
|
||||
if ! timeout "$DOCKER_TIMEOUT" docker inspect "$TRANSCODE_EMBY_CONTAINER" \
|
||||
--format '{{.State.Running}}' 2>/dev/null | grep -q "true"; then
|
||||
warn "$TRANSCODE_EMBY_CONTAINER is not running — skipping threshold checks"
|
||||
EMBY_RUNNING=false
|
||||
SOMETHING_HAPPENED=true
|
||||
else
|
||||
log "$TRANSCODE_EMBY_CONTAINER is running ✅"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Check 2 — Symlink integrity ───────────────────────────────────────────────────────────────
|
||||
log "Checking symlink integrity..."
|
||||
CURRENT_TARGET=$(readlink "$TRANSCODE_LINK" 2>/dev/null)
|
||||
|
||||
if [[ -z "$CURRENT_TARGET" ]]; then
|
||||
warn "$ICON_LINK Symlink missing — recreating pointing to ramdisk"
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
ln -sfn "$RAMDISK_PATH" "$TRANSCODE_LINK"
|
||||
CURRENT_TARGET="$RAMDISK_PATH"
|
||||
notify "Transcode symlink was missing on $(hostname) ($MY_ID) — recreated" \
|
||||
"Transcode Manager" "warning"
|
||||
fi
|
||||
SOMETHING_HAPPENED=true
|
||||
elif [[ ! -e "$CURRENT_TARGET" ]]; then
|
||||
warn "$ICON_LINK Symlink target missing: $CURRENT_TARGET — resetting to ramdisk"
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
ln -sfn "$RAMDISK_PATH" "$TRANSCODE_LINK"
|
||||
CURRENT_TARGET="$RAMDISK_PATH"
|
||||
notify "Transcode symlink target was missing on $(hostname) ($MY_ID) — reset to ramdisk" \
|
||||
"Transcode Manager" "warning"
|
||||
fi
|
||||
SOMETHING_HAPPENED=true
|
||||
else
|
||||
log "Symlink valid: $TRANSCODE_LINK → $CURRENT_TARGET ✅"
|
||||
fi
|
||||
|
||||
# ── Check 3 — Ramdisk health ──────────────────────────────────────────────────────────────────
|
||||
log "Checking ramdisk..."
|
||||
|
||||
if ! mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
|
||||
error "Ramdisk not mounted at $RAMDISK_PATH"
|
||||
RAMDISK_HEALTHY=false
|
||||
SOMETHING_HAPPENED=true
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
warn "Flipping symlink to SSD — ramdisk unavailable"
|
||||
flip_symlink "$TRANSCODE_SSD" "ramdisk disappeared"
|
||||
CURRENT_TARGET="$TRANSCODE_SSD"
|
||||
notify "Ramdisk disappeared on $(hostname) ($MY_ID) — transcodes falling back to SSD. Run ramdisk_setup.sh to restore." \
|
||||
"Transcode Manager" "warning"
|
||||
fi
|
||||
else
|
||||
RAMDISK_SIZE_ACTUAL=$(df "$RAMDISK_PATH" --output=size -h 2>/dev/null | tail -1 | tr -d ' ')
|
||||
log "Ramdisk mounted — size: $RAMDISK_SIZE_ACTUAL ✅"
|
||||
fix_permissions "$RAMDISK_PATH"
|
||||
|
||||
# Guarantee transcoding-temp exists on ramdisk
|
||||
TRANSCODE_TEMP_RAM="${RAMDISK_PATH}/transcoding-temp"
|
||||
if [[ ! -d "$TRANSCODE_TEMP_RAM" ]]; then
|
||||
warn "transcoding-temp missing from ramdisk — creating now"
|
||||
mkdir -p "$TRANSCODE_TEMP_RAM"
|
||||
chmod "$TRANSCODE_CHMOD" "$TRANSCODE_TEMP_RAM"
|
||||
chown "$TRANSCODE_OWNER" "$TRANSCODE_TEMP_RAM"
|
||||
warn "transcoding-temp created on ramdisk — new sessions will use ramdisk ✅"
|
||||
SOMETHING_HAPPENED=true
|
||||
else
|
||||
log "transcoding-temp exists on ramdisk ✅"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Check 4 — SSD health ─────────────────────────────────────────────────────────────────────
|
||||
log "Checking SSD fallback..."
|
||||
|
||||
if [[ ! -d "$TRANSCODE_SSD" ]]; then
|
||||
warn "SSD fallback path missing: $TRANSCODE_SSD"
|
||||
SSD_HEALTHY=false
|
||||
SOMETHING_HAPPENED=true
|
||||
if [[ "$TRANSCODE_MANAGER_MODE" == "ssd" ]]; then
|
||||
error "Mode is 'ssd' but SSD path is missing — cannot continue"
|
||||
notify "Transcode SSD path missing on $(hostname) ($MY_ID) — mode is 'ssd', manual intervention needed" \
|
||||
"Transcode Manager" "warning"
|
||||
exit 1
|
||||
else
|
||||
warn "SSD fallback disabled — will stay on ramdisk"
|
||||
fi
|
||||
else
|
||||
SSD_FREE_GB=$(get_ssd_free_gb)
|
||||
log "SSD available — ${SSD_FREE_GB}GB free ✅"
|
||||
fix_permissions "$TRANSCODE_SSD"
|
||||
fi
|
||||
|
||||
# ── Usage stats ───────────────────────────────────────────────────────────────────────────────
|
||||
RAMDISK_USED_GB="0.00"
|
||||
RAMDISK_AVAIL_GB="0.00"
|
||||
RAMDISK_FILES=0
|
||||
|
||||
if [[ "$RAMDISK_HEALTHY" == true ]]; then
|
||||
RAMDISK_USED_GB=$(get_ramdisk_used_gb)
|
||||
RAMDISK_AVAIL_GB=$(get_ramdisk_avail_gb)
|
||||
RAMDISK_FILES=$(find "$RAMDISK_PATH" -type f 2>/dev/null | wc -l)
|
||||
fi
|
||||
|
||||
SSD_FILES=$(find "$TRANSCODE_SSD" -type f 2>/dev/null | wc -l)
|
||||
FLIP_COUNT=$(get_flip_count)
|
||||
|
||||
log "Ramdisk: ${RAMDISK_USED_GB}GB used / ${RAMDISK_AVAIL_GB}GB available ($RAMDISK_FILES files)"
|
||||
log "SSD: $SSD_FILES files"
|
||||
log "Flips: $FLIP_COUNT this hour"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Active Transcode Sessions ━━━
|
||||
# ==============================================================================================
|
||||
TOTAL_SESSIONS=0
|
||||
LIVE_TV=0
|
||||
TRANSCODING=0
|
||||
DIRECT=0
|
||||
ANY_SERVER_RUNNING=false
|
||||
RAM_SESSION_COUNT=0
|
||||
SSD_SESSION_COUNT=0
|
||||
|
||||
for server_entry in "${TRANSCODE_SERVERS[@]}"; do
|
||||
IFS='|' read -r SRV_CONTAINER SRV_URL SRV_KEY SRV_TYPE <<< "$server_entry"
|
||||
|
||||
# Skip placeholder entries
|
||||
if [[ "$SRV_KEY" == *"api-key"* ]] || \
|
||||
[[ "$SRV_KEY" == *"token"* && ${#SRV_KEY} -lt 20 ]]; then
|
||||
log "Skipping $SRV_CONTAINER — placeholder API key"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Container running check with timeout
|
||||
if ! timeout "$DOCKER_TIMEOUT" docker inspect "$SRV_CONTAINER" \
|
||||
--format '{{.State.Running}}' 2>/dev/null | grep -q "true"; then
|
||||
log "$SRV_CONTAINER — not running, skipping"
|
||||
continue
|
||||
fi
|
||||
|
||||
ANY_SERVER_RUNNING=true
|
||||
|
||||
if ! check_api "$SRV_URL" "$SRV_CONTAINER" 5; then
|
||||
warn "$SRV_CONTAINER — API unreachable, skipping"
|
||||
continue
|
||||
fi
|
||||
|
||||
case "$SRV_TYPE" in
|
||||
emby|jellyfin)
|
||||
SESSION_DATA=$(curl -sf --max-time 10 \
|
||||
-H "X-Emby-Token: $SRV_KEY" \
|
||||
"${SRV_URL}/Sessions" 2>/dev/null) ;;
|
||||
plex)
|
||||
SESSION_DATA=$(curl -sf --max-time 10 \
|
||||
-H "X-Plex-Token: $SRV_KEY" \
|
||||
"${SRV_URL}/status/sessions" 2>/dev/null) ;;
|
||||
*)
|
||||
warn "Unknown server type '$SRV_TYPE' for $SRV_CONTAINER — skipping"
|
||||
continue ;;
|
||||
esac
|
||||
|
||||
[[ -z "$SESSION_DATA" ]] && continue
|
||||
! command -v jq >/dev/null 2>&1 && continue
|
||||
|
||||
if [[ "$SRV_TYPE" == "emby" || "$SRV_TYPE" == "jellyfin" ]]; then
|
||||
SRV_TOTAL=$(echo "$SESSION_DATA" | \
|
||||
jq '[.[] | select(.NowPlayingItem != null)] | length' 2>/dev/null || echo 0)
|
||||
SRV_LIVE=$(echo "$SESSION_DATA" | \
|
||||
jq '[.[] | select(.NowPlayingItem != null) |
|
||||
select(.NowPlayingItem.Type == "LiveTv" or
|
||||
.NowPlayingItem.Type == "TvChannel")] | length' \
|
||||
2>/dev/null || echo 0)
|
||||
SRV_TRANSCODE=$(echo "$SESSION_DATA" | \
|
||||
jq '[.[] | select(.NowPlayingItem != null) |
|
||||
select(.PlayState.PlayMethod == "Transcode")] | length' \
|
||||
2>/dev/null || echo 0)
|
||||
SRV_DIRECT=$(echo "$SESSION_DATA" | \
|
||||
jq '[.[] | select(.NowPlayingItem != null) |
|
||||
select(.PlayState.PlayMethod != "Transcode")] | length' \
|
||||
2>/dev/null || echo 0)
|
||||
|
||||
TOTAL_SESSIONS=$(( TOTAL_SESSIONS + SRV_TOTAL ))
|
||||
LIVE_TV=$(( LIVE_TV + SRV_LIVE ))
|
||||
TRANSCODING=$(( TRANSCODING + SRV_TRANSCODE ))
|
||||
DIRECT=$(( DIRECT + SRV_DIRECT ))
|
||||
|
||||
if [[ "$SRV_TOTAL" -gt 0 ]]; then
|
||||
SOMETHING_HAPPENED=true
|
||||
[[ "${#TRANSCODE_SERVERS[@]}" -gt 1 ]] && \
|
||||
echo " $ICON_EMBY $SRV_CONTAINER — $SRV_TOTAL streams"
|
||||
|
||||
while IFS= read -r session; do
|
||||
USER=$(echo "$session" | jq -r '.UserName // "Unknown"' 2>/dev/null)
|
||||
TITLE=$(echo "$session" | jq -r '.NowPlayingItem.Name // "Unknown"' 2>/dev/null)
|
||||
MTYPE=$(echo "$session" | jq -r '.NowPlayingItem.Type // "Unknown"' 2>/dev/null)
|
||||
METH=$(echo "$session" | jq -r '.PlayState.PlayMethod // "Unknown"' 2>/dev/null)
|
||||
echo " $ICON_EMBY $(printf '%-12s' "$USER") — $(printf '%-30s' "$TITLE") — $(get_media_type_label "$MTYPE") — $(get_play_method_label "$METH")"
|
||||
done < <(echo "$SESSION_DATA" | jq -c '.[] | select(.NowPlayingItem != null)' 2>/dev/null)
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
# Storage state
|
||||
RAM_SESSION_COUNT=$(find "$RAMDISK_PATH/transcoding-temp" \
|
||||
-mindepth 1 -maxdepth 1 -type d 2>/dev/null | wc -l)
|
||||
SSD_SESSION_COUNT=$(find "$TRANSCODE_SSD/transcoding-temp" \
|
||||
-mindepth 1 -maxdepth 1 -type d 2>/dev/null | wc -l)
|
||||
|
||||
if [[ "$TOTAL_SESSIONS" -gt 0 ]]; then
|
||||
echo ""
|
||||
echo " $ICON_EMBY Total: $TOTAL_SESSIONS | Live TV: $LIVE_TV | Transcoding: $TRANSCODING | Direct: $DIRECT"
|
||||
|
||||
if [[ "$RAM_SESSION_COUNT" -gt 0 && "$SSD_SESSION_COUNT" -gt 0 ]]; then
|
||||
warn "Split state — $RAM_SESSION_COUNT folder(s) ramdisk / $SSD_SESSION_COUNT SSD"
|
||||
warn "Older sessions remain on original location until they end naturally"
|
||||
elif [[ "$RAM_SESSION_COUNT" -gt 0 ]]; then
|
||||
log "Storage: ramdisk ($RAM_SESSION_COUNT sessions)"
|
||||
elif [[ "$SSD_SESSION_COUNT" -gt 0 ]]; then
|
||||
log "Storage: SSD ($SSD_SESSION_COUNT sessions)"
|
||||
fi
|
||||
fi
|
||||
|
||||
[[ "$ANY_SERVER_RUNNING" == false && "$TRANSCODE_CHECK_EMBY" == true ]] && \
|
||||
{ log "No configured media servers running — skipping threshold checks"; EMBY_RUNNING=false; }
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Mode Logic ━━━
|
||||
# ==============================================================================================
|
||||
case "$TRANSCODE_MANAGER_MODE" in
|
||||
|
||||
ramdisk)
|
||||
log "Mode: RAMDISK — forcing symlink to ramdisk"
|
||||
if [[ "$RAMDISK_HEALTHY" == false ]]; then
|
||||
error "Ramdisk mode selected but ramdisk is not available"
|
||||
notify "Transcode ramdisk mode failed on $(hostname) ($MY_ID) — ramdisk not mounted" \
|
||||
"Transcode Manager" "warning"
|
||||
SOMETHING_HAPPENED=true
|
||||
else
|
||||
if [[ "$CURRENT_TARGET" != "$RAMDISK_PATH" ]]; then
|
||||
flip_symlink "$RAMDISK_PATH" "ramdisk mode"
|
||||
increment_flip_count > /dev/null
|
||||
FLIP_COUNT=$(get_flip_count)
|
||||
SOMETHING_HAPPENED=true
|
||||
fi
|
||||
if (( $(awk "BEGIN {print ($RAMDISK_USED_GB >= $RAMDISK_WARN_GB) ? 1 : 0}") )); then
|
||||
warn "Ramdisk usage ${RAMDISK_USED_GB}GB above threshold ${RAMDISK_WARN_GB}GB — consider switching to smart mode"
|
||||
notify "Ramdisk high on $(hostname) ($MY_ID) — ${RAMDISK_USED_GB}GB in ramdisk-only mode" \
|
||||
"Transcode Manager" "warning"
|
||||
SOMETHING_HAPPENED=true
|
||||
fi
|
||||
fi
|
||||
;;
|
||||
|
||||
ssd)
|
||||
log "Mode: SSD — forcing symlink to SSD"
|
||||
if [[ "$SSD_HEALTHY" == false ]]; then
|
||||
error "SSD mode selected but SSD path is not available"
|
||||
SOMETHING_HAPPENED=true
|
||||
else
|
||||
if [[ "$CURRENT_TARGET" != "$TRANSCODE_SSD" ]]; then
|
||||
flip_symlink "$TRANSCODE_SSD" "ssd mode"
|
||||
increment_flip_count > /dev/null
|
||||
FLIP_COUNT=$(get_flip_count)
|
||||
SOMETHING_HAPPENED=true
|
||||
fi
|
||||
fi
|
||||
;;
|
||||
|
||||
smart)
|
||||
log "Mode: SMART — auto threshold management"
|
||||
|
||||
if [[ "$EMBY_RUNNING" == false ]]; then
|
||||
log "No media servers running — skipping threshold checks"
|
||||
|
||||
elif [[ "$RAMDISK_HEALTHY" == false ]]; then
|
||||
log "Ramdisk unavailable — staying on SSD until ramdisk recovers"
|
||||
|
||||
elif [[ "$CURRENT_TARGET" == "$RAMDISK_PATH" ]]; then
|
||||
if (( $(awk "BEGIN {print ($RAMDISK_USED_GB >= $RAMDISK_WARN_GB) ? 1 : 0}") )); then
|
||||
if [[ "$SSD_HEALTHY" == false ]]; then
|
||||
error "Ramdisk above threshold but SSD unavailable — cannot flip"
|
||||
notify "Transcode ramdisk full on $(hostname) ($MY_ID) and SSD unavailable" \
|
||||
"Transcode Manager" "warning"
|
||||
SOMETHING_HAPPENED=true
|
||||
else
|
||||
SSD_FREE_GB=$(get_ssd_free_gb)
|
||||
if (( $(awk "BEGIN {print ($SSD_FREE_GB < $RAMDISK_SSD_MIN_GB) ? 1 : 0}") )); then
|
||||
warn "SSD only ${SSD_FREE_GB}GB free — below ${RAMDISK_SSD_MIN_GB}GB minimum, not flipping"
|
||||
SOMETHING_HAPPENED=true
|
||||
else
|
||||
warn "Ramdisk ${RAMDISK_USED_GB}GB — above ${RAMDISK_WARN_GB}GB, flipping to SSD"
|
||||
flip_symlink "$TRANSCODE_SSD" "threshold exceeded"
|
||||
CURRENT_TARGET="$TRANSCODE_SSD"
|
||||
NEW_COUNT=$(increment_flip_count)
|
||||
FLIP_COUNT=$NEW_COUNT
|
||||
SOMETHING_HAPPENED=true
|
||||
if [[ "$NEW_COUNT" -ge "$TRANSCODE_FLIP_WARN" ]]; then
|
||||
notify "Transcode flipped to SSD on $(hostname) ($MY_ID) — ${NEW_COUNT} flips this hour — consider increasing HOST*_RAMDISK_SIZE" \
|
||||
"Transcode Manager" "warning"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
else
|
||||
log "Ramdisk ${RAMDISK_USED_GB}GB — below threshold — no action needed"
|
||||
fi
|
||||
|
||||
elif [[ "$CURRENT_TARGET" == "$TRANSCODE_SSD" ]]; then
|
||||
if (( $(awk "BEGIN {print ($RAMDISK_USED_GB <= $RAMDISK_LOW_GB) ? 1 : 0}") )); then
|
||||
warn "Ramdisk ${RAMDISK_USED_GB}GB — below ${RAMDISK_LOW_GB}GB, flipping back to ramdisk"
|
||||
flip_symlink "$RAMDISK_PATH" "usage recovered"
|
||||
CURRENT_TARGET="$RAMDISK_PATH"
|
||||
NEW_COUNT=$(increment_flip_count)
|
||||
FLIP_COUNT=$NEW_COUNT
|
||||
SOMETHING_HAPPENED=true
|
||||
else
|
||||
log "On SSD — ramdisk ${RAMDISK_USED_GB}GB still above low threshold ${RAMDISK_LOW_GB}GB"
|
||||
fi
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
|
||||
END=$(date +%s)
|
||||
CURRENT_TARGET=$(readlink "$TRANSCODE_LINK" 2>/dev/null || echo "unknown")
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Daily Log Write ━━━
|
||||
# ==============================================================================================
|
||||
# Read by weekly_health_digest.sh — format: DATE|USED_GB|FLIPS|RAM_SESSIONS|SSD_SESSIONS
|
||||
if [[ "$DRY_RUN" == false && "$NO_LOG" == false && -n "${TRANSCODE_DAILY_LOG:-}" ]]; then
|
||||
TODAY=$(date '+%Y-%m-%d')
|
||||
mkdir -p "$(dirname "$TRANSCODE_DAILY_LOG")"
|
||||
echo "${TODAY}|${RAMDISK_USED_GB}|${FLIP_COUNT}|${RAM_SESSION_COUNT}|${SSD_SESSION_COUNT}" \
|
||||
>> "$TRANSCODE_DAILY_LOG" 2>/dev/null || true
|
||||
log "Daily log written: $TRANSCODE_DAILY_LOG"
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary — only shown when something happened ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SOMETHING_HAPPENED" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY TRANSCODE MANAGER SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_GEAR Mode: $TRANSCODE_MANAGER_MODE"
|
||||
echo "$ICON_RAM Ramdisk: ${RAMDISK_USED_GB}GB used / ${RAMDISK_AVAIL_GB}GB available"
|
||||
echo "$ICON_LINK Symlink: $TRANSCODE_LINK → $CURRENT_TARGET"
|
||||
echo "$ICON_LINK Flips: $FLIP_COUNT this hour (warn at $TRANSCODE_FLIP_WARN)"
|
||||
echo "$ICON_EMBY Streams: $TOTAL_SESSIONS total | Live TV: $LIVE_TV | Transcoding: $TRANSCODING | Direct: $DIRECT"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
else
|
||||
echo "Transcode manager — all healthy — $MY_ID ($(format_duration $(( END - START ))))"
|
||||
fi
|
||||
+662
@@ -0,0 +1,662 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ================================= Transcode Manager ==========================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Monitors ramdisk usage and manages the transcode symlink direction. Called by
|
||||
# transcode_management.sh (Orchestrators/) every 7 minutes — always after
|
||||
# transcode_cleanup.sh runs first. Must be fast, non-blocking, and silent when
|
||||
# nothing has changed.
|
||||
#
|
||||
# Emby's transcode path points at TRANSCODE_LINK (a symlink). ffmpeg resolves
|
||||
# the symlink once at session start and holds a direct reference — existing
|
||||
# sessions are completely unaffected by symlink flips. Only new sessions care
|
||||
# where the symlink currently points.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Three Modes (TRANSCODE_MANAGER_MODE)
|
||||
# smart — auto-flips between ramdisk and SSD based on usage thresholds (default)
|
||||
# ramdisk above RAMDISK_WARN_GB → flip to SSD
|
||||
# ramdisk below RAMDISK_LOW_GB → flip back to ramdisk
|
||||
# ramdisk — always uses ramdisk, warns if above threshold, never flips
|
||||
# ssd — always uses SSD, never uses ramdisk (use during drain or maintenance)
|
||||
#
|
||||
# Safety Checks — Every Run Regardless of Mode
|
||||
# Symlink missing/broken → recreate pointing at ramdisk, notify
|
||||
# Ramdisk disappeared → flip to SSD immediately, notify warning
|
||||
# SSD path missing → disable SSD fallback (error if mode=ssd)
|
||||
# transcoding-temp missing → recreate on ramdisk silently
|
||||
# Permissions drift → fix silently every run
|
||||
# Emby not running → skip threshold checks, verify symlink only
|
||||
#
|
||||
# Session Display
|
||||
# Shows active streams from all configured TRANSCODE_SERVERS with user, title,
|
||||
# type (Live TV / TV Show / Movie), and play method (Transcode / Direct).
|
||||
# Split state shown when sessions exist on both ramdisk and SSD — normal during
|
||||
# a flip while ramdisk sessions drain.
|
||||
#
|
||||
# Daily Log
|
||||
# Appends one entry per run to TRANSCODE_DAILY_LOG, read by weekly_health_digest.sh.
|
||||
# Format: DATE|RAMDISK_USED_GB|FLIP_COUNT|RAM_SESSION_COUNT|SSD_SESSION_COUNT|FILES_CLEANED
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Wait Lock
|
||||
# acquire_lock "wait" — waits if the previous run is still active. The 7-minute
|
||||
# interval can overlap on a system under heavy load.
|
||||
#
|
||||
# Docker Timeout
|
||||
# DOCKER_TIMEOUT caps all docker calls against a hung daemon.
|
||||
#
|
||||
# Notification Validated
|
||||
# platform_require_cmd confirms the notify script is present before use.
|
||||
#
|
||||
# Silent by Default
|
||||
# Runs every 7 minutes — only speaks when something changes or needs attention.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# host*.conf
|
||||
#
|
||||
# HOST*_RAMDISK_PATH / HOST*_TRANSCODE_SSD
|
||||
# Ramdisk mount point and SSD fallback path.
|
||||
# Aliased by detect_hosts().
|
||||
#
|
||||
# HOST*_RAMDISK_WARN_GB / HOST*_RAMDISK_LOW_GB / HOST*_RAMDISK_SIZE
|
||||
# Thresholds and ceiling. Change all three together.
|
||||
# Aliased by detect_hosts().
|
||||
#
|
||||
# HOST*_TRANSCODE_SERVERS
|
||||
# Array of media server definitions: "ContainerName|URL|APIKey|Type"
|
||||
# Type: emby | jellyfin | plex
|
||||
# Aliased by detect_hosts() → TRANSCODE_SERVERS.
|
||||
#
|
||||
# master.conf
|
||||
#
|
||||
# TRANSCODE_MANAGER_MODE
|
||||
# smart | ramdisk | ssd. (default: smart)
|
||||
#
|
||||
# TRANSCODE_CHECK_EMBY
|
||||
# Skip threshold checks when Emby not running — prevents unnecessary flips
|
||||
# overnight when no sessions are active. (default: true)
|
||||
#
|
||||
# TRANSCODE_FLIP_WARN
|
||||
# Notify if symlink flips this many times in one hour — indicates ramdisk
|
||||
# is undersized for the load. (default: 3)
|
||||
#
|
||||
# ==============================================================================================
|
||||
# STATE FILES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# /tmp/transcode_state.db — current symlink target, flip count, last flip time
|
||||
# Lives in /tmp (ephemeral — resets correctly on reboot)
|
||||
# TRANSCODE_DAILY_LOG — per-run append, read by weekly_health_digest.sh
|
||||
# Trimmed to TRANSCODE_LOG_RETENTION days on each write
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# transcode_manager.sh
|
||||
# Check usage, flip if needed, run safety checks, display active sessions.
|
||||
#
|
||||
# transcode_manager.sh --dry-run
|
||||
# Show current usage and what flip decision would be made. No changes.
|
||||
#
|
||||
# transcode_manager.sh --status
|
||||
# Show current symlink target, ramdisk usage, session counts, and flip history.
|
||||
#
|
||||
# transcode_manager.sh --log
|
||||
# Verbose output including per-check results and session detail.
|
||||
#
|
||||
# transcode_manager.sh --no-log
|
||||
# Suppress daily log write. Used internally when called by transcode_cleanup.sh.
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
# Handle --no-log flag before parse_args
|
||||
NO_LOG=false
|
||||
FILTERED_ARGS=()
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--no-log) NO_LOG=true ;;
|
||||
*) FILTERED_ARGS+=("$arg") ;;
|
||||
esac
|
||||
done
|
||||
|
||||
parse_args "${FILTERED_ARGS[@]}"
|
||||
|
||||
DOCKER_TIMEOUT=15
|
||||
STATE_FILE="${TRANSCODE_STATE_FILE:-${STATE_DIR:-/tmp}/transcode_state.db}"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
acquire_lock "wait"
|
||||
|
||||
if ! command -v docker &>/dev/null; then
|
||||
error "Docker command not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# detect_hosts() sets MY_ID and aliases all HOST*_TRANSCODE_* vars
|
||||
detect_hosts
|
||||
|
||||
# Primary container — first entry in TRANSCODE_SERVERS
|
||||
TRANSCODE_EMBY_CONTAINER="Emby"
|
||||
if [[ "${#TRANSCODE_SERVERS[@]}" -gt 0 ]]; then
|
||||
IFS='|' read -r PRIMARY_CONTAINER _ _ _ <<< "${TRANSCODE_SERVERS[0]}"
|
||||
TRANSCODE_EMBY_CONTAINER="${PRIMARY_CONTAINER:-Emby}"
|
||||
fi
|
||||
|
||||
case "$TRANSCODE_MANAGER_MODE" in
|
||||
smart|ramdisk|ssd) log "Mode: $TRANSCODE_MANAGER_MODE" ;;
|
||||
*)
|
||||
warn "Unknown TRANSCODE_MANAGER_MODE: $TRANSCODE_MANAGER_MODE — defaulting to smart"
|
||||
TRANSCODE_MANAGER_MODE="smart"
|
||||
;;
|
||||
esac
|
||||
|
||||
log "$ICON_GEAR Config: ramdisk=$RAMDISK_PATH size=$RAMDISK_SIZE warn-at=${RAMDISK_WARN_GB}GB flip-back-at=${RAMDISK_LOW_GB}GB ssd-min-free=${RAMDISK_SSD_MIN_GB}GB flip-warn=${TRANSCODE_FLIP_WARN}/hr"
|
||||
log "$ICON_DISK SSD: $TRANSCODE_SSD"
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY TRANSCODE MANAGER STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_GEAR Mode: $TRANSCODE_MANAGER_MODE"
|
||||
echo "$ICON_RAM Ramdisk: $RAMDISK_PATH ($RAMDISK_SIZE)"
|
||||
echo "$ICON_LINK Symlink: $TRANSCODE_LINK"
|
||||
echo "$ICON_DISK SSD: $TRANSCODE_SSD"
|
||||
echo "$ICON_RAM Warn at: ${RAMDISK_WARN_GB}GB"
|
||||
echo "$ICON_RAM Low at: ${RAMDISK_LOW_GB}GB"
|
||||
echo "$ICON_DISK SSD min free: ${RAMDISK_SSD_MIN_GB}GB"
|
||||
echo "$ICON_RAM Flip warn: $TRANSCODE_FLIP_WARN per hour"
|
||||
echo "$ICON_GEAR Check Emby: $TRANSCODE_CHECK_EMBY ($TRANSCODE_EMBY_CONTAINER)"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
CURRENT_TARGET=$(readlink "$TRANSCODE_LINK" 2>/dev/null || echo "missing")
|
||||
echo "$ICON_LINK Symlink now: $TRANSCODE_LINK → $CURRENT_TARGET"
|
||||
if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
|
||||
USED_GB=$(df "$RAMDISK_PATH" --output=used 2>/dev/null | \
|
||||
tail -1 | tr -d ' ' | awk '{printf "%.2f", $1/1048576}')
|
||||
echo "$ICON_RAM Ramdisk now: ${USED_GB}GB used"
|
||||
else
|
||||
echo "$ICON_RAM Ramdisk: NOT MOUNTED"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ── HELPER FUNCTIONS ──────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
flip_symlink() {
|
||||
local target="$1" reason="$2"
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would flip symlink to: $target ($reason)"
|
||||
return 0
|
||||
fi
|
||||
ln -sfn "$target" "$TRANSCODE_LINK"
|
||||
warn "$ICON_LINK Symlink flipped to: $target ($reason)"
|
||||
}
|
||||
|
||||
fix_permissions() {
|
||||
local path="$1"
|
||||
[[ ! -d "$path" ]] && return
|
||||
chown -R "$TRANSCODE_OWNER" "$path" 2>/dev/null
|
||||
chmod -R "$TRANSCODE_CHMOD" "$path" 2>/dev/null
|
||||
log "Permissions fixed on $path"
|
||||
}
|
||||
|
||||
get_ramdisk_used_gb() {
|
||||
df "$RAMDISK_PATH" --output=used 2>/dev/null | \
|
||||
tail -1 | tr -d ' ' | awk '{printf "%.2f", $1/1048576}'
|
||||
}
|
||||
|
||||
get_ramdisk_avail_gb() {
|
||||
df "$RAMDISK_PATH" --output=avail 2>/dev/null | \
|
||||
tail -1 | tr -d ' ' | awk '{printf "%.2f", $1/1048576}'
|
||||
}
|
||||
|
||||
get_ssd_free_gb() {
|
||||
df "$TRANSCODE_SSD" --output=avail 2>/dev/null | \
|
||||
tail -1 | tr -d ' ' | awk '{printf "%.2f", $1/1048576}'
|
||||
}
|
||||
|
||||
get_flip_count() {
|
||||
local state_file="${STATE_DIR:-/tmp}/transcode_flip_state.db"
|
||||
local current_hour
|
||||
current_hour=$(date '+%Y-%m-%d-%H')
|
||||
[[ ! -f "$state_file" ]] && echo "0" && return
|
||||
local stored_hour stored_count
|
||||
stored_hour=$(awk -F'|' 'NR==1{print $1}' "$state_file" 2>/dev/null)
|
||||
stored_count=$(awk -F'|' 'NR==1{print $2}' "$state_file" 2>/dev/null)
|
||||
[[ "$stored_hour" == "$current_hour" ]] && echo "${stored_count:-0}" || echo "0"
|
||||
}
|
||||
|
||||
increment_flip_count() {
|
||||
local state_file="${STATE_DIR:-/tmp}/transcode_flip_state.db"
|
||||
local current_hour
|
||||
current_hour=$(date '+%Y-%m-%d-%H')
|
||||
local current_count
|
||||
current_count=$(get_flip_count)
|
||||
current_count=$(( current_count + 1 ))
|
||||
echo "${current_hour}|${current_count}" > "$state_file"
|
||||
echo "$current_count"
|
||||
}
|
||||
|
||||
get_media_type_label() {
|
||||
case "$1" in
|
||||
LiveTv|TvChannel) echo "Live TV" ;;
|
||||
Episode) echo "TV Show" ;;
|
||||
Movie) echo "Movie" ;;
|
||||
Audio) echo "Music" ;;
|
||||
MusicVideo) echo "Music Video" ;;
|
||||
*) echo "$1" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
get_play_method_label() {
|
||||
case "$1" in
|
||||
Transcode) echo "Transcode" ;;
|
||||
DirectStream) echo "Direct Stream" ;;
|
||||
DirectPlay) echo "Direct Play" ;;
|
||||
*) echo "$1" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Transcode Manager ━━━
|
||||
# ==============================================================================================
|
||||
START=$(date +%s)
|
||||
RAMDISK_HEALTHY=true
|
||||
SSD_HEALTHY=true
|
||||
EMBY_RUNNING=true
|
||||
SOMETHING_HAPPENED=false # controls whether summary is printed
|
||||
|
||||
# ── Check 1 — Emby running ───────────────────────────────────────────────────────────────────
|
||||
if [[ "$TRANSCODE_CHECK_EMBY" == true ]]; then
|
||||
log "Checking $TRANSCODE_EMBY_CONTAINER..."
|
||||
if ! timeout "$DOCKER_TIMEOUT" docker inspect "$TRANSCODE_EMBY_CONTAINER" \
|
||||
--format '{{.State.Running}}' 2>/dev/null | grep -q "true"; then
|
||||
warn "$TRANSCODE_EMBY_CONTAINER is not running — skipping threshold checks"
|
||||
EMBY_RUNNING=false
|
||||
SOMETHING_HAPPENED=true
|
||||
else
|
||||
log "$TRANSCODE_EMBY_CONTAINER is running ✅"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Check 2 — Symlink integrity ───────────────────────────────────────────────────────────────
|
||||
log "Checking symlink integrity..."
|
||||
CURRENT_TARGET=$(readlink "$TRANSCODE_LINK" 2>/dev/null)
|
||||
|
||||
if [[ -z "$CURRENT_TARGET" ]]; then
|
||||
warn "$ICON_LINK Symlink missing — recreating pointing to ramdisk"
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
ln -sfn "$RAMDISK_PATH" "$TRANSCODE_LINK"
|
||||
CURRENT_TARGET="$RAMDISK_PATH"
|
||||
notify "Transcode symlink was missing on $(hostname) ($MY_ID) — recreated" \
|
||||
"Transcode Manager" "warning"
|
||||
fi
|
||||
SOMETHING_HAPPENED=true
|
||||
elif [[ ! -e "$CURRENT_TARGET" ]]; then
|
||||
warn "$ICON_LINK Symlink target missing: $CURRENT_TARGET — resetting to ramdisk"
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
ln -sfn "$RAMDISK_PATH" "$TRANSCODE_LINK"
|
||||
CURRENT_TARGET="$RAMDISK_PATH"
|
||||
notify "Transcode symlink target was missing on $(hostname) ($MY_ID) — reset to ramdisk" \
|
||||
"Transcode Manager" "warning"
|
||||
fi
|
||||
SOMETHING_HAPPENED=true
|
||||
else
|
||||
log "Symlink valid: $TRANSCODE_LINK → $CURRENT_TARGET ✅"
|
||||
fi
|
||||
|
||||
# ── Check 3 — Ramdisk health ──────────────────────────────────────────────────────────────────
|
||||
log "Checking ramdisk..."
|
||||
|
||||
if ! mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
|
||||
error "Ramdisk not mounted at $RAMDISK_PATH"
|
||||
RAMDISK_HEALTHY=false
|
||||
SOMETHING_HAPPENED=true
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
warn "Flipping symlink to SSD — ramdisk unavailable"
|
||||
flip_symlink "$TRANSCODE_SSD" "ramdisk disappeared"
|
||||
CURRENT_TARGET="$TRANSCODE_SSD"
|
||||
notify "Ramdisk disappeared on $(hostname) ($MY_ID) — transcodes falling back to SSD. Run ramdisk_setup.sh to restore." \
|
||||
"Transcode Manager" "warning"
|
||||
fi
|
||||
else
|
||||
RAMDISK_SIZE_ACTUAL=$(df "$RAMDISK_PATH" --output=size -h 2>/dev/null | tail -1 | tr -d ' ')
|
||||
log "Ramdisk mounted — size: $RAMDISK_SIZE_ACTUAL ✅"
|
||||
fix_permissions "$RAMDISK_PATH"
|
||||
|
||||
# Guarantee transcoding-temp exists on ramdisk
|
||||
TRANSCODE_TEMP_RAM="${RAMDISK_PATH}/transcoding-temp"
|
||||
if [[ ! -d "$TRANSCODE_TEMP_RAM" ]]; then
|
||||
warn "transcoding-temp missing from ramdisk — creating now"
|
||||
mkdir -p "$TRANSCODE_TEMP_RAM"
|
||||
chmod "$TRANSCODE_CHMOD" "$TRANSCODE_TEMP_RAM"
|
||||
chown "$TRANSCODE_OWNER" "$TRANSCODE_TEMP_RAM"
|
||||
warn "transcoding-temp created on ramdisk — new sessions will use ramdisk ✅"
|
||||
SOMETHING_HAPPENED=true
|
||||
else
|
||||
log "transcoding-temp exists on ramdisk ✅"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Check 4 — SSD health ─────────────────────────────────────────────────────────────────────
|
||||
log "Checking SSD fallback..."
|
||||
|
||||
if [[ ! -d "$TRANSCODE_SSD" ]]; then
|
||||
warn "SSD fallback path missing: $TRANSCODE_SSD"
|
||||
SSD_HEALTHY=false
|
||||
SOMETHING_HAPPENED=true
|
||||
if [[ "$TRANSCODE_MANAGER_MODE" == "ssd" ]]; then
|
||||
error "Mode is 'ssd' but SSD path is missing — cannot continue"
|
||||
notify "Transcode SSD path missing on $(hostname) ($MY_ID) — mode is 'ssd', manual intervention needed" \
|
||||
"Transcode Manager" "warning"
|
||||
exit 1
|
||||
else
|
||||
warn "SSD fallback disabled — will stay on ramdisk"
|
||||
fi
|
||||
else
|
||||
SSD_FREE_GB=$(get_ssd_free_gb)
|
||||
log "SSD available — ${SSD_FREE_GB}GB free ✅"
|
||||
fix_permissions "$TRANSCODE_SSD"
|
||||
fi
|
||||
|
||||
# ── Usage stats ───────────────────────────────────────────────────────────────────────────────
|
||||
RAMDISK_USED_GB="0.00"
|
||||
RAMDISK_AVAIL_GB="0.00"
|
||||
RAMDISK_FILES=0
|
||||
|
||||
if [[ "$RAMDISK_HEALTHY" == true ]]; then
|
||||
RAMDISK_USED_GB=$(get_ramdisk_used_gb)
|
||||
RAMDISK_AVAIL_GB=$(get_ramdisk_avail_gb)
|
||||
RAMDISK_FILES=$(find "$RAMDISK_PATH" -type f 2>/dev/null | wc -l)
|
||||
fi
|
||||
|
||||
SSD_FILES=$(find "$TRANSCODE_SSD" -type f 2>/dev/null | wc -l)
|
||||
FLIP_COUNT=$(get_flip_count)
|
||||
|
||||
log "Ramdisk: ${RAMDISK_USED_GB}GB used / ${RAMDISK_AVAIL_GB}GB available ($RAMDISK_FILES files)"
|
||||
log "SSD: $SSD_FILES files"
|
||||
log "Flips: $FLIP_COUNT this hour"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Active Transcode Sessions ━━━
|
||||
# ==============================================================================================
|
||||
TOTAL_SESSIONS=0
|
||||
LIVE_TV=0
|
||||
TRANSCODING=0
|
||||
DIRECT=0
|
||||
ANY_SERVER_RUNNING=false
|
||||
RAM_SESSION_COUNT=0
|
||||
SSD_SESSION_COUNT=0
|
||||
|
||||
for server_entry in "${TRANSCODE_SERVERS[@]}"; do
|
||||
IFS='|' read -r SRV_CONTAINER SRV_URL SRV_KEY SRV_TYPE <<< "$server_entry"
|
||||
|
||||
# Skip placeholder entries
|
||||
if [[ "$SRV_KEY" == *"api-key"* ]] || \
|
||||
[[ "$SRV_KEY" == *"token"* && ${#SRV_KEY} -lt 20 ]]; then
|
||||
log "Skipping $SRV_CONTAINER — placeholder API key"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Container running check with timeout
|
||||
if ! timeout "$DOCKER_TIMEOUT" docker inspect "$SRV_CONTAINER" \
|
||||
--format '{{.State.Running}}' 2>/dev/null | grep -q "true"; then
|
||||
log "$SRV_CONTAINER — not running, skipping"
|
||||
continue
|
||||
fi
|
||||
|
||||
ANY_SERVER_RUNNING=true
|
||||
|
||||
if ! check_api "$SRV_URL" "$SRV_CONTAINER" 5; then
|
||||
warn "$SRV_CONTAINER — API unreachable, skipping"
|
||||
continue
|
||||
fi
|
||||
|
||||
case "$SRV_TYPE" in
|
||||
emby|jellyfin)
|
||||
SESSION_DATA=$(curl -sf --max-time 10 \
|
||||
-H "X-Emby-Token: $SRV_KEY" \
|
||||
"${SRV_URL}/Sessions" 2>/dev/null) ;;
|
||||
plex)
|
||||
SESSION_DATA=$(curl -sf --max-time 10 \
|
||||
-H "X-Plex-Token: $SRV_KEY" \
|
||||
"${SRV_URL}/status/sessions" 2>/dev/null) ;;
|
||||
*)
|
||||
warn "Unknown server type '$SRV_TYPE' for $SRV_CONTAINER — skipping"
|
||||
continue ;;
|
||||
esac
|
||||
|
||||
[[ -z "$SESSION_DATA" ]] && continue
|
||||
! command -v jq >/dev/null 2>&1 && continue
|
||||
|
||||
if [[ "$SRV_TYPE" == "emby" || "$SRV_TYPE" == "jellyfin" ]]; then
|
||||
SRV_TOTAL=$(echo "$SESSION_DATA" | \
|
||||
jq '[.[] | select(.NowPlayingItem != null)] | length' 2>/dev/null || echo 0)
|
||||
SRV_LIVE=$(echo "$SESSION_DATA" | \
|
||||
jq '[.[] | select(.NowPlayingItem != null) |
|
||||
select(.NowPlayingItem.Type == "LiveTv" or
|
||||
.NowPlayingItem.Type == "TvChannel")] | length' \
|
||||
2>/dev/null || echo 0)
|
||||
SRV_TRANSCODE=$(echo "$SESSION_DATA" | \
|
||||
jq '[.[] | select(.NowPlayingItem != null) |
|
||||
select(.PlayState.PlayMethod == "Transcode")] | length' \
|
||||
2>/dev/null || echo 0)
|
||||
SRV_DIRECT=$(echo "$SESSION_DATA" | \
|
||||
jq '[.[] | select(.NowPlayingItem != null) |
|
||||
select(.PlayState.PlayMethod != "Transcode")] | length' \
|
||||
2>/dev/null || echo 0)
|
||||
|
||||
TOTAL_SESSIONS=$(( TOTAL_SESSIONS + SRV_TOTAL ))
|
||||
LIVE_TV=$(( LIVE_TV + SRV_LIVE ))
|
||||
TRANSCODING=$(( TRANSCODING + SRV_TRANSCODE ))
|
||||
DIRECT=$(( DIRECT + SRV_DIRECT ))
|
||||
|
||||
if [[ "$SRV_TOTAL" -gt 0 ]]; then
|
||||
SOMETHING_HAPPENED=true
|
||||
[[ "${#TRANSCODE_SERVERS[@]}" -gt 1 ]] && \
|
||||
echo " $ICON_EMBY $SRV_CONTAINER — $SRV_TOTAL streams"
|
||||
|
||||
while IFS= read -r session; do
|
||||
USER=$(echo "$session" | jq -r '.UserName // "Unknown"' 2>/dev/null)
|
||||
TITLE=$(echo "$session" | jq -r '.NowPlayingItem.Name // "Unknown"' 2>/dev/null)
|
||||
MTYPE=$(echo "$session" | jq -r '.NowPlayingItem.Type // "Unknown"' 2>/dev/null)
|
||||
METH=$(echo "$session" | jq -r '.PlayState.PlayMethod // "Unknown"' 2>/dev/null)
|
||||
echo " $ICON_EMBY $(printf '%-12s' "$USER") — $(printf '%-30s' "$TITLE") — $(get_media_type_label "$MTYPE") — $(get_play_method_label "$METH")"
|
||||
done < <(echo "$SESSION_DATA" | jq -c '.[] | select(.NowPlayingItem != null)' 2>/dev/null)
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
# Storage state
|
||||
# Sessions may use subdirectories (legacy transcode) or flat files (Live TV / Direct Stream HLS).
|
||||
# Count both: subdirs per-session, and unique hex session ID prefixes for flat files.
|
||||
_count_sessions() {
|
||||
local base="$1/transcoding-temp"
|
||||
local _dirs _flat
|
||||
_dirs=$(find "$base" -mindepth 1 -maxdepth 1 -type d 2>/dev/null | wc -l)
|
||||
_flat=$(find "$base" -maxdepth 1 -type f -printf '%f\n' 2>/dev/null | \
|
||||
grep -oE '^[0-9a-f]{16,}' | sort -u | wc -l)
|
||||
echo $(( _dirs + _flat ))
|
||||
}
|
||||
RAM_SESSION_COUNT=$(_count_sessions "$RAMDISK_PATH")
|
||||
SSD_SESSION_COUNT=$(_count_sessions "$TRANSCODE_SSD")
|
||||
unset -f _count_sessions
|
||||
|
||||
if [[ "$TOTAL_SESSIONS" -gt 0 ]]; then
|
||||
echo ""
|
||||
echo " $ICON_EMBY Total: $TOTAL_SESSIONS | Live TV: $LIVE_TV | Transcoding: $TRANSCODING | Direct: $DIRECT"
|
||||
|
||||
if [[ "$RAM_SESSION_COUNT" -gt 0 && "$SSD_SESSION_COUNT" -gt 0 ]]; then
|
||||
warn "Split state — $RAM_SESSION_COUNT session(s) ramdisk / $SSD_SESSION_COUNT SSD"
|
||||
warn "Older sessions remain on original location until they end naturally"
|
||||
elif [[ "$RAM_SESSION_COUNT" -gt 0 ]]; then
|
||||
log "Storage: ramdisk ($RAM_SESSION_COUNT session(s))"
|
||||
elif [[ "$SSD_SESSION_COUNT" -gt 0 ]]; then
|
||||
log "Storage: SSD ($SSD_SESSION_COUNT session(s))"
|
||||
fi
|
||||
fi
|
||||
|
||||
[[ "$ANY_SERVER_RUNNING" == false && "$TRANSCODE_CHECK_EMBY" == true ]] && \
|
||||
{ log "No configured media servers running — skipping threshold checks"; EMBY_RUNNING=false; }
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Mode Logic ━━━
|
||||
# ==============================================================================================
|
||||
case "$TRANSCODE_MANAGER_MODE" in
|
||||
|
||||
ramdisk)
|
||||
log "Mode: RAMDISK — forcing symlink to ramdisk"
|
||||
if [[ "$RAMDISK_HEALTHY" == false ]]; then
|
||||
error "Ramdisk mode selected but ramdisk is not available"
|
||||
notify "Transcode ramdisk mode failed on $(hostname) ($MY_ID) — ramdisk not mounted" \
|
||||
"Transcode Manager" "warning"
|
||||
SOMETHING_HAPPENED=true
|
||||
else
|
||||
if [[ "$CURRENT_TARGET" != "$RAMDISK_PATH" ]]; then
|
||||
flip_symlink "$RAMDISK_PATH" "ramdisk mode"
|
||||
increment_flip_count > /dev/null
|
||||
FLIP_COUNT=$(get_flip_count)
|
||||
SOMETHING_HAPPENED=true
|
||||
fi
|
||||
if (( $(awk "BEGIN {print ($RAMDISK_USED_GB >= $RAMDISK_WARN_GB) ? 1 : 0}") )); then
|
||||
warn "Ramdisk usage ${RAMDISK_USED_GB}GB above threshold ${RAMDISK_WARN_GB}GB — consider switching to smart mode"
|
||||
notify "Ramdisk high on $(hostname) ($MY_ID) — ${RAMDISK_USED_GB}GB in ramdisk-only mode" \
|
||||
"Transcode Manager" "warning"
|
||||
SOMETHING_HAPPENED=true
|
||||
fi
|
||||
fi
|
||||
;;
|
||||
|
||||
ssd)
|
||||
log "Mode: SSD — forcing symlink to SSD"
|
||||
if [[ "$SSD_HEALTHY" == false ]]; then
|
||||
error "SSD mode selected but SSD path is not available"
|
||||
SOMETHING_HAPPENED=true
|
||||
else
|
||||
if [[ "$CURRENT_TARGET" != "$TRANSCODE_SSD" ]]; then
|
||||
flip_symlink "$TRANSCODE_SSD" "ssd mode"
|
||||
increment_flip_count > /dev/null
|
||||
FLIP_COUNT=$(get_flip_count)
|
||||
SOMETHING_HAPPENED=true
|
||||
fi
|
||||
fi
|
||||
;;
|
||||
|
||||
smart)
|
||||
log "Mode: SMART — auto threshold management"
|
||||
|
||||
if [[ "$EMBY_RUNNING" == false ]]; then
|
||||
log "No media servers running — skipping threshold checks"
|
||||
|
||||
elif [[ "$RAMDISK_HEALTHY" == false ]]; then
|
||||
log "Ramdisk unavailable — staying on SSD until ramdisk recovers"
|
||||
|
||||
elif [[ "$CURRENT_TARGET" == "$RAMDISK_PATH" ]]; then
|
||||
if (( $(awk "BEGIN {print ($RAMDISK_USED_GB >= $RAMDISK_WARN_GB) ? 1 : 0}") )); then
|
||||
if [[ "$SSD_HEALTHY" == false ]]; then
|
||||
error "Ramdisk above threshold but SSD unavailable — cannot flip"
|
||||
notify "Transcode ramdisk full on $(hostname) ($MY_ID) and SSD unavailable" \
|
||||
"Transcode Manager" "warning"
|
||||
SOMETHING_HAPPENED=true
|
||||
else
|
||||
SSD_FREE_GB=$(get_ssd_free_gb)
|
||||
if (( $(awk "BEGIN {print ($SSD_FREE_GB < $RAMDISK_SSD_MIN_GB) ? 1 : 0}") )); then
|
||||
warn "SSD only ${SSD_FREE_GB}GB free — below ${RAMDISK_SSD_MIN_GB}GB minimum, not flipping"
|
||||
SOMETHING_HAPPENED=true
|
||||
else
|
||||
warn "Ramdisk ${RAMDISK_USED_GB}GB — above ${RAMDISK_WARN_GB}GB, flipping to SSD"
|
||||
flip_symlink "$TRANSCODE_SSD" "threshold exceeded"
|
||||
CURRENT_TARGET="$TRANSCODE_SSD"
|
||||
NEW_COUNT=$(increment_flip_count)
|
||||
FLIP_COUNT=$NEW_COUNT
|
||||
SOMETHING_HAPPENED=true
|
||||
if [[ "$NEW_COUNT" -ge "$TRANSCODE_FLIP_WARN" ]]; then
|
||||
notify "Transcode flipped to SSD on $(hostname) ($MY_ID) — ${NEW_COUNT} flips this hour — consider increasing HOST*_RAMDISK_SIZE" \
|
||||
"Transcode Manager" "warning"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
else
|
||||
log "Ramdisk ${RAMDISK_USED_GB}GB — below threshold — no action needed"
|
||||
fi
|
||||
|
||||
elif [[ "$CURRENT_TARGET" == "$TRANSCODE_SSD" ]]; then
|
||||
if (( $(awk "BEGIN {print ($RAMDISK_USED_GB <= $RAMDISK_LOW_GB) ? 1 : 0}") )); then
|
||||
warn "Ramdisk ${RAMDISK_USED_GB}GB — below ${RAMDISK_LOW_GB}GB, flipping back to ramdisk"
|
||||
flip_symlink "$RAMDISK_PATH" "usage recovered"
|
||||
CURRENT_TARGET="$RAMDISK_PATH"
|
||||
NEW_COUNT=$(increment_flip_count)
|
||||
FLIP_COUNT=$NEW_COUNT
|
||||
SOMETHING_HAPPENED=true
|
||||
else
|
||||
log "On SSD — ramdisk ${RAMDISK_USED_GB}GB still above low threshold ${RAMDISK_LOW_GB}GB"
|
||||
fi
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
|
||||
END=$(date +%s)
|
||||
CURRENT_TARGET=$(readlink "$TRANSCODE_LINK" 2>/dev/null || echo "unknown")
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Daily Log Write ━━━
|
||||
# ==============================================================================================
|
||||
# Read by weekly_health_digest.sh — format: DATE|USED_GB|FLIPS|RAM_SESSIONS|SSD_SESSIONS
|
||||
if [[ "$DRY_RUN" == false && "$NO_LOG" == false && -n "${TRANSCODE_DAILY_LOG:-}" ]]; then
|
||||
TODAY=$(date '+%Y-%m-%d')
|
||||
mkdir -p "$(dirname "$TRANSCODE_DAILY_LOG")"
|
||||
echo "${TODAY}|${RAMDISK_USED_GB}|${FLIP_COUNT}|${RAM_SESSION_COUNT}|${SSD_SESSION_COUNT}" \
|
||||
>> "$TRANSCODE_DAILY_LOG" 2>/dev/null || true
|
||||
log "Daily log written: $TRANSCODE_DAILY_LOG"
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary — only shown when something happened ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SOMETHING_HAPPENED" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY TRANSCODE MANAGER SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_GEAR Mode: $TRANSCODE_MANAGER_MODE"
|
||||
echo "$ICON_RAM Ramdisk: ${RAMDISK_USED_GB}GB used / ${RAMDISK_AVAIL_GB}GB available"
|
||||
echo "$ICON_LINK Symlink: $TRANSCODE_LINK → $CURRENT_TARGET"
|
||||
echo "$ICON_LINK Flips: $FLIP_COUNT this hour (warn at $TRANSCODE_FLIP_WARN)"
|
||||
echo "$ICON_EMBY Streams: $TOTAL_SESSIONS total | Live TV: $LIVE_TV | Transcoding: $TRANSCODING | Direct: $DIRECT"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
else
|
||||
echo "Transcode manager — all healthy — $MY_ID ($(format_duration $(( END - START ))))"
|
||||
fi
|
||||
+321
@@ -0,0 +1,321 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ================================= Transcode Cleanup ==========================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Removes stale transcode files from both ramdisk and SSD fallback locations.
|
||||
# Called by transcode_management.sh (Orchestrators/) before transcode_manager.sh —
|
||||
# cleanup must run first so the manager sees real active-session usage, not
|
||||
# inflated usage from stale files. Must be fast and non-blocking.
|
||||
#
|
||||
# A file is eligible for deletion only if ALL conditions are true:
|
||||
# 1. Older than TRANSCODE_MAX_AGE minutes (mtime — last write time)
|
||||
# 2. Not currently open by any process (checked via lsof pre-built map)
|
||||
#
|
||||
# ==============================================================================================
|
||||
# DESIGN PRINCIPLES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# lsof Called Once, Not Per File
|
||||
# On a busy Live TV system the ramdisk contains thousands of HLS segment files.
|
||||
# Calling lsof once per file creates thousands of subprocess calls every 7 minutes.
|
||||
# lsof is called once per location to build a complete open-file map. All subsequent
|
||||
# checks are O(1) lookups against that map — thousands of files, one lsof call.
|
||||
#
|
||||
# No Session-Aware Cleanup
|
||||
# ffmpeg generates folder names independently of the media server API session IDs.
|
||||
# There is no reliable correlation between API session IDs and transcoding-temp
|
||||
# subfolder names. Attempting to correlate them would falsely treat active sessions
|
||||
# as ended. lsof is the correct check — if ffmpeg has a file open, it is active
|
||||
# regardless of folder naming or session state.
|
||||
#
|
||||
# transcoding-temp Is Never Deleted
|
||||
# If cleanup removes the empty transcoding-temp folder from the ramdisk, Emby
|
||||
# searches all accessible paths for an existing one, finds the SSD fallback version,
|
||||
# and routes all new sessions there until Emby restarts. The directory is excluded
|
||||
# from find by name — protected even when completely empty.
|
||||
#
|
||||
# Post-Cleanup Flip-Back
|
||||
# After removing stale files, checks whether ramdisk usage dropped below
|
||||
# RAMDISK_LOW_GB. If so — and symlink currently points at SSD — triggers a
|
||||
# flip back to ramdisk. This is the recovery path; the manager handles
|
||||
# the fill-up path.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Wait Lock
|
||||
# acquire_lock "wait" — waits if a previous cleanup run is still active rather
|
||||
# than exiting. The caller's 7-minute interval can overlap on a slow system.
|
||||
#
|
||||
# lsof Timeout
|
||||
# lsof call capped at 15 seconds per location — prevents blocking indefinitely
|
||||
# on a system with many open files.
|
||||
#
|
||||
# transcoding-temp Guard
|
||||
# `! -name "transcoding-temp"` in the find command — protected unconditionally.
|
||||
#
|
||||
# Silent by Default
|
||||
# Runs every 7 minutes — must not produce noise when healthy.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# host*.conf
|
||||
#
|
||||
# HOST*_RAMDISK_PATH / HOST*_TRANSCODE_SSD / HOST*_RAMDISK_LOW_GB
|
||||
# Aliased by detect_hosts() → RAMDISK_PATH / TRANSCODE_SSD / RAMDISK_LOW_GB.
|
||||
#
|
||||
# master.conf
|
||||
#
|
||||
# TRANSCODE_MAX_AGE
|
||||
# Minutes before an inactive transcode file is eligible for deletion. (default: 20)
|
||||
#
|
||||
# TRANSCODE_ORPHAN_AGE
|
||||
# Minutes for orphan folder detection. (default: 30)
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# transcode_cleanup.sh
|
||||
# Remove stale files from ramdisk and SSD. Check for flip-back opportunity.
|
||||
#
|
||||
# transcode_cleanup.sh --dry-run
|
||||
# Show which files would be deleted. No deletions, no flip.
|
||||
#
|
||||
# transcode_cleanup.sh --status
|
||||
# Show current file counts, ages, and open-file status per location.
|
||||
#
|
||||
# transcode_cleanup.sh --log
|
||||
# Verbose per-file output including age, open status, and deletion result.
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
STATE_FILE="${TRANSCODE_STATE_FILE:-${STATE_DIR:-/tmp}/transcode_state.db}"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
acquire_lock "wait"
|
||||
|
||||
# detect_hosts() sets MY_ID and aliases RAMDISK_PATH, TRANSCODE_SSD, RAMDISK_LOW_GB
|
||||
detect_hosts
|
||||
|
||||
# lsof availability check
|
||||
LSOF_AVAILABLE=false
|
||||
if command -v lsof >/dev/null 2>&1; then
|
||||
LSOF_AVAILABLE=true
|
||||
log "lsof available — active file check enabled"
|
||||
else
|
||||
warn "lsof not available — active file check skipped, all aged files eligible for deletion"
|
||||
fi
|
||||
|
||||
log "Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
log "$ICON_GEAR Config: max-age=${TRANSCODE_MAX_AGE}min orphan-age=${TRANSCODE_ORPHAN_AGE}min flip-back-below=${RAMDISK_LOW_GB}GB"
|
||||
log "$ICON_RAM Locations: ramdisk=$RAMDISK_PATH ssd=$TRANSCODE_SSD"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_RAM Ramdisk: $RAMDISK_PATH"
|
||||
echo "$ICON_DISK SSD fallback: $TRANSCODE_SSD"
|
||||
echo "$ICON_TRASH Max age: ${TRANSCODE_MAX_AGE} minutes"
|
||||
echo "$ICON_TRASH Orphan age: ${TRANSCODE_ORPHAN_AGE} minutes"
|
||||
echo "$ICON_RAM Flip at: ${RAMDISK_LOW_GB}GB"
|
||||
echo "$ICON_GEAR lsof check: $LSOF_AVAILABLE"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo ""
|
||||
|
||||
if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
|
||||
USAGE=$(df -BG "$RAMDISK_PATH" | awk 'NR==2 {print $3}')
|
||||
AVAIL=$(df -BG "$RAMDISK_PATH" | awk 'NR==2 {print $4}')
|
||||
echo " $ICON_RAM Ramdisk: mounted — $USAGE used / $AVAIL available ✅"
|
||||
else
|
||||
echo " $ICON_RAM Ramdisk: not mounted"
|
||||
fi
|
||||
|
||||
CURRENT_TARGET=$(grep "^current_target=" "$STATE_FILE" 2>/dev/null | cut -d= -f2)
|
||||
echo " $ICON_LINK Current target: ${CURRENT_TARGET:-unknown}"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be deleted"
|
||||
|
||||
# ==============================================================================================
|
||||
# ── CLEANUP FUNCTION ──────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Scans a location and removes eligible files.
|
||||
# Calls lsof ONCE per location — builds in-memory OPEN_FILES_MAP for O(1) lookup.
|
||||
# Returns via LOCATION_REMOVED, LOCATION_FREED, LOCATION_SKIPPED, LOCATION_ACTIVE
|
||||
|
||||
cleanup_location() {
|
||||
local location="$1" label="$2" max_age="$3"
|
||||
local files_removed=0 bytes_freed=0 files_skipped=0 files_active=0 files_streaming=0
|
||||
|
||||
if [[ ! -d "$location" ]]; then
|
||||
log "$label does not exist — skipping"
|
||||
LOCATION_REMOVED=0 LOCATION_FREED="0B" LOCATION_SKIPPED=0 LOCATION_ACTIVE=0 LOCATION_STREAMING=0
|
||||
return
|
||||
fi
|
||||
|
||||
local file_count
|
||||
file_count=$(find "$location" -type f 2>/dev/null | wc -l)
|
||||
log "$label: $file_count files to scan (age threshold: ${max_age}min)"
|
||||
|
||||
# Build in-memory open file map — O(1) lookup per file
|
||||
# One lsof call per location — never per file
|
||||
declare -A OPEN_FILES_MAP
|
||||
if [[ "$LSOF_AVAILABLE" == true ]]; then
|
||||
log "Building open file map for $label..."
|
||||
while IFS= read -r open_file; do
|
||||
[[ -n "$open_file" ]] && OPEN_FILES_MAP["$open_file"]=1
|
||||
done < <(timeout 15 lsof +D "$location" 2>/dev/null | awk 'NR>1 {print $9}' | sort -u)
|
||||
files_streaming=${#OPEN_FILES_MAP[@]}
|
||||
log "$files_streaming files currently open in $label"
|
||||
fi
|
||||
|
||||
# Process aged files
|
||||
while IFS= read -r file; do
|
||||
[[ -z "$file" ]] && continue
|
||||
|
||||
# O(1) open file check — in-memory map
|
||||
if [[ -n "${OPEN_FILES_MAP[$file]:-}" ]]; then
|
||||
(( files_active++ ))
|
||||
log "Skipping open file: $file"
|
||||
continue
|
||||
fi
|
||||
|
||||
local file_size
|
||||
file_size=$(stat -c%s "$file" 2>/dev/null || echo 0)
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
log "DRY RUN — would delete: $(basename "$file")"
|
||||
(( files_skipped++ ))
|
||||
else
|
||||
if rm -f "$file" 2>/dev/null; then
|
||||
(( files_removed++ ))
|
||||
bytes_freed=$(( bytes_freed + file_size ))
|
||||
log "Deleted: $file"
|
||||
else
|
||||
warn "Could not delete: $file"
|
||||
(( files_skipped++ ))
|
||||
fi
|
||||
fi
|
||||
|
||||
done < <(find "$location" -type f -mmin +"$max_age" 2>/dev/null)
|
||||
|
||||
# Remove empty directories — but NEVER remove transcoding-temp
|
||||
# transcoding-temp must always exist on ramdisk so Emby finds it there first
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
find "$location" -mindepth 1 -type d -empty \
|
||||
! -name "transcoding-temp" -delete 2>/dev/null
|
||||
fi
|
||||
|
||||
# Format bytes freed
|
||||
local freed_human
|
||||
if (( bytes_freed > 1073741824 )); then
|
||||
freed_human=$(awk "BEGIN {printf \"%.1fGB\", $bytes_freed / 1073741824}")
|
||||
elif (( bytes_freed > 1048576 )); then
|
||||
freed_human=$(awk "BEGIN {printf \"%.1fMB\", $bytes_freed / 1048576}")
|
||||
elif (( bytes_freed > 0 )); then
|
||||
freed_human="${bytes_freed}B"
|
||||
else
|
||||
freed_human="0B"
|
||||
fi
|
||||
|
||||
log "$label — removed $files_removed files ($freed_human freed) | streaming: $files_streaming | aged+open: $files_active | skipped: $files_skipped"
|
||||
|
||||
LOCATION_REMOVED=$files_removed
|
||||
LOCATION_FREED=$freed_human
|
||||
LOCATION_SKIPPED=$files_skipped
|
||||
LOCATION_ACTIVE=$files_active
|
||||
LOCATION_STREAMING=$files_streaming
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Transcode Cleanup ━━━
|
||||
# ==============================================================================================
|
||||
START=$(date +%s)
|
||||
TOTAL_REMOVED=0 TOTAL_SKIPPED=0 TOTAL_ACTIVE=0 TOTAL_STREAMING=0
|
||||
RAMDISK_FREED="0B" SSD_FREED="0B"
|
||||
|
||||
# Cleanup ramdisk
|
||||
if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
|
||||
cleanup_location "$RAMDISK_PATH" "Ramdisk" "$TRANSCODE_MAX_AGE"
|
||||
TOTAL_REMOVED=$(( TOTAL_REMOVED + LOCATION_REMOVED ))
|
||||
TOTAL_SKIPPED=$(( TOTAL_SKIPPED + LOCATION_SKIPPED ))
|
||||
TOTAL_ACTIVE=$(( TOTAL_ACTIVE + LOCATION_ACTIVE ))
|
||||
TOTAL_STREAMING=$(( TOTAL_STREAMING + LOCATION_STREAMING ))
|
||||
RAMDISK_FREED=$LOCATION_FREED
|
||||
else
|
||||
log "Ramdisk not mounted — skipping ramdisk cleanup"
|
||||
fi
|
||||
|
||||
# Cleanup SSD fallback
|
||||
if [[ -d "$TRANSCODE_SSD" ]]; then
|
||||
cleanup_location "$TRANSCODE_SSD" "SSD fallback" "$TRANSCODE_MAX_AGE"
|
||||
TOTAL_REMOVED=$(( TOTAL_REMOVED + LOCATION_REMOVED ))
|
||||
TOTAL_SKIPPED=$(( TOTAL_SKIPPED + LOCATION_SKIPPED ))
|
||||
TOTAL_ACTIVE=$(( TOTAL_ACTIVE + LOCATION_ACTIVE ))
|
||||
TOTAL_STREAMING=$(( TOTAL_STREAMING + LOCATION_STREAMING ))
|
||||
SSD_FREED=$LOCATION_FREED
|
||||
else
|
||||
log "SSD fallback not found — skipping SSD cleanup"
|
||||
fi
|
||||
|
||||
# Post-cleanup — check if ramdisk recovered enough to flip symlink back to ramdisk
|
||||
if [[ "$DRY_RUN" == false ]] && mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
|
||||
RAMDISK_USED_KB=$(df "$RAMDISK_PATH" --output=used 2>/dev/null | tail -1 | tr -d ' ')
|
||||
RAMDISK_USED_GB=$(awk "BEGIN {printf \"%.2f\", ${RAMDISK_USED_KB:-0} / 1048576}")
|
||||
LOW_RECOVERED=$(awk "BEGIN {print ($RAMDISK_USED_GB < $RAMDISK_LOW_GB) ? 1 : 0}")
|
||||
CURRENT_TARGET=$(grep "^current_target=" "$STATE_FILE" 2>/dev/null | cut -d= -f2)
|
||||
|
||||
if [[ "$LOW_RECOVERED" == "1" && "$CURRENT_TARGET" == "$TRANSCODE_SSD" ]]; then
|
||||
echo "Ramdisk has space after cleanup (${RAMDISK_USED_GB}GB < ${RAMDISK_LOW_GB}GB) — triggering manager to flip back"
|
||||
bash "$SCRIPT_DIR/transcode_manager.sh" --no-log
|
||||
fi
|
||||
fi
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo "━━━━━ $ICON_SUMMARY TRANSCODE CLEANUP SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_RAM Ramdisk freed: $RAMDISK_FREED"
|
||||
echo "$ICON_DISK SSD freed: $SSD_FREED"
|
||||
echo "$ICON_TRASH Removed: $TOTAL_REMOVED files"
|
||||
echo "$ICON_RUNNING Streaming: $TOTAL_STREAMING files (open by Emby — untouched)"
|
||||
echo "$ICON_SHIELD Protected: $TOTAL_ACTIVE aged files saved by open-file check"
|
||||
echo "$ICON_TRASH Skipped: $TOTAL_SKIPPED files"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no files deleted"
|
||||
else
|
||||
echo "$ICON_DONE Status: done ✅"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
+329
@@ -0,0 +1,329 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ================================= Transcode Cleanup ==========================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Removes stale transcode files from both ramdisk and SSD fallback locations.
|
||||
# Called by transcode_management.sh (Orchestrators/) before transcode_manager.sh —
|
||||
# cleanup must run first so the manager sees real active-session usage, not
|
||||
# inflated usage from stale files. Must be fast and non-blocking.
|
||||
#
|
||||
# A file is eligible for deletion only if ALL conditions are true:
|
||||
# 1. Older than TRANSCODE_MAX_AGE minutes (mtime — last write time)
|
||||
# 2. Not currently open by any process (checked via lsof pre-built map)
|
||||
#
|
||||
# ==============================================================================================
|
||||
# DESIGN PRINCIPLES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# lsof Called Once, Not Per File
|
||||
# On a busy Live TV system the ramdisk contains thousands of HLS segment files.
|
||||
# Calling lsof once per file creates thousands of subprocess calls every 7 minutes.
|
||||
# lsof is called once per location to build a complete open-file map. All subsequent
|
||||
# checks are O(1) lookups against that map — thousands of files, one lsof call.
|
||||
#
|
||||
# No Session-Aware Cleanup
|
||||
# ffmpeg generates folder names independently of the media server API session IDs.
|
||||
# There is no reliable correlation between API session IDs and transcoding-temp
|
||||
# subfolder names. Attempting to correlate them would falsely treat active sessions
|
||||
# as ended. lsof is the correct check — if ffmpeg has a file open, it is active
|
||||
# regardless of folder naming or session state.
|
||||
#
|
||||
# transcoding-temp Is Never Deleted
|
||||
# If cleanup removes the empty transcoding-temp folder from the ramdisk, Emby
|
||||
# searches all accessible paths for an existing one, finds the SSD fallback version,
|
||||
# and routes all new sessions there until Emby restarts. The directory is excluded
|
||||
# from find by name — protected even when completely empty.
|
||||
#
|
||||
# Post-Cleanup Flip-Back
|
||||
# After removing stale files, checks whether ramdisk usage dropped below
|
||||
# RAMDISK_LOW_GB. If so — and symlink currently points at SSD — triggers a
|
||||
# flip back to ramdisk. This is the recovery path; the manager handles
|
||||
# the fill-up path.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Wait Lock
|
||||
# acquire_lock "wait" — waits if a previous cleanup run is still active rather
|
||||
# than exiting. The caller's 7-minute interval can overlap on a slow system.
|
||||
#
|
||||
# lsof Timeout
|
||||
# lsof call capped at 15 seconds per location — prevents blocking indefinitely
|
||||
# on a system with many open files.
|
||||
#
|
||||
# transcoding-temp Guard
|
||||
# `! -name "transcoding-temp"` in the find command — protected unconditionally.
|
||||
#
|
||||
# Silent by Default
|
||||
# Runs every 7 minutes — must not produce noise when healthy.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# host*.conf
|
||||
#
|
||||
# HOST*_RAMDISK_PATH / HOST*_TRANSCODE_SSD / HOST*_RAMDISK_LOW_GB
|
||||
# Aliased by detect_hosts() → RAMDISK_PATH / TRANSCODE_SSD / RAMDISK_LOW_GB.
|
||||
#
|
||||
# master.conf
|
||||
#
|
||||
# TRANSCODE_MAX_AGE
|
||||
# Minutes before an inactive transcode file is eligible for deletion. (default: 20)
|
||||
#
|
||||
# TRANSCODE_ORPHAN_AGE
|
||||
# Minutes for orphan folder detection. (default: 30)
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# transcode_cleanup.sh
|
||||
# Remove stale files from ramdisk and SSD. Check for flip-back opportunity.
|
||||
#
|
||||
# transcode_cleanup.sh --dry-run
|
||||
# Show which files would be deleted. No deletions, no flip.
|
||||
#
|
||||
# transcode_cleanup.sh --status
|
||||
# Show current file counts, ages, and open-file status per location.
|
||||
#
|
||||
# transcode_cleanup.sh --log
|
||||
# Verbose per-file output including age, open status, and deletion result.
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
STATE_FILE="${TRANSCODE_STATE_FILE:-${STATE_DIR:-/tmp}/transcode_state.db}"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
acquire_lock "wait"
|
||||
|
||||
# detect_hosts() sets MY_ID and aliases RAMDISK_PATH, TRANSCODE_SSD, RAMDISK_LOW_GB
|
||||
detect_hosts
|
||||
|
||||
# lsof availability check
|
||||
LSOF_AVAILABLE=false
|
||||
if command -v lsof >/dev/null 2>&1; then
|
||||
LSOF_AVAILABLE=true
|
||||
log "lsof available — active file check enabled"
|
||||
else
|
||||
warn "lsof not available — active file check skipped, all aged files eligible for deletion"
|
||||
fi
|
||||
|
||||
log "Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
log "$ICON_GEAR Config: max-age=${TRANSCODE_MAX_AGE}min orphan-age=${TRANSCODE_ORPHAN_AGE}min flip-back-below=${RAMDISK_LOW_GB}GB"
|
||||
log "$ICON_RAM Locations: ramdisk=$RAMDISK_PATH ssd=$TRANSCODE_SSD"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_RAM Ramdisk: $RAMDISK_PATH"
|
||||
echo "$ICON_DISK SSD fallback: $TRANSCODE_SSD"
|
||||
echo "$ICON_TRASH Max age: ${TRANSCODE_MAX_AGE} minutes"
|
||||
echo "$ICON_TRASH Orphan age: ${TRANSCODE_ORPHAN_AGE} minutes"
|
||||
echo "$ICON_RAM Flip at: ${RAMDISK_LOW_GB}GB"
|
||||
echo "$ICON_GEAR lsof check: $LSOF_AVAILABLE"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo ""
|
||||
|
||||
if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
|
||||
USAGE=$(df -BG "$RAMDISK_PATH" | awk 'NR==2 {print $3}')
|
||||
AVAIL=$(df -BG "$RAMDISK_PATH" | awk 'NR==2 {print $4}')
|
||||
echo " $ICON_RAM Ramdisk: mounted — $USAGE used / $AVAIL available ✅"
|
||||
else
|
||||
echo " $ICON_RAM Ramdisk: not mounted"
|
||||
fi
|
||||
|
||||
CURRENT_TARGET=$(grep "^current_target=" "$STATE_FILE" 2>/dev/null | cut -d= -f2)
|
||||
echo " $ICON_LINK Current target: ${CURRENT_TARGET:-unknown}"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be deleted"
|
||||
|
||||
# ==============================================================================================
|
||||
# ── CLEANUP FUNCTION ──────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Scans a location and removes eligible files.
|
||||
# Calls lsof ONCE per location — builds in-memory OPEN_FILES_MAP for O(1) lookup.
|
||||
# Returns via LOCATION_REMOVED, LOCATION_FREED, LOCATION_SKIPPED, LOCATION_ACTIVE
|
||||
|
||||
cleanup_location() {
|
||||
local location="$1" label="$2" max_age="$3"
|
||||
local files_removed=0 bytes_freed=0 files_skipped=0 files_active=0 files_streaming=0 files_too_young=0
|
||||
|
||||
if [[ ! -d "$location" ]]; then
|
||||
log "$label does not exist — skipping"
|
||||
LOCATION_REMOVED=0 LOCATION_FREED="0B" LOCATION_SKIPPED=0 LOCATION_ACTIVE=0 LOCATION_STREAMING=0 LOCATION_TOO_YOUNG=0
|
||||
return
|
||||
fi
|
||||
|
||||
local file_count eligible_count
|
||||
file_count=$(find "$location" -type f 2>/dev/null | wc -l)
|
||||
eligible_count=$(find "$location" -type f -mmin +"$max_age" 2>/dev/null | wc -l)
|
||||
files_too_young=$(( file_count - eligible_count ))
|
||||
log "$label: $file_count files ($eligible_count eligible, $files_too_young < ${max_age}min)"
|
||||
|
||||
# Build in-memory open file map — O(1) lookup per file
|
||||
# One lsof call per location — never per file
|
||||
# Note: HLS/remux segments (Live TV, Direct Stream) are written atomically and immediately
|
||||
# closed — lsof will not detect them. The age check is the effective guard for those files.
|
||||
declare -A OPEN_FILES_MAP
|
||||
if [[ "$LSOF_AVAILABLE" == true ]]; then
|
||||
log "Building open file map for $label..."
|
||||
while IFS= read -r open_file; do
|
||||
[[ -n "$open_file" ]] && OPEN_FILES_MAP["$open_file"]=1
|
||||
done < <(timeout 15 lsof +D "$location" 2>/dev/null | awk 'NR>1 {print $9}' | sort -u)
|
||||
files_streaming=${#OPEN_FILES_MAP[@]}
|
||||
log "$files_streaming files currently open in $label"
|
||||
fi
|
||||
|
||||
# Process aged files
|
||||
while IFS= read -r file; do
|
||||
[[ -z "$file" ]] && continue
|
||||
|
||||
# O(1) open file check — in-memory map
|
||||
if [[ -n "${OPEN_FILES_MAP[$file]:-}" ]]; then
|
||||
(( files_active++ ))
|
||||
log "Skipping open file: $file"
|
||||
continue
|
||||
fi
|
||||
|
||||
local file_size
|
||||
file_size=$(stat -c%s "$file" 2>/dev/null || echo 0)
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
log "DRY RUN — would delete: $(basename "$file")"
|
||||
(( files_skipped++ ))
|
||||
else
|
||||
if rm -f "$file" 2>/dev/null; then
|
||||
(( files_removed++ ))
|
||||
bytes_freed=$(( bytes_freed + file_size ))
|
||||
log "Deleted: $file"
|
||||
else
|
||||
warn "Could not delete: $file"
|
||||
(( files_skipped++ ))
|
||||
fi
|
||||
fi
|
||||
|
||||
done < <(find "$location" -type f -mmin +"$max_age" 2>/dev/null)
|
||||
|
||||
# Remove empty directories — but NEVER remove transcoding-temp
|
||||
# transcoding-temp must always exist on ramdisk so Emby finds it there first
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
find "$location" -mindepth 1 -type d -empty \
|
||||
! -name "transcoding-temp" -delete 2>/dev/null
|
||||
fi
|
||||
|
||||
# Format bytes freed
|
||||
local freed_human
|
||||
if (( bytes_freed > 1073741824 )); then
|
||||
freed_human=$(awk "BEGIN {printf \"%.1fGB\", $bytes_freed / 1073741824}")
|
||||
elif (( bytes_freed > 1048576 )); then
|
||||
freed_human=$(awk "BEGIN {printf \"%.1fMB\", $bytes_freed / 1048576}")
|
||||
elif (( bytes_freed > 0 )); then
|
||||
freed_human="${bytes_freed}B"
|
||||
else
|
||||
freed_human="0B"
|
||||
fi
|
||||
|
||||
log "$label — removed $files_removed ($freed_human) | active(fresh): $files_too_young | streaming(lsof): $files_streaming | protected(old+open): $files_active | skipped: $files_skipped"
|
||||
|
||||
LOCATION_REMOVED=$files_removed
|
||||
LOCATION_FREED=$freed_human
|
||||
LOCATION_SKIPPED=$files_skipped
|
||||
LOCATION_ACTIVE=$files_active
|
||||
LOCATION_STREAMING=$files_streaming
|
||||
LOCATION_TOO_YOUNG=$files_too_young
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Transcode Cleanup ━━━
|
||||
# ==============================================================================================
|
||||
START=$(date +%s)
|
||||
TOTAL_REMOVED=0 TOTAL_SKIPPED=0 TOTAL_ACTIVE=0 TOTAL_STREAMING=0 TOTAL_TOO_YOUNG=0
|
||||
RAMDISK_FREED="0B" SSD_FREED="0B"
|
||||
|
||||
# Cleanup ramdisk
|
||||
if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
|
||||
cleanup_location "$RAMDISK_PATH" "Ramdisk" "$TRANSCODE_MAX_AGE"
|
||||
TOTAL_REMOVED=$(( TOTAL_REMOVED + LOCATION_REMOVED ))
|
||||
TOTAL_SKIPPED=$(( TOTAL_SKIPPED + LOCATION_SKIPPED ))
|
||||
TOTAL_ACTIVE=$(( TOTAL_ACTIVE + LOCATION_ACTIVE ))
|
||||
TOTAL_STREAMING=$(( TOTAL_STREAMING + LOCATION_STREAMING ))
|
||||
TOTAL_TOO_YOUNG=$(( TOTAL_TOO_YOUNG + LOCATION_TOO_YOUNG ))
|
||||
RAMDISK_FREED=$LOCATION_FREED
|
||||
else
|
||||
log "Ramdisk not mounted — skipping ramdisk cleanup"
|
||||
fi
|
||||
|
||||
# Cleanup SSD fallback
|
||||
if [[ -d "$TRANSCODE_SSD" ]]; then
|
||||
cleanup_location "$TRANSCODE_SSD" "SSD fallback" "$TRANSCODE_MAX_AGE"
|
||||
TOTAL_REMOVED=$(( TOTAL_REMOVED + LOCATION_REMOVED ))
|
||||
TOTAL_SKIPPED=$(( TOTAL_SKIPPED + LOCATION_SKIPPED ))
|
||||
TOTAL_ACTIVE=$(( TOTAL_ACTIVE + LOCATION_ACTIVE ))
|
||||
TOTAL_STREAMING=$(( TOTAL_STREAMING + LOCATION_STREAMING ))
|
||||
TOTAL_TOO_YOUNG=$(( TOTAL_TOO_YOUNG + LOCATION_TOO_YOUNG ))
|
||||
SSD_FREED=$LOCATION_FREED
|
||||
else
|
||||
log "SSD fallback not found — skipping SSD cleanup"
|
||||
fi
|
||||
|
||||
# Post-cleanup — check if ramdisk recovered enough to flip symlink back to ramdisk
|
||||
if [[ "$DRY_RUN" == false ]] && mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
|
||||
RAMDISK_USED_KB=$(df "$RAMDISK_PATH" --output=used 2>/dev/null | tail -1 | tr -d ' ')
|
||||
RAMDISK_USED_GB=$(awk "BEGIN {printf \"%.2f\", ${RAMDISK_USED_KB:-0} / 1048576}")
|
||||
LOW_RECOVERED=$(awk "BEGIN {print ($RAMDISK_USED_GB < $RAMDISK_LOW_GB) ? 1 : 0}")
|
||||
CURRENT_TARGET=$(grep "^current_target=" "$STATE_FILE" 2>/dev/null | cut -d= -f2)
|
||||
|
||||
if [[ "$LOW_RECOVERED" == "1" && "$CURRENT_TARGET" == "$TRANSCODE_SSD" ]]; then
|
||||
echo "Ramdisk has space after cleanup (${RAMDISK_USED_GB}GB < ${RAMDISK_LOW_GB}GB) — triggering manager to flip back"
|
||||
bash "$SCRIPT_DIR/transcode_manager.sh" --no-log
|
||||
fi
|
||||
fi
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo "━━━━━ $ICON_SUMMARY TRANSCODE CLEANUP SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_RAM Ramdisk freed: $RAMDISK_FREED"
|
||||
echo "$ICON_DISK SSD freed: $SSD_FREED"
|
||||
echo "$ICON_TRASH Removed: $TOTAL_REMOVED files"
|
||||
echo "$ICON_RUNNING Active: $TOTAL_TOO_YOUNG files (< ${TRANSCODE_MAX_AGE}min old — Live TV / Direct Stream segments)"
|
||||
echo "$ICON_RUNNING Streaming: $TOTAL_STREAMING files (open file handle — long-running transcode)"
|
||||
echo "$ICON_SHIELD Protected: $TOTAL_ACTIVE aged files saved by open-file check"
|
||||
echo "$ICON_TRASH Skipped: $TOTAL_SKIPPED files"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no files deleted"
|
||||
else
|
||||
echo "$ICON_DONE Status: done ✅"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
+650
@@ -0,0 +1,650 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ================================= Downloaders Reset ==========================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Maintenance reset for all download clients on this server. Clears accumulated
|
||||
# state that download clients generate but never clean up themselves — stuck
|
||||
# searches, dead transfers, failed imports, stale queue entries, completed history.
|
||||
#
|
||||
# Called every 30 minutes by critical_sync_maintenance.sh via
|
||||
# CRITICAL_MAINTENANCE_SCRIPTS. Can also be run manually for ad hoc cleanup.
|
||||
# If a downloader is not configured for this host, that section skips cleanly.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# slskd
|
||||
# Stuck searches — clears Completed/Errored searches left by Soularr crashes
|
||||
# prevents 409 Conflict on next Soularr startup
|
||||
# Dead transfers — removes completed/errored/aborted transfer records per user
|
||||
# prevents Soularr 404 loop when polling a user whose transfer is gone
|
||||
# NEVER removes InProgress or Queued transfers
|
||||
# Failed imports — purges albums Soularr downloaded but Lidarr rejected
|
||||
# Soularr moves these to failed_imports/ and never cleans them up
|
||||
#
|
||||
# SABnzbd
|
||||
# Completed history — removes completed download records older than DOWNLOADER_RETENTION_DAYS
|
||||
# Failed history — removes failed download records older than DOWNLOADER_RETENTION_DAYS
|
||||
# Stalled queue — removes Paused or Stuck queue items no longer progressing
|
||||
# active downloading items are never touched
|
||||
#
|
||||
# qBittorrent
|
||||
# Age failsafe — removes torrents older than QBIT_FAILSAFE_MIN_DAYS
|
||||
# deleteFiles=false — removes from qBit, leaves files for arrs to manage
|
||||
# optional ratio requirement via QBIT_FAILSAFE_MIN_RATIO
|
||||
#
|
||||
# ==============================================================================================
|
||||
# DESIGN PRINCIPLES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Never Interrupt Active Downloads
|
||||
# Each downloader section checks for active state before any removal. slskd
|
||||
# skips users with InProgress or Queued transfers. SABnzbd only removes items
|
||||
# past the retention threshold. qBittorrent applies minimum age and optional
|
||||
# ratio requirements. In-progress work is never touched.
|
||||
#
|
||||
# Graceful Skip on Unavailability
|
||||
# If a downloader's URL is empty or the service is unreachable, that section
|
||||
# skips cleanly with a log message. The script never exits fatally on a single
|
||||
# unreachable downloader — the others still run.
|
||||
#
|
||||
# Host-Aware Configuration
|
||||
# detect_hosts() aliases all HOST*_SLSKD_*, HOST*_SABNZBD_*, HOST*_QBIT_* vars
|
||||
# to their unprefixed names. Downloaders not configured for this host are absent
|
||||
# from the aliased vars and skip automatically.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Active Transfer Protection
|
||||
# slskd: skips users with InProgress or Queued transfers before any removal.
|
||||
# SABnzbd: age threshold enforced before deletion.
|
||||
# qBittorrent: minimum age plus optional ratio gate before failsafe removal.
|
||||
#
|
||||
# Reachability Check
|
||||
# Each section validates its downloader URL before API calls. Missing or
|
||||
# unreachable downloaders skip without affecting other sections.
|
||||
#
|
||||
# Host Detection
|
||||
# detect_hosts() identifies which server is running the script and aliases
|
||||
# all HOST*_SLSKD_*, HOST*_SABNZBD_*, and HOST*_QBIT_* vars to the correct
|
||||
# host's values. Downloaders not configured on this host skip automatically.
|
||||
#
|
||||
# Lock Acquisition
|
||||
# acquire_lock "wait" — waits for previous run to finish since this runs every
|
||||
# 30 minutes and prior execution may still be completing.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# host*.conf
|
||||
#
|
||||
# HOST*_SLSKD_URL / HOST*_SLSKD_API_KEY / HOST*_SLSKD_FAILED_IMPORTS_DIR
|
||||
# slskd connection and failed imports path. Aliased by detect_hosts()
|
||||
#
|
||||
# HOST*_SABNZBD_URL / HOST*_SABNZBD_API_KEY
|
||||
# SABnzbd connection details. Aliased by detect_hosts()
|
||||
#
|
||||
# HOST*_QBIT_URL / HOST*_QBIT_USERNAME / HOST*_QBIT_PASSWORD
|
||||
# qBittorrent connection details. Aliased by detect_hosts()
|
||||
#
|
||||
# master.conf
|
||||
#
|
||||
# DOWNLOADER_RETENTION_DAYS
|
||||
# Days before SABnzbd history entries (completed or failed) are removed
|
||||
#
|
||||
# QBIT_FAILSAFE_MIN_DAYS
|
||||
# Minimum torrent age in days before failsafe removal is considered
|
||||
#
|
||||
# QBIT_FAILSAFE_MIN_RATIO
|
||||
# Minimum seeding ratio required alongside age gate (0 = age only)
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# downloaders_reset.sh
|
||||
# Run maintenance reset for all configured download clients
|
||||
#
|
||||
# downloaders_reset.sh --dry-run
|
||||
# Preview what would be removed without making any changes
|
||||
#
|
||||
# downloaders_reset.sh --status
|
||||
# Show configured downloaders, current queue depths, and retention settings
|
||||
#
|
||||
# downloaders_reset.sh --log
|
||||
# Verbose per-client per-item output
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Lock first — wait mode since this runs every 30min and previous may still be finishing
|
||||
acquire_lock "wait"
|
||||
|
||||
# detect_hosts() sets MY_ID and aliases all HOST*_SLSKD_*, HOST*_SABNZBD_*, HOST*_QBIT_* vars
|
||||
detect_hosts
|
||||
|
||||
START_TIME=$(date +%s)
|
||||
CUTOFF=$(( $(date +%s) - (DOWNLOADER_RETENTION_DAYS * 86400) ))
|
||||
TOTAL_PASS=0
|
||||
TOTAL_FAIL=0
|
||||
|
||||
log "$ICON_GEAR Config: retention=${DOWNLOADER_RETENTION_DAYS}d qbit-age=${QBIT_FAILSAFE_MIN_DAYS}d qbit-ratio=${QBIT_FAILSAFE_MIN_RATIO}"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_GEAR slskd: ${SLSKD_URL:-not configured}"
|
||||
echo "$ICON_GEAR SABnzbd: ${SABNZBD_URL:-not configured}"
|
||||
echo "$ICON_GEAR qBittorrent: ${QBIT_URL:-not configured}"
|
||||
echo "$ICON_TIME Retention: ${DOWNLOADER_RETENTION_DAYS} days"
|
||||
echo "$ICON_GEAR qBit age: ${QBIT_FAILSAFE_MIN_DAYS} days"
|
||||
echo "$ICON_GEAR qBit ratio: ${QBIT_FAILSAFE_MIN_RATIO} (0=age only)"
|
||||
echo "$ICON_NOTIFY Notify: unRAID=${NOTIFY_UNRAID:-false} Discord=$([[ -n "${MY_DISCORD_WEBHOOK:-}" ]] && echo enabled || echo disabled)"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
|
||||
# Log which downloaders are active on this host
|
||||
if [[ -z "$SLSKD_URL" ]] && [[ -z "$SABNZBD_URL" ]] && [[ -z "$QBIT_URL" ]]; then
|
||||
warn "No downloaders configured for $MY_ID — nothing to reset"
|
||||
exit 0
|
||||
fi
|
||||
[[ -n "$SLSKD_URL" ]] && log "slskd active on $MY_ID"
|
||||
[[ -n "$SABNZBD_URL" ]] && log "SABnzbd active on $MY_ID"
|
||||
[[ -n "$QBIT_URL" ]] && log "qBittorrent active on $MY_ID"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ slskd — Connection Check ━━━
|
||||
# ==============================================================================================
|
||||
# slskd's internal watchdog doesn't always recover from disconnection. Check before
|
||||
# running API-dependent sections; attempt reconnect if down.
|
||||
|
||||
SLSKD_CONNECTED=false
|
||||
|
||||
if [[ -n "$SLSKD_URL" ]] && [[ -n "$SLSKD_API_KEY" ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC slskd — Connection Check ━━━"
|
||||
|
||||
_slskd_is_connected() {
|
||||
local state
|
||||
state=$(curl -sf --max-time 10 \
|
||||
-H "X-Api-Key: $SLSKD_API_KEY" \
|
||||
"$SLSKD_URL/api/v0/application" 2>/dev/null | \
|
||||
jq -r '.server.isConnected // false' 2>/dev/null)
|
||||
[[ "$state" == "true" ]]
|
||||
}
|
||||
|
||||
if _slskd_is_connected; then
|
||||
log "slskd connected to Soulseek ✅"
|
||||
SLSKD_CONNECTED=true
|
||||
else
|
||||
warn "slskd disconnected — triggering reconnect"
|
||||
curl -sf --max-time 10 -X PUT \
|
||||
-H "X-Api-Key: $SLSKD_API_KEY" \
|
||||
-H "Content-Type: application/json" \
|
||||
"$SLSKD_URL/api/v0/server" \
|
||||
-d '{"address":"server.slsknet.org","port":2242}' \
|
||||
>/dev/null 2>&1
|
||||
|
||||
_ELAPSED=0
|
||||
while [[ "$_ELAPSED" -lt 60 ]]; do
|
||||
sleep 10
|
||||
_ELAPSED=$(( _ELAPSED + 10 ))
|
||||
if _slskd_is_connected; then
|
||||
log "slskd reconnected after ${_ELAPSED}s ✅"
|
||||
SLSKD_CONNECTED=true
|
||||
break
|
||||
fi
|
||||
log " waiting... (${_ELAPSED}s / 60s)"
|
||||
done
|
||||
|
||||
[[ "$SLSKD_CONNECTED" != true ]] && \
|
||||
warn "slskd still disconnected after 60s — skipping API-dependent sections"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ slskd — Stuck Searches ━━━
|
||||
# ==============================================================================================
|
||||
# Clears searches in Completed/Errored state left by Soularr crashes.
|
||||
# Prevents 409 Conflict error on next Soularr startup when it tries to
|
||||
# create a search with the same ID that already exists in a terminal state.
|
||||
|
||||
if [[ -n "$SLSKD_URL" ]] && [[ -n "$SLSKD_API_KEY" ]] && [[ "$SLSKD_CONNECTED" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━ 🔍 slskd — Stuck Searches ━━━"
|
||||
|
||||
SEARCHES=$(curl -sf --max-time 10 -X GET "$SLSKD_URL/api/v0/searches" \
|
||||
-H "X-Api-Key: $SLSKD_API_KEY" 2>/dev/null)
|
||||
|
||||
if [[ -z "$SEARCHES" ]]; then
|
||||
warn "slskd not reachable — skipping searches"
|
||||
else
|
||||
IDS=$(echo "$SEARCHES" | tr '{' '\n' | \
|
||||
grep '"isComplete":true' | grep '"searchText":' | \
|
||||
grep -o '"id":"[^"]*"' | sed 's/"id":"//;s/"//')
|
||||
COUNT=$(echo "$IDS" | grep -c . 2>/dev/null || echo 0)
|
||||
COUNT="${COUNT//[^0-9]/}"; COUNT="${COUNT:-0}"
|
||||
|
||||
if [[ "$COUNT" -eq 0 ]]; then
|
||||
success "No stuck searches found ✅"
|
||||
else
|
||||
log "Found $COUNT stuck search(es)"
|
||||
SUCCESS=0; FAIL=0
|
||||
while IFS= read -r ID; do
|
||||
[[ -z "$ID" ]] && continue
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would delete search: $ID"
|
||||
((SUCCESS++))
|
||||
continue
|
||||
fi
|
||||
RESULT=$(curl -sf --max-time 10 -o /dev/null -w "%{http_code}" -X DELETE \
|
||||
"$SLSKD_URL/api/v0/searches/$ID" \
|
||||
-H "X-Api-Key: $SLSKD_API_KEY")
|
||||
if [[ "$RESULT" == "200" || "$RESULT" == "204" ]]; then
|
||||
log "$ICON_TRASH Cleared search: $ID"
|
||||
((SUCCESS++))
|
||||
else
|
||||
error "Failed: $ID (HTTP $RESULT)"
|
||||
((FAIL++))
|
||||
fi
|
||||
done <<< "$IDS"
|
||||
success "Searches: $SUCCESS cleared, $FAIL failed"
|
||||
(( TOTAL_FAIL += FAIL ))
|
||||
(( TOTAL_PASS += SUCCESS ))
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ slskd — Dead Transfer Records ━━━
|
||||
# ==============================================================================================
|
||||
# Removes completed/errored/aborted transfer records per user.
|
||||
# Prevents Soularr 404 loop when polling a user whose transfer no longer exists.
|
||||
# Safety: NEVER removes transfers that are InProgress or Queued — active downloads protected.
|
||||
|
||||
if [[ -n "$SLSKD_URL" ]] && [[ -n "$SLSKD_API_KEY" ]] && [[ "$SLSKD_CONNECTED" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━ 🔍 slskd — Dead Transfer Records ━━━"
|
||||
|
||||
TRANSFERS=$(curl -sf --max-time 10 -X GET "$SLSKD_URL/api/v0/transfers/downloads" \
|
||||
-H "X-Api-Key: $SLSKD_API_KEY" 2>/dev/null)
|
||||
|
||||
if [[ -z "$TRANSFERS" ]]; then
|
||||
warn "slskd not reachable — skipping transfers"
|
||||
else
|
||||
USERNAMES=$(echo "$TRANSFERS" | grep -o '"username":"[^"]*"' | \
|
||||
sed 's/"username":"//;s/"//' | sort -u)
|
||||
|
||||
if [[ -z "$USERNAMES" ]]; then
|
||||
success "No transfer records found ✅"
|
||||
else
|
||||
USER_COUNT=$(echo "$USERNAMES" | grep -c . 2>/dev/null || echo 0)
|
||||
log "Found $USER_COUNT user(s) with transfer records"
|
||||
SUCCESS=0; SKIPPED=0; FAIL=0
|
||||
while IFS= read -r USER; do
|
||||
[[ -z "$USER" ]] && continue
|
||||
|
||||
USER_DATA=$(curl -sf --max-time 10 \
|
||||
"$SLSKD_URL/api/v0/transfers/downloads/$USER" \
|
||||
-H "X-Api-Key: $SLSKD_API_KEY" 2>/dev/null)
|
||||
|
||||
# Skip users with any active or queued transfers — never interrupt downloads
|
||||
ACTIVE=$(echo "$USER_DATA" | grep -c '"state":"InProgress"\|"state":"Queued"')
|
||||
if [[ "${ACTIVE:-0}" -gt 0 ]]; then
|
||||
log "$ICON_SKIP Skipping $USER — has active/queued transfer(s)"
|
||||
((SKIPPED++))
|
||||
continue
|
||||
fi
|
||||
|
||||
# Extract IDs of terminal-state file transfers
|
||||
# Split at { so each file object lands on its own line, then grep for state
|
||||
FILE_IDS=$(echo "$USER_DATA" | tr '{' '\n' | \
|
||||
grep '"state":"Completed"\|"state":"Errored"\|"state":"Aborted"\|"state":"Cancelled"' | \
|
||||
grep -o '"id":"[^"]*"' | sed 's/"id":"//;s/"//')
|
||||
|
||||
if [[ -z "$FILE_IDS" ]]; then
|
||||
log "$ICON_SKIP Skipping $USER — no terminal-state transfers"
|
||||
((SKIPPED++))
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
F_COUNT=$(echo "$FILE_IDS" | grep -c .)
|
||||
warn "DRY RUN — would clear $F_COUNT transfer(s) for: $USER"
|
||||
((SUCCESS++))
|
||||
continue
|
||||
fi
|
||||
|
||||
F_SUCCESS=0; F_FAIL=0
|
||||
while IFS= read -r FILE_ID; do
|
||||
[[ -z "$FILE_ID" ]] && continue
|
||||
RESULT=$(curl -sf --max-time 10 -o /dev/null -w "%{http_code}" -X DELETE \
|
||||
"$SLSKD_URL/api/v0/transfers/downloads/$USER/$FILE_ID" \
|
||||
-H "X-Api-Key: $SLSKD_API_KEY")
|
||||
if [[ "$RESULT" == "200" || "$RESULT" == "204" ]]; then
|
||||
((F_SUCCESS++))
|
||||
else
|
||||
((F_FAIL++))
|
||||
fi
|
||||
done <<< "$FILE_IDS"
|
||||
|
||||
log "$ICON_TRASH Cleared $F_SUCCESS transfer(s) for: $USER ($F_FAIL failed)"
|
||||
((SUCCESS += F_SUCCESS))
|
||||
((FAIL += F_FAIL))
|
||||
done <<< "$USERNAMES"
|
||||
success "Transfers: $SUCCESS cleared, $SKIPPED skipped (active/empty), $FAIL failed"
|
||||
(( TOTAL_FAIL += FAIL ))
|
||||
(( TOTAL_PASS += SUCCESS ))
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ slskd — Purge Expired Failed Imports ━━━
|
||||
# ==============================================================================================
|
||||
# Removes albums Soularr downloaded but Lidarr rejected.
|
||||
# Soularr moves rejected albums to failed_imports/ and never cleans them up.
|
||||
# Purges directories older than DOWNLOADER_RETENTION_DAYS to prevent unbounded growth.
|
||||
|
||||
if [[ -n "$SLSKD_FAILED_IMPORTS_DIR" ]]; then
|
||||
echo ""
|
||||
echo "━━━ 🔍 slskd — Failed Imports (older than ${DOWNLOADER_RETENTION_DAYS} days) ━━━"
|
||||
|
||||
if [[ ! -d "$SLSKD_FAILED_IMPORTS_DIR" ]]; then
|
||||
warn "Directory not found: $SLSKD_FAILED_IMPORTS_DIR — skipping"
|
||||
else
|
||||
OLD_IMPORTS=$(find "$SLSKD_FAILED_IMPORTS_DIR" \
|
||||
-mindepth 1 -maxdepth 1 -mtime +"${DOWNLOADER_RETENTION_DAYS}")
|
||||
IMPORT_COUNT=$(echo "$OLD_IMPORTS" | grep -c . 2>/dev/null || echo 0)
|
||||
IMPORT_COUNT="${IMPORT_COUNT//[^0-9]/}"; IMPORT_COUNT="${IMPORT_COUNT:-0}"
|
||||
|
||||
if [[ "$IMPORT_COUNT" -eq 0 ]]; then
|
||||
success "No expired failed imports found ✅"
|
||||
else
|
||||
log "Found $IMPORT_COUNT expired failed import(s)"
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would delete:"
|
||||
echo "$OLD_IMPORTS"
|
||||
else
|
||||
find "$SLSKD_FAILED_IMPORTS_DIR" \
|
||||
-mindepth 1 -maxdepth 1 -mtime +"${DOWNLOADER_RETENTION_DAYS}" \
|
||||
-exec rm -rf {} \;
|
||||
success "$ICON_TRASH Purged $IMPORT_COUNT expired failed import(s)"
|
||||
(( TOTAL_PASS += IMPORT_COUNT ))
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ SABnzbd — Clear Completed History ━━━
|
||||
# ==============================================================================================
|
||||
# Removes completed download history older than DOWNLOADER_RETENTION_DAYS.
|
||||
# Keeps recent history for reference — only purges what's past the retention window.
|
||||
|
||||
if [[ -n "$SABNZBD_URL" ]] && [[ -n "$SABNZBD_API_KEY" ]]; then
|
||||
echo ""
|
||||
echo "━━━ 🔍 SABnzbd — Completed History (older than ${DOWNLOADER_RETENTION_DAYS} days) ━━━"
|
||||
|
||||
HISTORY=$(curl -sf --max-time 15 \
|
||||
"$SABNZBD_URL/api?mode=history&output=json&limit=1000&apikey=$SABNZBD_API_KEY" 2>/dev/null)
|
||||
|
||||
if [[ -z "$HISTORY" ]]; then
|
||||
warn "SABnzbd not reachable — skipping completed history"
|
||||
else
|
||||
COMPLETED_IDS=$(echo "$HISTORY" | grep -o '"nzo_id":"[^"]*"' | \
|
||||
sed 's/"nzo_id":"//;s/"//')
|
||||
|
||||
if [[ -z "$COMPLETED_IDS" ]]; then
|
||||
success "No completed history found ✅"
|
||||
else
|
||||
HIST_TOTAL=$(echo "$COMPLETED_IDS" | grep -c . 2>/dev/null || echo 0)
|
||||
log "Found $HIST_TOTAL completed history entries"
|
||||
DELETED=0; SKIPPED=0
|
||||
while IFS= read -r NZO_ID; do
|
||||
[[ -z "$NZO_ID" ]] && continue
|
||||
JOB_TIME=$(echo "$HISTORY" | grep -A5 "$NZO_ID" | \
|
||||
grep -o '"completed":[0-9]*' | grep -o '[0-9]*' | head -1)
|
||||
[[ -z "$JOB_TIME" ]] && continue
|
||||
[[ "$JOB_TIME" -gt "$CUTOFF" ]] && ((SKIPPED++)) && continue
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would delete completed job: $NZO_ID"
|
||||
((DELETED++))
|
||||
else
|
||||
curl -sf --max-time 10 \
|
||||
"$SABNZBD_URL/api?mode=history&name=delete&value=$NZO_ID&apikey=$SABNZBD_API_KEY" \
|
||||
>/dev/null
|
||||
log "$ICON_TRASH Deleted: $NZO_ID"
|
||||
((DELETED++))
|
||||
fi
|
||||
done <<< "$COMPLETED_IDS"
|
||||
success "Completed: $DELETED deleted, $SKIPPED within retention"
|
||||
(( TOTAL_PASS += DELETED ))
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ SABnzbd — Clear Failed History ━━━
|
||||
# ==============================================================================================
|
||||
# Removes failed download history older than DOWNLOADER_RETENTION_DAYS.
|
||||
# Failed history is kept briefly for diagnosis but purged after the retention window.
|
||||
|
||||
if [[ -n "$SABNZBD_URL" ]] && [[ -n "$SABNZBD_API_KEY" ]]; then
|
||||
echo ""
|
||||
echo "━━━ 🔍 SABnzbd — Failed History (older than ${DOWNLOADER_RETENTION_DAYS} days) ━━━"
|
||||
|
||||
FAILED_HIST=$(curl -sf --max-time 15 \
|
||||
"$SABNZBD_URL/api?mode=history&output=json&limit=1000&failed_only=1&apikey=$SABNZBD_API_KEY" 2>/dev/null)
|
||||
|
||||
if [[ -z "$FAILED_HIST" ]]; then
|
||||
warn "SABnzbd not reachable — skipping failed history"
|
||||
else
|
||||
FAILED_IDS=$(echo "$FAILED_HIST" | grep -o '"nzo_id":"[^"]*"' | \
|
||||
sed 's/"nzo_id":"//;s/"//')
|
||||
|
||||
if [[ -z "$FAILED_IDS" ]]; then
|
||||
success "No failed history found ✅"
|
||||
else
|
||||
FAILED_TOTAL=$(echo "$FAILED_IDS" | grep -c . 2>/dev/null || echo 0)
|
||||
log "Found $FAILED_TOTAL failed history entries"
|
||||
DELETED=0; SKIPPED=0
|
||||
while IFS= read -r NZO_ID; do
|
||||
[[ -z "$NZO_ID" ]] && continue
|
||||
JOB_TIME=$(echo "$FAILED_HIST" | grep -A5 "$NZO_ID" | \
|
||||
grep -o '"completed":[0-9]*' | grep -o '[0-9]*' | head -1)
|
||||
[[ -z "$JOB_TIME" ]] && continue
|
||||
[[ "$JOB_TIME" -gt "$CUTOFF" ]] && ((SKIPPED++)) && continue
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would delete failed job: $NZO_ID"
|
||||
((DELETED++))
|
||||
else
|
||||
curl -sf --max-time 10 \
|
||||
"$SABNZBD_URL/api?mode=history&name=delete&value=$NZO_ID&apikey=$SABNZBD_API_KEY" \
|
||||
>/dev/null
|
||||
log "$ICON_TRASH Deleted: $NZO_ID"
|
||||
((DELETED++))
|
||||
fi
|
||||
done <<< "$FAILED_IDS"
|
||||
success "Failed: $DELETED deleted, $SKIPPED within retention"
|
||||
(( TOTAL_PASS += DELETED ))
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ SABnzbd — Remove Stalled Queue Items ━━━
|
||||
# ==============================================================================================
|
||||
# Removes queue items in Paused or Stuck state that are no longer progressing.
|
||||
# Active downloading items (Downloading, Grabbing) are never touched.
|
||||
# Paused items may be intentional pauses — but in an automated environment
|
||||
# a Paused item sitting in the queue indefinitely is effectively stalled.
|
||||
|
||||
if [[ -n "$SABNZBD_URL" ]] && [[ -n "$SABNZBD_API_KEY" ]]; then
|
||||
echo ""
|
||||
echo "━━━ 🔍 SABnzbd — Stalled Queue Items ━━━"
|
||||
|
||||
QUEUE=$(curl -sf --max-time 10 \
|
||||
"$SABNZBD_URL/api?mode=queue&output=json&apikey=$SABNZBD_API_KEY" 2>/dev/null)
|
||||
|
||||
if [[ -z "$QUEUE" ]]; then
|
||||
warn "SABnzbd not reachable — skipping queue"
|
||||
else
|
||||
STALLED_IDS=$(echo "$QUEUE" | grep -o '"nzo_id":"[^"]*"' | \
|
||||
sed 's/"nzo_id":"//;s/"//')
|
||||
|
||||
if [[ -z "$STALLED_IDS" ]]; then
|
||||
success "No stalled queue items found ✅"
|
||||
else
|
||||
QUEUE_TOTAL=$(echo "$STALLED_IDS" | grep -c . 2>/dev/null || echo 0)
|
||||
log "Found $QUEUE_TOTAL queue item(s) — checking status"
|
||||
DELETED=0; SKIPPED=0
|
||||
while IFS= read -r NZO_ID; do
|
||||
[[ -z "$NZO_ID" ]] && continue
|
||||
STATUS=$(echo "$QUEUE" | grep -A10 "$NZO_ID" | \
|
||||
grep -o '"status":"[^"]*"' | sed 's/"status":"//;s/"//')
|
||||
# Only remove Paused or Stuck items — Downloading/Grabbing are active
|
||||
if [[ "$STATUS" != "Paused" ]] && [[ "$STATUS" != "Stuck" ]]; then
|
||||
((SKIPPED++))
|
||||
continue
|
||||
fi
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would remove stalled item: $NZO_ID ($STATUS)"
|
||||
((DELETED++))
|
||||
else
|
||||
curl -sf --max-time 10 \
|
||||
"$SABNZBD_URL/api?mode=queue&name=delete&value=$NZO_ID&apikey=$SABNZBD_API_KEY" \
|
||||
>/dev/null
|
||||
log "$ICON_TRASH Removed stalled ($STATUS): $NZO_ID"
|
||||
((DELETED++))
|
||||
fi
|
||||
done <<< "$STALLED_IDS"
|
||||
success "Queue: $DELETED removed, $SKIPPED active (skipped)"
|
||||
(( TOTAL_PASS += DELETED ))
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ qBittorrent — Age Failsafe Cleanup ━━━
|
||||
# ==============================================================================================
|
||||
# Last-chance cleanup for torrents that have been sitting in qBit past their useful life.
|
||||
# deleteFiles=false — removes the torrent record from qBit but leaves files on disk.
|
||||
# Radarr/Sonarr manage actual files independently — this only cleans up the qBit entry.
|
||||
#
|
||||
# Safety checks before deletion:
|
||||
# Age must exceed QBIT_FAILSAFE_MIN_DAYS
|
||||
# Ratio must meet QBIT_FAILSAFE_MIN_RATIO (0 = age only, no ratio requirement)
|
||||
|
||||
if [[ -n "$QBIT_URL" ]] && [[ -n "$QBIT_USERNAME" ]]; then
|
||||
echo ""
|
||||
echo "━━━ 🔍 qBittorrent — Failsafe (older than ${QBIT_FAILSAFE_MIN_DAYS} days) ━━━"
|
||||
[[ "$QBIT_FAILSAFE_MIN_RATIO" != "0" ]] && \
|
||||
log "Ratio requirement: >= ${QBIT_FAILSAFE_MIN_RATIO}"
|
||||
|
||||
QBIT_COOKIE=$(curl -sf --max-time 10 -c - \
|
||||
"$QBIT_URL/api/v2/auth/login" \
|
||||
--data "username=$QBIT_USERNAME&password=$QBIT_PASSWORD" 2>/dev/null | \
|
||||
grep SID | awk '{print "SID="$NF}')
|
||||
|
||||
if [[ -z "$QBIT_COOKIE" ]]; then
|
||||
error "Failed to authenticate with qBittorrent — check QBIT_USERNAME/PASSWORD"
|
||||
notify "qBittorrent auth failed on $(hostname) — check credentials in host*.conf" "Downloaders Reset" "warning"
|
||||
((TOTAL_FAIL++))
|
||||
else
|
||||
TORRENTS=$(curl -sf --max-time 15 \
|
||||
"$QBIT_URL/api/v2/torrents/info" \
|
||||
-H "Cookie: $QBIT_COOKIE" 2>/dev/null)
|
||||
|
||||
NOW=$(date +%s)
|
||||
TORRENT_TOTAL=$(echo "$TORRENTS" | tr '}' '\n' | grep -c '"hash"' 2>/dev/null || echo 0)
|
||||
log "Found $TORRENT_TOTAL torrent(s) — applying age/ratio filter"
|
||||
DELETED=0; SKIPPED=0
|
||||
|
||||
while read -r TORRENT; do
|
||||
[[ -z "$TORRENT" ]] && continue
|
||||
HASH=$(echo "$TORRENT" | grep -o '"hash":"[^"]*"' | sed 's/"hash":"//;s/"//')
|
||||
NAME=$(echo "$TORRENT" | grep -o '"name":"[^"]*"' | sed 's/"name":"//;s/"//')
|
||||
ADDED=$(echo "$TORRENT" | grep -o '"added_on":[0-9]*' | grep -o '[0-9]*')
|
||||
RATIO=$(echo "$TORRENT" | grep -o '"ratio":[0-9.]*' | grep -o '[0-9.]*')
|
||||
[[ -z "$HASH" || -z "$ADDED" ]] && continue
|
||||
|
||||
AGE_DAYS=$(( (NOW - ADDED) / 86400 ))
|
||||
|
||||
# Age check — must be old enough
|
||||
[[ "$AGE_DAYS" -lt "$QBIT_FAILSAFE_MIN_DAYS" ]] && ((SKIPPED++)) && continue
|
||||
|
||||
# Ratio check — if configured
|
||||
if [[ "$QBIT_FAILSAFE_MIN_RATIO" != "0" ]]; then
|
||||
RATIO_INT="${RATIO%.*}"
|
||||
MIN_RATIO_INT="${QBIT_FAILSAFE_MIN_RATIO%.*}"
|
||||
[[ "$RATIO_INT" -lt "$MIN_RATIO_INT" ]] && ((SKIPPED++)) && continue
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would delete: $NAME (${AGE_DAYS}d old, ratio: $RATIO)"
|
||||
((DELETED++))
|
||||
else
|
||||
curl -sf --max-time 10 -X POST \
|
||||
"$QBIT_URL/api/v2/torrents/delete" \
|
||||
-H "Cookie: $QBIT_COOKIE" \
|
||||
--data "hashes=$HASH&deleteFiles=false" >/dev/null
|
||||
log "$ICON_TRASH Deleted: $NAME (${AGE_DAYS}d old, ratio: $RATIO)"
|
||||
((DELETED++))
|
||||
fi
|
||||
done < <(echo "$TORRENTS" | tr '}' '\n')
|
||||
|
||||
success "qBittorrent: $DELETED deleted, $SKIPPED skipped (under threshold)"
|
||||
(( TOTAL_PASS += DELETED ))
|
||||
fi
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY DOWNLOADERS RESET SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( $(date +%s) - START_TIME )))"
|
||||
echo "$ICON_SUCCESS Actions: $TOTAL_PASS"
|
||||
echo "$ICON_ERROR Failures: $TOTAL_FAIL"
|
||||
echo ""
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "$ICON_WARN Status: DRY RUN — no changes made"
|
||||
elif [[ "$TOTAL_FAIL" -gt 0 ]]; then
|
||||
echo "$ICON_ERROR Status: $TOTAL_FAIL failure(s) — check logs"
|
||||
notify "Downloaders reset completed with failures on $(hostname)" "Downloaders Reset" "warning"
|
||||
exit 1
|
||||
else
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS DONE"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
+650
@@ -0,0 +1,650 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ================================= Downloaders Reset ==========================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Maintenance reset for all download clients on this server. Clears accumulated
|
||||
# state that download clients generate but never clean up themselves — stuck
|
||||
# searches, dead transfers, failed imports, stale queue entries, completed history.
|
||||
#
|
||||
# Called every 30 minutes by critical_sync_maintenance.sh via
|
||||
# CRITICAL_MAINTENANCE_SCRIPTS. Can also be run manually for ad hoc cleanup.
|
||||
# If a downloader is not configured for this host, that section skips cleanly.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# slskd
|
||||
# Stuck searches — clears Completed/Errored searches left by Soularr crashes
|
||||
# prevents 409 Conflict on next Soularr startup
|
||||
# Dead transfers — removes completed/errored/aborted transfer records per user
|
||||
# prevents Soularr 404 loop when polling a user whose transfer is gone
|
||||
# NEVER removes InProgress or Queued transfers
|
||||
# Failed imports — purges albums Soularr downloaded but Lidarr rejected
|
||||
# Soularr moves these to failed_imports/ and never cleans them up
|
||||
#
|
||||
# SABnzbd
|
||||
# Completed history — removes completed download records older than DOWNLOADER_RETENTION_DAYS
|
||||
# Failed history — removes failed download records older than DOWNLOADER_RETENTION_DAYS
|
||||
# Stalled queue — removes Paused or Stuck queue items no longer progressing
|
||||
# active downloading items are never touched
|
||||
#
|
||||
# qBittorrent
|
||||
# Age failsafe — removes torrents older than QBIT_FAILSAFE_MIN_DAYS
|
||||
# deleteFiles=false — removes from qBit, leaves files for arrs to manage
|
||||
# optional ratio requirement via QBIT_FAILSAFE_MIN_RATIO
|
||||
#
|
||||
# ==============================================================================================
|
||||
# DESIGN PRINCIPLES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Never Interrupt Active Downloads
|
||||
# Each downloader section checks for active state before any removal. slskd
|
||||
# skips users with InProgress or Queued transfers. SABnzbd only removes items
|
||||
# past the retention threshold. qBittorrent applies minimum age and optional
|
||||
# ratio requirements. In-progress work is never touched.
|
||||
#
|
||||
# Graceful Skip on Unavailability
|
||||
# If a downloader's URL is empty or the service is unreachable, that section
|
||||
# skips cleanly with a log message. The script never exits fatally on a single
|
||||
# unreachable downloader — the others still run.
|
||||
#
|
||||
# Host-Aware Configuration
|
||||
# detect_hosts() aliases all HOST*_SLSKD_*, HOST*_SABNZBD_*, HOST*_QBIT_* vars
|
||||
# to their unprefixed names. Downloaders not configured for this host are absent
|
||||
# from the aliased vars and skip automatically.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Active Transfer Protection
|
||||
# slskd: skips users with InProgress or Queued transfers before any removal.
|
||||
# SABnzbd: age threshold enforced before deletion.
|
||||
# qBittorrent: minimum age plus optional ratio gate before failsafe removal.
|
||||
#
|
||||
# Reachability Check
|
||||
# Each section validates its downloader URL before API calls. Missing or
|
||||
# unreachable downloaders skip without affecting other sections.
|
||||
#
|
||||
# Host Detection
|
||||
# detect_hosts() identifies which server is running the script and aliases
|
||||
# all HOST*_SLSKD_*, HOST*_SABNZBD_*, and HOST*_QBIT_* vars to the correct
|
||||
# host's values. Downloaders not configured on this host skip automatically.
|
||||
#
|
||||
# Lock Acquisition
|
||||
# acquire_lock "wait" — waits for previous run to finish since this runs every
|
||||
# 30 minutes and prior execution may still be completing.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# host*.conf
|
||||
#
|
||||
# HOST*_SLSKD_URL / HOST*_SLSKD_API_KEY / HOST*_SLSKD_FAILED_IMPORTS_DIR
|
||||
# slskd connection and failed imports path. Aliased by detect_hosts()
|
||||
#
|
||||
# HOST*_SABNZBD_URL / HOST*_SABNZBD_API_KEY
|
||||
# SABnzbd connection details. Aliased by detect_hosts()
|
||||
#
|
||||
# HOST*_QBIT_URL / HOST*_QBIT_USERNAME / HOST*_QBIT_PASSWORD
|
||||
# qBittorrent connection details. Aliased by detect_hosts()
|
||||
#
|
||||
# master.conf
|
||||
#
|
||||
# DOWNLOADER_RETENTION_DAYS
|
||||
# Days before SABnzbd history entries (completed or failed) are removed
|
||||
#
|
||||
# QBIT_FAILSAFE_MIN_DAYS
|
||||
# Minimum torrent age in days before failsafe removal is considered
|
||||
#
|
||||
# QBIT_FAILSAFE_MIN_RATIO
|
||||
# Minimum seeding ratio required alongside age gate (0 = age only)
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# downloaders_reset.sh
|
||||
# Run maintenance reset for all configured download clients
|
||||
#
|
||||
# downloaders_reset.sh --dry-run
|
||||
# Preview what would be removed without making any changes
|
||||
#
|
||||
# downloaders_reset.sh --status
|
||||
# Show configured downloaders, current queue depths, and retention settings
|
||||
#
|
||||
# downloaders_reset.sh --log
|
||||
# Verbose per-client per-item output
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Lock first — wait mode since this runs every 30min and previous may still be finishing
|
||||
acquire_lock "wait"
|
||||
|
||||
# detect_hosts() sets MY_ID and aliases all HOST*_SLSKD_*, HOST*_SABNZBD_*, HOST*_QBIT_* vars
|
||||
detect_hosts
|
||||
|
||||
START_TIME=$(date +%s)
|
||||
CUTOFF=$(( $(date +%s) - (DOWNLOADER_RETENTION_DAYS * 86400) ))
|
||||
TOTAL_PASS=0
|
||||
TOTAL_FAIL=0
|
||||
|
||||
log "$ICON_GEAR Config: retention=${DOWNLOADER_RETENTION_DAYS}d qbit-age=${QBIT_FAILSAFE_MIN_DAYS}d qbit-ratio=${QBIT_FAILSAFE_MIN_RATIO}"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_GEAR slskd: ${SLSKD_URL:-not configured}"
|
||||
echo "$ICON_GEAR SABnzbd: ${SABNZBD_URL:-not configured}"
|
||||
echo "$ICON_GEAR qBittorrent: ${QBIT_URL:-not configured}"
|
||||
echo "$ICON_TIME Retention: ${DOWNLOADER_RETENTION_DAYS} days"
|
||||
echo "$ICON_GEAR qBit age: ${QBIT_FAILSAFE_MIN_DAYS} days"
|
||||
echo "$ICON_GEAR qBit ratio: ${QBIT_FAILSAFE_MIN_RATIO} (0=age only)"
|
||||
echo "$ICON_NOTIFY Notify: unRAID=${NOTIFY_UNRAID:-false} Discord=$([[ -n "${MY_DISCORD_WEBHOOK:-}" ]] && echo enabled || echo disabled)"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
|
||||
# Log which downloaders are active on this host
|
||||
if [[ -z "$SLSKD_URL" ]] && [[ -z "$SABNZBD_URL" ]] && [[ -z "$QBIT_URL" ]]; then
|
||||
warn "No downloaders configured for $MY_ID — nothing to reset"
|
||||
exit 0
|
||||
fi
|
||||
[[ -n "$SLSKD_URL" ]] && log "slskd active on $MY_ID"
|
||||
[[ -n "$SABNZBD_URL" ]] && log "SABnzbd active on $MY_ID"
|
||||
[[ -n "$QBIT_URL" ]] && log "qBittorrent active on $MY_ID"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ slskd — Connection Check ━━━
|
||||
# ==============================================================================================
|
||||
# slskd's internal watchdog doesn't always recover from disconnection. Check before
|
||||
# running API-dependent sections; attempt reconnect if down.
|
||||
|
||||
SLSKD_CONNECTED=false
|
||||
|
||||
if [[ -n "$SLSKD_URL" ]] && [[ -n "$SLSKD_API_KEY" ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC slskd — Connection Check ━━━"
|
||||
|
||||
_slskd_is_connected() {
|
||||
local state
|
||||
state=$(curl -sf --max-time 10 \
|
||||
-H "X-Api-Key: $SLSKD_API_KEY" \
|
||||
"$SLSKD_URL/api/v0/application" 2>/dev/null | \
|
||||
jq -r '.server.isConnected // false' 2>/dev/null)
|
||||
[[ "$state" == "true" ]]
|
||||
}
|
||||
|
||||
if _slskd_is_connected; then
|
||||
echo "$ICON_DONE [OK] slskd connected to Soulseek ✅"
|
||||
SLSKD_CONNECTED=true
|
||||
else
|
||||
warn "slskd disconnected — triggering reconnect"
|
||||
curl -sf --max-time 10 -X PUT \
|
||||
-H "X-Api-Key: $SLSKD_API_KEY" \
|
||||
-H "Content-Type: application/json" \
|
||||
"$SLSKD_URL/api/v0/server" \
|
||||
-d '{"address":"server.slsknet.org","port":2242}' \
|
||||
>/dev/null 2>&1
|
||||
|
||||
_ELAPSED=0
|
||||
while [[ "$_ELAPSED" -lt 60 ]]; do
|
||||
sleep 10
|
||||
_ELAPSED=$(( _ELAPSED + 10 ))
|
||||
if _slskd_is_connected; then
|
||||
log "slskd reconnected after ${_ELAPSED}s ✅"
|
||||
SLSKD_CONNECTED=true
|
||||
break
|
||||
fi
|
||||
log " waiting... (${_ELAPSED}s / 60s)"
|
||||
done
|
||||
|
||||
[[ "$SLSKD_CONNECTED" != true ]] && \
|
||||
warn "slskd still disconnected after 60s — skipping API-dependent sections"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ slskd — Stuck Searches ━━━
|
||||
# ==============================================================================================
|
||||
# Clears searches in Completed/Errored state left by Soularr crashes.
|
||||
# Prevents 409 Conflict error on next Soularr startup when it tries to
|
||||
# create a search with the same ID that already exists in a terminal state.
|
||||
|
||||
if [[ -n "$SLSKD_URL" ]] && [[ -n "$SLSKD_API_KEY" ]] && [[ "$SLSKD_CONNECTED" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━ 🔍 slskd — Stuck Searches ━━━"
|
||||
|
||||
SEARCHES=$(curl -sf --max-time 10 -X GET "$SLSKD_URL/api/v0/searches" \
|
||||
-H "X-Api-Key: $SLSKD_API_KEY" 2>/dev/null)
|
||||
|
||||
if [[ -z "$SEARCHES" ]]; then
|
||||
warn "slskd not reachable — skipping searches"
|
||||
else
|
||||
IDS=$(echo "$SEARCHES" | tr '{' '\n' | \
|
||||
grep '"isComplete":true' | grep '"searchText":' | \
|
||||
grep -o '"id":"[^"]*"' | sed 's/"id":"//;s/"//')
|
||||
COUNT=$(echo "$IDS" | grep -c . 2>/dev/null || echo 0)
|
||||
COUNT="${COUNT//[^0-9]/}"; COUNT="${COUNT:-0}"
|
||||
|
||||
if [[ "$COUNT" -eq 0 ]]; then
|
||||
success "No stuck searches found ✅"
|
||||
else
|
||||
log "Found $COUNT stuck search(es)"
|
||||
SUCCESS=0; FAIL=0
|
||||
while IFS= read -r ID; do
|
||||
[[ -z "$ID" ]] && continue
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would delete search: $ID"
|
||||
((SUCCESS++))
|
||||
continue
|
||||
fi
|
||||
RESULT=$(curl -sf --max-time 10 -o /dev/null -w "%{http_code}" -X DELETE \
|
||||
"$SLSKD_URL/api/v0/searches/$ID" \
|
||||
-H "X-Api-Key: $SLSKD_API_KEY")
|
||||
if [[ "$RESULT" == "200" || "$RESULT" == "204" ]]; then
|
||||
log "$ICON_TRASH Cleared search: $ID"
|
||||
((SUCCESS++))
|
||||
else
|
||||
error "Failed: $ID (HTTP $RESULT)"
|
||||
((FAIL++))
|
||||
fi
|
||||
done <<< "$IDS"
|
||||
success "Searches: $SUCCESS cleared, $FAIL failed"
|
||||
(( TOTAL_FAIL += FAIL ))
|
||||
(( TOTAL_PASS += SUCCESS ))
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ slskd — Dead Transfer Records ━━━
|
||||
# ==============================================================================================
|
||||
# Removes completed/errored/aborted transfer records per user.
|
||||
# Prevents Soularr 404 loop when polling a user whose transfer no longer exists.
|
||||
# Safety: NEVER removes transfers that are InProgress or Queued — active downloads protected.
|
||||
|
||||
if [[ -n "$SLSKD_URL" ]] && [[ -n "$SLSKD_API_KEY" ]] && [[ "$SLSKD_CONNECTED" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━ 🔍 slskd — Dead Transfer Records ━━━"
|
||||
|
||||
TRANSFERS=$(curl -sf --max-time 10 -X GET "$SLSKD_URL/api/v0/transfers/downloads" \
|
||||
-H "X-Api-Key: $SLSKD_API_KEY" 2>/dev/null)
|
||||
|
||||
if [[ -z "$TRANSFERS" ]]; then
|
||||
warn "slskd not reachable — skipping transfers"
|
||||
else
|
||||
USERNAMES=$(echo "$TRANSFERS" | grep -o '"username":"[^"]*"' | \
|
||||
sed 's/"username":"//;s/"//' | sort -u)
|
||||
|
||||
if [[ -z "$USERNAMES" ]]; then
|
||||
success "No transfer records found ✅"
|
||||
else
|
||||
USER_COUNT=$(echo "$USERNAMES" | grep -c . 2>/dev/null || echo 0)
|
||||
log "Found $USER_COUNT user(s) with transfer records"
|
||||
SUCCESS=0; SKIPPED=0; FAIL=0
|
||||
while IFS= read -r USER; do
|
||||
[[ -z "$USER" ]] && continue
|
||||
|
||||
USER_DATA=$(curl -sf --max-time 10 \
|
||||
"$SLSKD_URL/api/v0/transfers/downloads/$USER" \
|
||||
-H "X-Api-Key: $SLSKD_API_KEY" 2>/dev/null)
|
||||
|
||||
# Skip users with any active or queued transfers — never interrupt downloads
|
||||
ACTIVE=$(echo "$USER_DATA" | grep -c '"state":"InProgress"\|"state":"Queued"')
|
||||
if [[ "${ACTIVE:-0}" -gt 0 ]]; then
|
||||
log "$ICON_SKIP Skipping $USER — has active/queued transfer(s)"
|
||||
((SKIPPED++))
|
||||
continue
|
||||
fi
|
||||
|
||||
# Extract IDs of terminal-state file transfers
|
||||
# Split at { so each file object lands on its own line, then grep for state
|
||||
FILE_IDS=$(echo "$USER_DATA" | tr '{' '\n' | \
|
||||
grep '"state":"Completed"\|"state":"Errored"\|"state":"Aborted"\|"state":"Cancelled"' | \
|
||||
grep -o '"id":"[^"]*"' | sed 's/"id":"//;s/"//')
|
||||
|
||||
if [[ -z "$FILE_IDS" ]]; then
|
||||
log "$ICON_SKIP Skipping $USER — no terminal-state transfers"
|
||||
((SKIPPED++))
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
F_COUNT=$(echo "$FILE_IDS" | grep -c .)
|
||||
warn "DRY RUN — would clear $F_COUNT transfer(s) for: $USER"
|
||||
((SUCCESS++))
|
||||
continue
|
||||
fi
|
||||
|
||||
F_SUCCESS=0; F_FAIL=0
|
||||
while IFS= read -r FILE_ID; do
|
||||
[[ -z "$FILE_ID" ]] && continue
|
||||
RESULT=$(curl -sf --max-time 10 -o /dev/null -w "%{http_code}" -X DELETE \
|
||||
"$SLSKD_URL/api/v0/transfers/downloads/$USER/$FILE_ID" \
|
||||
-H "X-Api-Key: $SLSKD_API_KEY")
|
||||
if [[ "$RESULT" == "200" || "$RESULT" == "204" ]]; then
|
||||
((F_SUCCESS++))
|
||||
else
|
||||
((F_FAIL++))
|
||||
fi
|
||||
done <<< "$FILE_IDS"
|
||||
|
||||
log "$ICON_TRASH Cleared $F_SUCCESS transfer(s) for: $USER ($F_FAIL failed)"
|
||||
((SUCCESS += F_SUCCESS))
|
||||
((FAIL += F_FAIL))
|
||||
done <<< "$USERNAMES"
|
||||
success "Transfers: $SUCCESS cleared, $SKIPPED skipped (active/empty), $FAIL failed"
|
||||
(( TOTAL_FAIL += FAIL ))
|
||||
(( TOTAL_PASS += SUCCESS ))
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ slskd — Purge Expired Failed Imports ━━━
|
||||
# ==============================================================================================
|
||||
# Removes albums Soularr downloaded but Lidarr rejected.
|
||||
# Soularr moves rejected albums to failed_imports/ and never cleans them up.
|
||||
# Purges directories older than DOWNLOADER_RETENTION_DAYS to prevent unbounded growth.
|
||||
|
||||
if [[ -n "$SLSKD_FAILED_IMPORTS_DIR" ]]; then
|
||||
echo ""
|
||||
echo "━━━ 🔍 slskd — Failed Imports (older than ${DOWNLOADER_RETENTION_DAYS} days) ━━━"
|
||||
|
||||
if [[ ! -d "$SLSKD_FAILED_IMPORTS_DIR" ]]; then
|
||||
warn "Directory not found: $SLSKD_FAILED_IMPORTS_DIR — skipping"
|
||||
else
|
||||
OLD_IMPORTS=$(find "$SLSKD_FAILED_IMPORTS_DIR" \
|
||||
-mindepth 1 -maxdepth 1 -mtime +"${DOWNLOADER_RETENTION_DAYS}")
|
||||
IMPORT_COUNT=$(echo "$OLD_IMPORTS" | grep -c . 2>/dev/null || echo 0)
|
||||
IMPORT_COUNT="${IMPORT_COUNT//[^0-9]/}"; IMPORT_COUNT="${IMPORT_COUNT:-0}"
|
||||
|
||||
if [[ "$IMPORT_COUNT" -eq 0 ]]; then
|
||||
success "No expired failed imports found ✅"
|
||||
else
|
||||
log "Found $IMPORT_COUNT expired failed import(s)"
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would delete:"
|
||||
echo "$OLD_IMPORTS"
|
||||
else
|
||||
find "$SLSKD_FAILED_IMPORTS_DIR" \
|
||||
-mindepth 1 -maxdepth 1 -mtime +"${DOWNLOADER_RETENTION_DAYS}" \
|
||||
-exec rm -rf {} \;
|
||||
success "$ICON_TRASH Purged $IMPORT_COUNT expired failed import(s)"
|
||||
(( TOTAL_PASS += IMPORT_COUNT ))
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ SABnzbd — Clear Completed History ━━━
|
||||
# ==============================================================================================
|
||||
# Removes completed download history older than DOWNLOADER_RETENTION_DAYS.
|
||||
# Keeps recent history for reference — only purges what's past the retention window.
|
||||
|
||||
if [[ -n "$SABNZBD_URL" ]] && [[ -n "$SABNZBD_API_KEY" ]]; then
|
||||
echo ""
|
||||
echo "━━━ 🔍 SABnzbd — Completed History (older than ${DOWNLOADER_RETENTION_DAYS} days) ━━━"
|
||||
|
||||
HISTORY=$(curl -sf --max-time 15 \
|
||||
"$SABNZBD_URL/api?mode=history&output=json&limit=1000&apikey=$SABNZBD_API_KEY" 2>/dev/null)
|
||||
|
||||
if [[ -z "$HISTORY" ]]; then
|
||||
warn "SABnzbd not reachable — skipping completed history"
|
||||
else
|
||||
COMPLETED_IDS=$(echo "$HISTORY" | grep -o '"nzo_id":"[^"]*"' | \
|
||||
sed 's/"nzo_id":"//;s/"//')
|
||||
|
||||
if [[ -z "$COMPLETED_IDS" ]]; then
|
||||
success "No completed history found ✅"
|
||||
else
|
||||
HIST_TOTAL=$(echo "$COMPLETED_IDS" | grep -c . 2>/dev/null || echo 0)
|
||||
log "Found $HIST_TOTAL completed history entries"
|
||||
DELETED=0; SKIPPED=0
|
||||
while IFS= read -r NZO_ID; do
|
||||
[[ -z "$NZO_ID" ]] && continue
|
||||
JOB_TIME=$(echo "$HISTORY" | grep -A5 "$NZO_ID" | \
|
||||
grep -o '"completed":[0-9]*' | grep -o '[0-9]*' | head -1)
|
||||
[[ -z "$JOB_TIME" ]] && continue
|
||||
[[ "$JOB_TIME" -gt "$CUTOFF" ]] && ((SKIPPED++)) && continue
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would delete completed job: $NZO_ID"
|
||||
((DELETED++))
|
||||
else
|
||||
curl -sf --max-time 10 \
|
||||
"$SABNZBD_URL/api?mode=history&name=delete&value=$NZO_ID&apikey=$SABNZBD_API_KEY" \
|
||||
>/dev/null
|
||||
log "$ICON_TRASH Deleted: $NZO_ID"
|
||||
((DELETED++))
|
||||
fi
|
||||
done <<< "$COMPLETED_IDS"
|
||||
success "Completed: $DELETED deleted, $SKIPPED within retention"
|
||||
(( TOTAL_PASS += DELETED ))
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ SABnzbd — Clear Failed History ━━━
|
||||
# ==============================================================================================
|
||||
# Removes failed download history older than DOWNLOADER_RETENTION_DAYS.
|
||||
# Failed history is kept briefly for diagnosis but purged after the retention window.
|
||||
|
||||
if [[ -n "$SABNZBD_URL" ]] && [[ -n "$SABNZBD_API_KEY" ]]; then
|
||||
echo ""
|
||||
echo "━━━ 🔍 SABnzbd — Failed History (older than ${DOWNLOADER_RETENTION_DAYS} days) ━━━"
|
||||
|
||||
FAILED_HIST=$(curl -sf --max-time 15 \
|
||||
"$SABNZBD_URL/api?mode=history&output=json&limit=1000&failed_only=1&apikey=$SABNZBD_API_KEY" 2>/dev/null)
|
||||
|
||||
if [[ -z "$FAILED_HIST" ]]; then
|
||||
warn "SABnzbd not reachable — skipping failed history"
|
||||
else
|
||||
FAILED_IDS=$(echo "$FAILED_HIST" | grep -o '"nzo_id":"[^"]*"' | \
|
||||
sed 's/"nzo_id":"//;s/"//')
|
||||
|
||||
if [[ -z "$FAILED_IDS" ]]; then
|
||||
success "No failed history found ✅"
|
||||
else
|
||||
FAILED_TOTAL=$(echo "$FAILED_IDS" | grep -c . 2>/dev/null || echo 0)
|
||||
log "Found $FAILED_TOTAL failed history entries"
|
||||
DELETED=0; SKIPPED=0
|
||||
while IFS= read -r NZO_ID; do
|
||||
[[ -z "$NZO_ID" ]] && continue
|
||||
JOB_TIME=$(echo "$FAILED_HIST" | grep -A5 "$NZO_ID" | \
|
||||
grep -o '"completed":[0-9]*' | grep -o '[0-9]*' | head -1)
|
||||
[[ -z "$JOB_TIME" ]] && continue
|
||||
[[ "$JOB_TIME" -gt "$CUTOFF" ]] && ((SKIPPED++)) && continue
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would delete failed job: $NZO_ID"
|
||||
((DELETED++))
|
||||
else
|
||||
curl -sf --max-time 10 \
|
||||
"$SABNZBD_URL/api?mode=history&name=delete&value=$NZO_ID&apikey=$SABNZBD_API_KEY" \
|
||||
>/dev/null
|
||||
log "$ICON_TRASH Deleted: $NZO_ID"
|
||||
((DELETED++))
|
||||
fi
|
||||
done <<< "$FAILED_IDS"
|
||||
success "Failed: $DELETED deleted, $SKIPPED within retention"
|
||||
(( TOTAL_PASS += DELETED ))
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ SABnzbd — Remove Stalled Queue Items ━━━
|
||||
# ==============================================================================================
|
||||
# Removes queue items in Paused or Stuck state that are no longer progressing.
|
||||
# Active downloading items (Downloading, Grabbing) are never touched.
|
||||
# Paused items may be intentional pauses — but in an automated environment
|
||||
# a Paused item sitting in the queue indefinitely is effectively stalled.
|
||||
|
||||
if [[ -n "$SABNZBD_URL" ]] && [[ -n "$SABNZBD_API_KEY" ]]; then
|
||||
echo ""
|
||||
echo "━━━ 🔍 SABnzbd — Stalled Queue Items ━━━"
|
||||
|
||||
QUEUE=$(curl -sf --max-time 10 \
|
||||
"$SABNZBD_URL/api?mode=queue&output=json&apikey=$SABNZBD_API_KEY" 2>/dev/null)
|
||||
|
||||
if [[ -z "$QUEUE" ]]; then
|
||||
warn "SABnzbd not reachable — skipping queue"
|
||||
else
|
||||
STALLED_IDS=$(echo "$QUEUE" | grep -o '"nzo_id":"[^"]*"' | \
|
||||
sed 's/"nzo_id":"//;s/"//')
|
||||
|
||||
if [[ -z "$STALLED_IDS" ]]; then
|
||||
success "No stalled queue items found ✅"
|
||||
else
|
||||
QUEUE_TOTAL=$(echo "$STALLED_IDS" | grep -c . 2>/dev/null || echo 0)
|
||||
log "Found $QUEUE_TOTAL queue item(s) — checking status"
|
||||
DELETED=0; SKIPPED=0
|
||||
while IFS= read -r NZO_ID; do
|
||||
[[ -z "$NZO_ID" ]] && continue
|
||||
STATUS=$(echo "$QUEUE" | grep -A10 "$NZO_ID" | \
|
||||
grep -o '"status":"[^"]*"' | sed 's/"status":"//;s/"//')
|
||||
# Only remove Paused or Stuck items — Downloading/Grabbing are active
|
||||
if [[ "$STATUS" != "Paused" ]] && [[ "$STATUS" != "Stuck" ]]; then
|
||||
((SKIPPED++))
|
||||
continue
|
||||
fi
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would remove stalled item: $NZO_ID ($STATUS)"
|
||||
((DELETED++))
|
||||
else
|
||||
curl -sf --max-time 10 \
|
||||
"$SABNZBD_URL/api?mode=queue&name=delete&value=$NZO_ID&apikey=$SABNZBD_API_KEY" \
|
||||
>/dev/null
|
||||
log "$ICON_TRASH Removed stalled ($STATUS): $NZO_ID"
|
||||
((DELETED++))
|
||||
fi
|
||||
done <<< "$STALLED_IDS"
|
||||
success "Queue: $DELETED removed, $SKIPPED active (skipped)"
|
||||
(( TOTAL_PASS += DELETED ))
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ qBittorrent — Age Failsafe Cleanup ━━━
|
||||
# ==============================================================================================
|
||||
# Last-chance cleanup for torrents that have been sitting in qBit past their useful life.
|
||||
# deleteFiles=false — removes the torrent record from qBit but leaves files on disk.
|
||||
# Radarr/Sonarr manage actual files independently — this only cleans up the qBit entry.
|
||||
#
|
||||
# Safety checks before deletion:
|
||||
# Age must exceed QBIT_FAILSAFE_MIN_DAYS
|
||||
# Ratio must meet QBIT_FAILSAFE_MIN_RATIO (0 = age only, no ratio requirement)
|
||||
|
||||
if [[ -n "$QBIT_URL" ]] && [[ -n "$QBIT_USERNAME" ]]; then
|
||||
echo ""
|
||||
echo "━━━ 🔍 qBittorrent — Failsafe (older than ${QBIT_FAILSAFE_MIN_DAYS} days) ━━━"
|
||||
[[ "$QBIT_FAILSAFE_MIN_RATIO" != "0" ]] && \
|
||||
log "Ratio requirement: >= ${QBIT_FAILSAFE_MIN_RATIO}"
|
||||
|
||||
QBIT_COOKIE=$(curl -sf --max-time 10 -c - \
|
||||
"$QBIT_URL/api/v2/auth/login" \
|
||||
--data "username=$QBIT_USERNAME&password=$QBIT_PASSWORD" 2>/dev/null | \
|
||||
grep SID | awk '{print "SID="$NF}')
|
||||
|
||||
if [[ -z "$QBIT_COOKIE" ]]; then
|
||||
error "Failed to authenticate with qBittorrent — check QBIT_USERNAME/PASSWORD"
|
||||
notify "qBittorrent auth failed on $(hostname) — check credentials in host*.conf" "Downloaders Reset" "warning"
|
||||
((TOTAL_FAIL++))
|
||||
else
|
||||
TORRENTS=$(curl -sf --max-time 15 \
|
||||
"$QBIT_URL/api/v2/torrents/info" \
|
||||
-H "Cookie: $QBIT_COOKIE" 2>/dev/null)
|
||||
|
||||
NOW=$(date +%s)
|
||||
TORRENT_TOTAL=$(echo "$TORRENTS" | tr '}' '\n' | grep -c '"hash"' 2>/dev/null || echo 0)
|
||||
log "Found $TORRENT_TOTAL torrent(s) — applying age/ratio filter"
|
||||
DELETED=0; SKIPPED=0
|
||||
|
||||
while read -r TORRENT; do
|
||||
[[ -z "$TORRENT" ]] && continue
|
||||
HASH=$(echo "$TORRENT" | grep -o '"hash":"[^"]*"' | sed 's/"hash":"//;s/"//')
|
||||
NAME=$(echo "$TORRENT" | grep -o '"name":"[^"]*"' | sed 's/"name":"//;s/"//')
|
||||
ADDED=$(echo "$TORRENT" | grep -o '"added_on":[0-9]*' | grep -o '[0-9]*')
|
||||
RATIO=$(echo "$TORRENT" | grep -o '"ratio":[0-9.]*' | grep -o '[0-9.]*')
|
||||
[[ -z "$HASH" || -z "$ADDED" ]] && continue
|
||||
|
||||
AGE_DAYS=$(( (NOW - ADDED) / 86400 ))
|
||||
|
||||
# Age check — must be old enough
|
||||
[[ "$AGE_DAYS" -lt "$QBIT_FAILSAFE_MIN_DAYS" ]] && ((SKIPPED++)) && continue
|
||||
|
||||
# Ratio check — if configured
|
||||
if [[ "$QBIT_FAILSAFE_MIN_RATIO" != "0" ]]; then
|
||||
RATIO_INT="${RATIO%.*}"
|
||||
MIN_RATIO_INT="${QBIT_FAILSAFE_MIN_RATIO%.*}"
|
||||
[[ "$RATIO_INT" -lt "$MIN_RATIO_INT" ]] && ((SKIPPED++)) && continue
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would delete: $NAME (${AGE_DAYS}d old, ratio: $RATIO)"
|
||||
((DELETED++))
|
||||
else
|
||||
curl -sf --max-time 10 -X POST \
|
||||
"$QBIT_URL/api/v2/torrents/delete" \
|
||||
-H "Cookie: $QBIT_COOKIE" \
|
||||
--data "hashes=$HASH&deleteFiles=false" >/dev/null
|
||||
log "$ICON_TRASH Deleted: $NAME (${AGE_DAYS}d old, ratio: $RATIO)"
|
||||
((DELETED++))
|
||||
fi
|
||||
done < <(echo "$TORRENTS" | tr '}' '\n')
|
||||
|
||||
success "qBittorrent: $DELETED deleted, $SKIPPED skipped (under threshold)"
|
||||
(( TOTAL_PASS += DELETED ))
|
||||
fi
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY DOWNLOADERS RESET SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( $(date +%s) - START_TIME )))"
|
||||
echo "$ICON_SUCCESS Actions: $TOTAL_PASS"
|
||||
echo "$ICON_ERROR Failures: $TOTAL_FAIL"
|
||||
echo ""
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "$ICON_WARN Status: DRY RUN — no changes made"
|
||||
elif [[ "$TOTAL_FAIL" -gt 0 ]]; then
|
||||
echo "$ICON_ERROR Status: $TOTAL_FAIL failure(s) — check logs"
|
||||
notify "Downloaders reset completed with failures on $(hostname)" "Downloaders Reset" "warning"
|
||||
exit 1
|
||||
else
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS DONE"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
+330
@@ -0,0 +1,330 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ============================= Backup Verify ==================================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# rsync mirror integrity verification via independent MD5 checksums. Scheduled
|
||||
# weekly (Sunday 10am). Randomly samples BACKUP_VERIFY_SAMPLE files per share
|
||||
# above BACKUP_VERIFY_MIN_SIZE, computes checksums locally, then computes the
|
||||
# same checksums on the remote via SSH and compares.
|
||||
#
|
||||
# Per file: MATCH (checksums identical) | MISMATCH (file exists on both but
|
||||
# checksums differ — sync failure or corruption) | MISSING (file exists locally
|
||||
# but not on remote). All MISMATCHes and significant MISSINGs trigger notification.
|
||||
# rsync exit code 0 is not trusted — this script verifies actual content.
|
||||
#
|
||||
# Share list from HOST*_BACKUP_VERIFY_SHARES if defined, otherwise falls back
|
||||
# to HOST*_DAILY_SYNC_SHARES. Both aliased by detect_hosts().
|
||||
#
|
||||
# ==============================================================================================
|
||||
# DESIGN PRINCIPLES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Independent Verification
|
||||
# rsync reports success when the transfer completed without network errors and
|
||||
# file sizes and modification times match. It does not detect silent corruption
|
||||
# during transfer (bitflip in transit), corruption written to storage at rest
|
||||
# (faulty drive sector), or files that matched size/mtime but had wrong content.
|
||||
# All of these produce exit code 0. This script checks whether "done" means "correct."
|
||||
#
|
||||
# Intentionally Small Sample
|
||||
# 10 files per share (default) — a spot check, not an exhaustive verify.
|
||||
# Catches systematic problems and hardware issues while running in minutes, not
|
||||
# hours. Full verification would take longer than the rsync itself.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Single Instance Lock
|
||||
# acquire_lock prevents concurrent runs producing conflicting results.
|
||||
#
|
||||
# Remote Connectivity Check
|
||||
# check_connectivity() verifies the remote Tailscale IP is reachable before
|
||||
# any SSH calls. Without this, all files show as MISSING on a network hiccup.
|
||||
#
|
||||
# Remote Array Check
|
||||
# check_remote_array() verifies /mnt/user is mounted on the remote before
|
||||
# computing checksums. Array not started = all files "missing" = false alarm.
|
||||
#
|
||||
# Version Parity
|
||||
# Refuses to run if remote unRAID version doesn't match local. A mismatch
|
||||
# may mean the remote is in an unexpected state.
|
||||
#
|
||||
# SSH Timeout
|
||||
# SSH_TIMEOUT caps all SSH calls. One hung connection does not block the run.
|
||||
#
|
||||
# Notification Validated
|
||||
# platform_require_cmd confirms the notify script is present before use.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# host*.conf
|
||||
#
|
||||
# HOST*_BACKUP_VERIFY_SHARES
|
||||
# Shares to verify. Leave empty to use HOST*_DAILY_SYNC_SHARES automatically.
|
||||
# Aliased by detect_hosts() → BACKUP_VERIFY_SHARES.
|
||||
#
|
||||
# HOST*_DAILY_SYNC_SHARES
|
||||
# Fallback share list if BACKUP_VERIFY_SHARES is empty. Aliased by detect_hosts().
|
||||
#
|
||||
# master.conf
|
||||
#
|
||||
# BACKUP_VERIFY_SAMPLE
|
||||
# Random files checked per share per run. (default: 10)
|
||||
#
|
||||
# BACKUP_VERIFY_MIN_SIZE
|
||||
# Minimum file size to include in sample — tiny files have low corruption
|
||||
# risk and slow checksums. (default: 1M)
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# backup_verify.sh
|
||||
# Sample files from all shares and compare checksums. Notify on MISMATCH
|
||||
# or significant MISSING count. Silent when all samples match.
|
||||
#
|
||||
# backup_verify.sh --dry-run
|
||||
# Show which files would be sampled. No checksums computed, no notifications.
|
||||
#
|
||||
# backup_verify.sh --status
|
||||
# Show share list, sample size, and min file size configuration. Then exit.
|
||||
#
|
||||
# backup_verify.sh --log
|
||||
# Verbose per-file checksum comparison output during the run.
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
SSH_TIMEOUT=15
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
acquire_lock
|
||||
|
||||
# detect_hosts() sets MY_ID and aliases BACKUP_VERIFY_SHARES + DAILY_SYNC_SHARES
|
||||
detect_hosts
|
||||
|
||||
# Share selection — configured list or fallback to daily sync shares
|
||||
if [[ ${#BACKUP_VERIFY_SHARES[@]} -gt 0 ]]; then
|
||||
VERIFY_SHARES=("${BACKUP_VERIFY_SHARES[@]}")
|
||||
log "Using BACKUP_VERIFY_SHARES (${#VERIFY_SHARES[@]} shares)"
|
||||
else
|
||||
VERIFY_SHARES=("${DAILY_SYNC_SHARES[@]}")
|
||||
log "BACKUP_VERIFY_SHARES not set — using DAILY_SYNC_SHARES (${#VERIFY_SHARES[@]} shares)"
|
||||
fi
|
||||
|
||||
if [[ ${#VERIFY_SHARES[@]} -eq 0 ]]; then
|
||||
warn "No shares configured for $MY_ID — nothing to verify"
|
||||
warn "Check HOST*_BACKUP_VERIFY_SHARES or HOST*_DAILY_SYNC_SHARES in host*.conf"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
log "$ICON_GEAR Config: sample=${BACKUP_VERIFY_SAMPLE} min-size=${BACKUP_VERIFY_MIN_SIZE} ssh-timeout=${SSH_TIMEOUT}s"
|
||||
log "$ICON_GEAR Remote: $REMOTE_ID ($REMOTE_SERVER_NAME — $REMOTE_SERVER)"
|
||||
log "$ICON_GEAR Shares: ${VERIFY_SHARES[*]}"
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — showing sample selection only, no checksums computed"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_HOST My ID: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_HOST Remote: $REMOTE_ID ($REMOTE_SERVER_NAME — $REMOTE_SERVER)"
|
||||
echo "$ICON_VERIFY Shares: ${#VERIFY_SHARES[@]}"
|
||||
echo "$ICON_VERIFY Sample: $BACKUP_VERIFY_SAMPLE files per share"
|
||||
echo "$ICON_VERIFY Min size: $BACKUP_VERIFY_MIN_SIZE"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo ""
|
||||
echo " Shares to verify:"
|
||||
for share in "${VERIFY_SHARES[@]}"; do
|
||||
echo " $share"
|
||||
done
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Pre-flight ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_SHIELD Pre-flight ━━━"
|
||||
|
||||
resolve_remote_ip
|
||||
|
||||
# Connectivity — no point making 100+ SSH calls if remote is unreachable
|
||||
check_connectivity
|
||||
log "Connectivity to $REMOTE_SERVER_NAME ✅"
|
||||
|
||||
# Version parity — mismatched unRAID could cause md5sum path differences
|
||||
check_os_version_parity || {
|
||||
warn "Version parity check failed — proceeding with caution"
|
||||
warn "Checksum results may be unreliable if md5sum path changed between versions"
|
||||
}
|
||||
log "Version parity with $REMOTE_SERVER_NAME ✅"
|
||||
|
||||
# Remote array — if array is down all files appear "missing" = false alarm
|
||||
if ! check_remote_array; then
|
||||
error "Remote array not mounted on $REMOTE_SERVER_NAME"
|
||||
error "All files would appear as MISSING — aborting to prevent false alarm"
|
||||
notify "Backup verify aborted on $(hostname) — remote array not mounted on $REMOTE_SERVER_NAME" \
|
||||
"Backup Verify" "warning"
|
||||
exit 1
|
||||
fi
|
||||
log "Remote array mounted on $REMOTE_SERVER_NAME ✅"
|
||||
|
||||
echo "Pre-flight passed ✅"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Backup Verification ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_VERIFY Backup Verification — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME) → $REMOTE_ID ($REMOTE_SERVER_NAME)"
|
||||
echo "$ICON_VERIFY Sample: $BACKUP_VERIFY_SAMPLE files per share (min: $BACKUP_VERIFY_MIN_SIZE)"
|
||||
echo ""
|
||||
|
||||
START=$(date +%s)
|
||||
TOTAL_CHECKED=0
|
||||
TOTAL_MATCH=0
|
||||
TOTAL_MISMATCH=0
|
||||
TOTAL_MISSING=0
|
||||
SHARES_WITH_ISSUES=()
|
||||
|
||||
for share in "${VERIFY_SHARES[@]}"; do
|
||||
SHARE_NAME=$(basename "$share")
|
||||
echo "━━━ $ICON_VERIFY $SHARE_NAME ━━━"
|
||||
|
||||
if [[ ! -d "$share" ]]; then
|
||||
warn "$SHARE_NAME not found locally — skipping"
|
||||
echo ""
|
||||
continue
|
||||
fi
|
||||
|
||||
# Sample random files above minimum size
|
||||
mapfile -t SAMPLE_FILES < <(
|
||||
find "$share" -type f -size +"$BACKUP_VERIFY_MIN_SIZE" 2>/dev/null | \
|
||||
shuf | head -n "$BACKUP_VERIFY_SAMPLE"
|
||||
)
|
||||
|
||||
if [[ ${#SAMPLE_FILES[@]} -eq 0 ]]; then
|
||||
log "$SHARE_NAME — no files found above $BACKUP_VERIFY_MIN_SIZE"
|
||||
echo ""
|
||||
continue
|
||||
fi
|
||||
|
||||
log "$SHARE_NAME — sampled ${#SAMPLE_FILES[@]} files"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
for f in "${SAMPLE_FILES[@]}"; do
|
||||
warn "DRY RUN — would check: $(basename "$f")"
|
||||
done
|
||||
echo ""
|
||||
continue
|
||||
fi
|
||||
|
||||
SHARE_MATCH=0
|
||||
SHARE_MISMATCH=0
|
||||
SHARE_MISSING=0
|
||||
|
||||
for local_file in "${SAMPLE_FILES[@]}"; do
|
||||
[[ -z "$local_file" ]] && continue
|
||||
|
||||
# Local checksum
|
||||
local_md5=$(md5sum "$local_file" 2>/dev/null | awk '{print $1}')
|
||||
if [[ -z "$local_md5" ]]; then
|
||||
warn "Could not checksum locally: $(basename "$local_file") — skipping"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Remote checksum via SSH — timeout protected
|
||||
remote_md5=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" \
|
||||
-o StrictHostKeyChecking=no \
|
||||
root@"$REMOTE_SERVER" \
|
||||
"md5sum '$local_file' 2>/dev/null | awk '{print \$1}'" 2>/dev/null)
|
||||
|
||||
(( TOTAL_CHECKED++ ))
|
||||
|
||||
if [[ -z "$remote_md5" ]]; then
|
||||
warn "$ICON_ERROR MISSING: $(basename "$local_file")"
|
||||
(( SHARE_MISSING++ ))
|
||||
(( TOTAL_MISSING++ ))
|
||||
elif [[ "$local_md5" == "$remote_md5" ]]; then
|
||||
log "MATCH: $(basename "$local_file")"
|
||||
(( SHARE_MATCH++ ))
|
||||
(( TOTAL_MATCH++ ))
|
||||
else
|
||||
error "MISMATCH: $(basename "$local_file")"
|
||||
error " local: $local_md5"
|
||||
error " remote: $remote_md5"
|
||||
(( SHARE_MISMATCH++ ))
|
||||
(( TOTAL_MISMATCH++ ))
|
||||
fi
|
||||
done
|
||||
|
||||
# Per-share result — only visible if issues found
|
||||
if [[ "$SHARE_MISMATCH" -gt 0 || "$SHARE_MISSING" -gt 0 ]]; then
|
||||
warn "$SHARE_NAME — match: $SHARE_MATCH missing: $SHARE_MISSING mismatch: $SHARE_MISMATCH"
|
||||
SHARES_WITH_ISSUES+=("$SHARE_NAME")
|
||||
else
|
||||
log "$SHARE_NAME — all $SHARE_MATCH files match ✅"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
done
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo "━━━━━ $ICON_SUMMARY BACKUP VERIFY SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST My ID: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_HOST Remote: $REMOTE_ID ($REMOTE_SERVER_NAME)"
|
||||
echo "$ICON_VERIFY Checked: $TOTAL_CHECKED files"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
|
||||
if [[ "$TOTAL_MISMATCH" -gt 0 || "$TOTAL_MISSING" -gt 0 ]]; then
|
||||
echo "$ICON_SUCCESS Match: $TOTAL_MATCH"
|
||||
warn "Missing: $TOTAL_MISSING"
|
||||
[[ "$TOTAL_MISMATCH" -gt 0 ]] && echo "$ICON_ERROR Mismatch: $TOTAL_MISMATCH"
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no checksums computed"
|
||||
elif [[ "$TOTAL_MISMATCH" -gt 0 || "$TOTAL_MISSING" -gt 0 ]]; then
|
||||
echo "$ICON_ERROR Status: ISSUES FOUND — ${#SHARES_WITH_ISSUES[@]} share(s) need attention: ${SHARES_WITH_ISSUES[*]}"
|
||||
notify "Backup verify FAILED on $(hostname) → $REMOTE_SERVER_NAME — mismatches: $TOTAL_MISMATCH missing: $TOTAL_MISSING — shares: ${SHARES_WITH_ISSUES[*]}" \
|
||||
"Backup Verify" "warning"
|
||||
else
|
||||
echo "$ICON_DONE Status: all $TOTAL_CHECKED files match across ${#VERIFY_SHARES[@]} shares ✅"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ "$TOTAL_MISMATCH" -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
+382
@@ -0,0 +1,382 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ============================= Emby Database Repair ===========================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Stops Emby, runs SQLite PRAGMA integrity_check on all Emby databases, and
|
||||
# restarts. Use when Emby reports corruption, unexpected crashes, or playback
|
||||
# state issues.
|
||||
#
|
||||
# Reports which databases are corrupted. Does NOT automatically repair.
|
||||
# Repair requires manual steps — guidance is printed in the summary output.
|
||||
# Always take a backup before deleting any database file.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Databases Checked
|
||||
# library.db — media library metadata (largest, most critical)
|
||||
# library.db-wal — write-ahead log (if present — uncommitted transactions)
|
||||
# librarydb.db — legacy library database
|
||||
# users.db — user accounts and settings
|
||||
# authentication.db — API keys and sessions
|
||||
# activity.db — activity log (least critical, safe to delete if corrupt)
|
||||
#
|
||||
# Missing databases are skipped gracefully — not all files exist on all setups.
|
||||
# Emby's config path is detected from the Docker mount — no hardcoded paths.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Guaranteed Restart
|
||||
# EXIT trap ensures Emby is always restarted even if the script crashes
|
||||
# mid-check — Emby is never left stopped due to a script error.
|
||||
#
|
||||
# Docker Timeout
|
||||
# DOCKER_TIMEOUT (30s) protects all docker calls against a hung daemon.
|
||||
# Emby can take time to stop cleanly — 30s is intentionally generous.
|
||||
#
|
||||
# Tool Validation
|
||||
# platform_require_cmd confirms sqlite3 and the notify script are present
|
||||
# before use. jq is checked separately — required for config path detection.
|
||||
#
|
||||
# Post-Restart Verify
|
||||
# Checks that Emby is still running after restart — detects cases where
|
||||
# Emby crashes immediately after start (which would indicate deeper trouble).
|
||||
#
|
||||
# Silent When Healthy
|
||||
# Only corruption produces visible output and a notification.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# host*.conf
|
||||
#
|
||||
# HOST*_EMBY_CONTAINER
|
||||
# Name of the Emby Docker container on this host.
|
||||
# Aliased by detect_hosts() → EMBY_CONTAINER.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# emby_database_repair.sh
|
||||
# Stop Emby, check all databases with PRAGMA integrity_check, restart.
|
||||
#
|
||||
# emby_database_repair.sh --dry-run
|
||||
# Show which databases would be checked and Emby container name. No stop.
|
||||
#
|
||||
# emby_database_repair.sh --log
|
||||
# Verbose output with per-database check result.
|
||||
#
|
||||
# emby_database_repair.sh --status
|
||||
# Show Emby container name and config path detected from Docker. Then exit.
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
DOCKER_TIMEOUT=30 # Emby can take time to stop cleanly
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Validate required tools
|
||||
platform_require_cmd \
|
||||
"$(command -v sqlite3 2>/dev/null || echo /usr/bin/sqlite3)" \
|
||||
"--version" "." \
|
||||
"sqlite3" || { error "sqlite3 not found — install sqlite package"; exit 1; }
|
||||
|
||||
|
||||
if ! command -v jq >/dev/null 2>&1; then
|
||||
error "jq not found — required to detect Emby config path from Docker mounts"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
acquire_lock
|
||||
|
||||
if ! command -v docker &>/dev/null; then
|
||||
error "Docker command not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# detect_hosts() sets MY_ID and aliases HOST*_EMBY_CONTAINER → EMBY_CONTAINER
|
||||
detect_hosts
|
||||
|
||||
if [[ -z "${EMBY_CONTAINER:-}" ]]; then
|
||||
error "EMBY_CONTAINER not set for $MY_ID — check HOST*_EMBY_CONTAINER in host*.conf"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log "Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
log "Emby container: $EMBY_CONTAINER"
|
||||
|
||||
# Detect Emby config path from Docker container mounts
|
||||
EMBY_CONFIG_HOST=$(timeout "$DOCKER_TIMEOUT" docker inspect "$EMBY_CONTAINER" 2>/dev/null | \
|
||||
jq -r '.[] | .Mounts[] | select(.Destination == "/config") | .Source' 2>/dev/null)
|
||||
|
||||
if [[ -z "$EMBY_CONFIG_HOST" ]]; then
|
||||
error "Could not detect Emby config path from Docker mounts"
|
||||
error "Is $EMBY_CONTAINER the correct container name? Check HOST*_EMBY_CONTAINER in host*.conf"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log "Emby config: $EMBY_CONFIG_HOST"
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — Emby will not be stopped, no checks run"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_EMBY Container: $EMBY_CONTAINER"
|
||||
echo "$ICON_EMBY Config: $EMBY_CONFIG_HOST"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo ""
|
||||
echo "━━━ Database Files ━━━"
|
||||
for db_rel in "data/library.db" "data/library.db-wal" "data/librarydb.db" \
|
||||
"data/users.db" "data/authentication.db" "data/activity.db"; do
|
||||
db_path="${EMBY_CONFIG_HOST}/${db_rel}"
|
||||
db_name=$(basename "$db_rel")
|
||||
if [[ -f "$db_path" ]]; then
|
||||
db_size=$(du -sh "$db_path" 2>/dev/null | cut -f1)
|
||||
echo " $ICON_SUCCESS $db_name ($db_size)"
|
||||
else
|
||||
echo " $ICON_SKIP $db_name — not found"
|
||||
fi
|
||||
done
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ── EXIT trap — Emby always restarted if it was running ───────────────────────────────────────
|
||||
EMBY_WAS_RUNNING=false
|
||||
|
||||
cleanup_on_exit() {
|
||||
local exit_code=$?
|
||||
if [[ "$EMBY_WAS_RUNNING" == true && "$DRY_RUN" == false ]]; then
|
||||
local status
|
||||
status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \
|
||||
"$EMBY_CONTAINER" 2>/dev/null)
|
||||
if [[ "$status" != "true" ]]; then
|
||||
warn "Restarting $EMBY_CONTAINER (cleanup)..."
|
||||
timeout "$DOCKER_TIMEOUT" docker start "$EMBY_CONTAINER" >/dev/null 2>&1 || \
|
||||
error "Failed to restart $EMBY_CONTAINER — start it manually"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
trap cleanup_on_exit EXIT
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Stop Emby ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_STOP Stop Emby ━━━"
|
||||
|
||||
STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \
|
||||
"$EMBY_CONTAINER" 2>/dev/null)
|
||||
|
||||
case "$STATUS" in
|
||||
true)
|
||||
EMBY_WAS_RUNNING=true
|
||||
warn "Stopping $EMBY_CONTAINER — active sessions will be interrupted"
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
if timeout "$DOCKER_TIMEOUT" docker stop "$EMBY_CONTAINER" >/dev/null 2>&1; then
|
||||
log "$EMBY_CONTAINER stopped ✅"
|
||||
sleep 3 # let file handles release
|
||||
else
|
||||
error "Failed to stop $EMBY_CONTAINER — aborting"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
warn "DRY RUN — would stop $EMBY_CONTAINER"
|
||||
fi
|
||||
;;
|
||||
false)
|
||||
log "$EMBY_CONTAINER is not running — proceeding with checks"
|
||||
;;
|
||||
"")
|
||||
error "$EMBY_CONTAINER not found — check container name"
|
||||
exit 1
|
||||
;;
|
||||
*)
|
||||
warn "$EMBY_CONTAINER status: $STATUS — proceeding with caution"
|
||||
;;
|
||||
esac
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Database Integrity Check ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_HEALTH Database Integrity Check ━━━"
|
||||
|
||||
START=$(date +%s)
|
||||
|
||||
DB_FILES=(
|
||||
"data/library.db"
|
||||
"data/library.db-wal"
|
||||
"data/librarydb.db"
|
||||
"data/users.db"
|
||||
"data/authentication.db"
|
||||
"data/activity.db"
|
||||
)
|
||||
|
||||
PASS_DBS=()
|
||||
FAIL_DBS=()
|
||||
MISSING_DBS=()
|
||||
|
||||
for db_rel in "${DB_FILES[@]}"; do
|
||||
db_path="${EMBY_CONFIG_HOST}/${db_rel}"
|
||||
db_name=$(basename "$db_rel")
|
||||
|
||||
if [[ ! -f "$db_path" ]]; then
|
||||
log "$db_name — not found, skipping"
|
||||
MISSING_DBS+=("$db_name")
|
||||
continue
|
||||
fi
|
||||
|
||||
DB_SIZE=$(du -sh "$db_path" 2>/dev/null | cut -f1)
|
||||
log "Checking $db_name ($DB_SIZE)..."
|
||||
|
||||
# WAL file — different check (not a full SQLite database)
|
||||
if [[ "$db_name" == *.wal ]]; then
|
||||
if [[ -s "$db_path" ]]; then
|
||||
warn "$db_name exists and is non-empty (${DB_SIZE})"
|
||||
warn "Uncommitted WAL data — will be merged when Emby next starts cleanly"
|
||||
PASS_DBS+=("$db_name (WAL — see warning)")
|
||||
else
|
||||
log "$db_name exists but is empty — no pending transactions ✅"
|
||||
PASS_DBS+=("$db_name")
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would check: $db_path"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Full integrity check
|
||||
RESULT=$(sqlite3 "$db_path" "PRAGMA integrity_check;" 2>/dev/null)
|
||||
EXIT_CODE=$?
|
||||
|
||||
if [[ "$EXIT_CODE" -ne 0 ]]; then
|
||||
error "$db_name — sqlite3 could not open database (locked or corrupt)"
|
||||
FAIL_DBS+=("$db_name")
|
||||
elif [[ "$RESULT" == "ok" ]]; then
|
||||
log "$db_name — integrity check passed ✅"
|
||||
PASS_DBS+=("$db_name")
|
||||
else
|
||||
error "$db_name — CORRUPTION DETECTED"
|
||||
echo "$RESULT" | head -10 | while IFS= read -r line; do
|
||||
error " $line"
|
||||
done
|
||||
FAIL_DBS+=("$db_name")
|
||||
fi
|
||||
done
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Restart Emby ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_START Restart Emby ━━━"
|
||||
|
||||
RESTART_OK=false
|
||||
|
||||
if [[ "$EMBY_WAS_RUNNING" == true ]]; then
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
log "Restarting $EMBY_CONTAINER..."
|
||||
if timeout "$DOCKER_TIMEOUT" docker start "$EMBY_CONTAINER" >/dev/null 2>&1; then
|
||||
sleep 5 # Emby takes longer to initialise than most containers
|
||||
POST_STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
|
||||
'{{.State.Running}}' "$EMBY_CONTAINER" 2>/dev/null)
|
||||
if [[ "$POST_STATUS" == "true" ]]; then
|
||||
log "$EMBY_CONTAINER restarted and running ✅"
|
||||
RESTART_OK=true
|
||||
else
|
||||
error "$EMBY_CONTAINER started but crashed — database may be corrupt"
|
||||
error "Check Docker logs: docker logs $EMBY_CONTAINER"
|
||||
notify "$EMBY_CONTAINER crashed on restart — possible database corruption on $(hostname)" \
|
||||
"Emby DB Repair" "warning"
|
||||
fi
|
||||
else
|
||||
error "Failed to restart $EMBY_CONTAINER — start it manually"
|
||||
notify "$EMBY_CONTAINER failed to restart after integrity check on $(hostname)" \
|
||||
"Emby DB Repair" "warning"
|
||||
fi
|
||||
else
|
||||
warn "DRY RUN — would restart $EMBY_CONTAINER"
|
||||
RESTART_OK=true
|
||||
fi
|
||||
else
|
||||
log "$EMBY_CONTAINER was not running — leaving stopped (state respected) ✅"
|
||||
RESTART_OK=true
|
||||
fi
|
||||
|
||||
# Clear EXIT trap — clean exit
|
||||
trap - EXIT
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY EMBY DATABASE REPAIR SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_EMBY Container: $EMBY_CONTAINER"
|
||||
echo "$ICON_EMBY Config: $EMBY_CONFIG_HOST"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
echo " $ICON_SUCCESS Passed: ${#PASS_DBS[@]}"
|
||||
[[ ${#FAIL_DBS[@]} -gt 0 ]] && echo " $ICON_ERROR Failed: ${#FAIL_DBS[@]}"
|
||||
[[ ${#MISSING_DBS[@]} -gt 0 ]] && echo " Skipped: ${#MISSING_DBS[@]} (not found)"
|
||||
echo ""
|
||||
|
||||
[[ ${#PASS_DBS[@]} -gt 0 ]] && for db in "${PASS_DBS[@]}"; do log " $ICON_SUCCESS $db"; done
|
||||
[[ ${#FAIL_DBS[@]} -gt 0 ]] && for db in "${FAIL_DBS[@]}"; do echo " $ICON_ERROR $db"; done
|
||||
|
||||
echo ""
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no checks performed"
|
||||
elif [[ ${#FAIL_DBS[@]} -gt 0 ]]; then
|
||||
echo "$ICON_ERROR Status: CORRUPTION FOUND — manual intervention needed"
|
||||
echo ""
|
||||
echo "$ICON_INFO Next steps per corrupted database:"
|
||||
echo " library.db — Delete file, restart Emby — rebuilds from media (slow first start)"
|
||||
echo " library.db-wal — Delete WAL file, restart Emby — safe, no permanent data loss"
|
||||
echo " librarydb.db — Delete file, restart Emby — legacy, Emby recreates"
|
||||
echo " users.db — Restore from backup or delete — deleting resets all user accounts"
|
||||
echo " authentication.db — Delete file, restart Emby — API keys regenerated automatically"
|
||||
echo " activity.db — Delete file, restart Emby — activity log only, no media data"
|
||||
echo ""
|
||||
warn "⚠️ Always take a backup before deleting any database file"
|
||||
warn " Run: container_data_export.sh $EMBY_CONTAINER <config_path> <backup_dir>"
|
||||
notify "Emby database CORRUPTION on $(hostname) — failed: ${FAIL_DBS[*]} — manual intervention needed" \
|
||||
"Emby DB Repair" "warning"
|
||||
else
|
||||
echo "$ICON_DONE Status: all ${#PASS_DBS[@]} databases healthy ✅"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ ${#FAIL_DBS[@]} -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
+312
@@ -0,0 +1,312 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ============================= Container Data Export ==========================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Exports a container's appdata directory to a compressed tar archive. Stops
|
||||
# the container before archiving and restarts it after — ensures a clean,
|
||||
# consistent backup. Use before major updates, pool migrations, destructive
|
||||
# appdata operations, or when archiving a container being removed from the stack.
|
||||
#
|
||||
# Output: ContainerName_YYYY-MM-DD_HH-MM.tar.gz — timestamped, no overwrite.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# DESIGN PRINCIPLES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Archive Verification Before Restart
|
||||
# The archive is tested with tar --test-file before the container is restarted.
|
||||
# A corrupt archive is not a usable backup — this catches tar failures, I/O
|
||||
# errors, and truncated writes before declaring success. If verification fails,
|
||||
# the container is still restarted (appdata is unchanged) and an error logged.
|
||||
#
|
||||
# Conservative Space Estimate
|
||||
# Required space is estimated as appdata size × 1.1 (10% buffer). The actual
|
||||
# compressed archive will typically be much smaller — database files compress
|
||||
# well, media files do not. The estimate is a conservative floor, not a
|
||||
# prediction.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Container Restart Rule
|
||||
# Tracks whether the container was running before the export. Running containers
|
||||
# are restarted after completion; already-stopped containers are left stopped.
|
||||
# The restart happens on every exit path — a failed tar does not leave the
|
||||
# container stuck stopped.
|
||||
#
|
||||
# Partial Archive Cleanup
|
||||
# If tar fails, the incomplete archive is removed. A partial archive is worse
|
||||
# than no archive — it can look valid but restore to an incomplete state.
|
||||
#
|
||||
# Docker Timeout
|
||||
# DOCKER_TIMEOUT (default: 30s) caps all docker calls. Guards against a hung
|
||||
# daemon blocking the script indefinitely.
|
||||
#
|
||||
# Notification Validated
|
||||
# platform_require_cmd confirms the notify script is present before use.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# container_data_export.sh ContainerName /path/to/appdata /path/to/output/dir
|
||||
# Stop container, create archive, verify, restart container.
|
||||
# Example: container_data_export.sh Emby /mnt/media-servers/.../Emby /mnt/user/Backups/
|
||||
#
|
||||
# container_data_export.sh ContainerName /path/to/appdata /path/to/output/dir --dry-run
|
||||
# Show what would be archived and estimated size. No container stop, no tar.
|
||||
#
|
||||
# container_data_export.sh ContainerName /path/to/appdata /path/to/output/dir --log
|
||||
# Verbose output: space check, tar progress, verification result.
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
DOCKER_TIMEOUT=30 # longer timeout — stop can take time on large containers
|
||||
|
||||
# ── Positional args ───────────────────────────────────────────────────────────────────────────
|
||||
CONTAINER_NAME="${PARSED_ARGS[0]:-}"
|
||||
APPDATA_PATH="${PARSED_ARGS[1]:-}"
|
||||
OUTPUT_DIR="${PARSED_ARGS[2]:-}"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
acquire_lock
|
||||
|
||||
if ! command -v docker &>/dev/null; then
|
||||
error "Docker command not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
# detect_hosts() sets MY_ID — used in summary
|
||||
detect_hosts
|
||||
|
||||
# Arg validation
|
||||
if [[ -z "$CONTAINER_NAME" || -z "$APPDATA_PATH" || -z "$OUTPUT_DIR" ]]; then
|
||||
error "Usage: container_data_export.sh <ContainerName> <appdata_path> <output_dir>"
|
||||
error "Example: container_data_export.sh Emby /mnt/media-servers/Media_Server/Emby /mnt/user/Backups/"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -d "$APPDATA_PATH" ]]; then
|
||||
error "Appdata path not found: $APPDATA_PATH"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -d "$OUTPUT_DIR" ]]; then
|
||||
error "Output directory not found: $OUTPUT_DIR"
|
||||
error "Create it first: mkdir -p \"$OUTPUT_DIR\""
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Space check — conservative: appdata × 1.1
|
||||
APPDATA_SIZE_KB=$(du -sk "$APPDATA_PATH" 2>/dev/null | cut -f1)
|
||||
OUTPUT_FREE_KB=$(df "$OUTPUT_DIR" --output=avail 2>/dev/null | tail -1 | tr -d ' ')
|
||||
REQUIRED_KB=$(( APPDATA_SIZE_KB * 11 / 10 ))
|
||||
APPDATA_SIZE_H=$(du -sh "$APPDATA_PATH" 2>/dev/null | cut -f1)
|
||||
OUTPUT_FREE_H=$(df -h "$OUTPUT_DIR" --output=avail 2>/dev/null | tail -1 | tr -d ' ')
|
||||
|
||||
if [[ "$OUTPUT_FREE_KB" -lt "$REQUIRED_KB" ]]; then
|
||||
error "Insufficient space in $OUTPUT_DIR"
|
||||
error "Estimated need: ~${APPDATA_SIZE_H} (×1.1 conservative) — available: ${OUTPUT_FREE_H}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log "Container: $CONTAINER_NAME"
|
||||
log "Appdata: $APPDATA_PATH ($APPDATA_SIZE_H)"
|
||||
log "Output: $OUTPUT_DIR ($OUTPUT_FREE_H free)"
|
||||
log "Space check passed"
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
|
||||
# ── Ensure container is restarted on any exit if it was running ────────────────────────────────
|
||||
CONTAINER_WAS_RUNNING=false
|
||||
ARCHIVE_PATH=""
|
||||
|
||||
cleanup_on_exit() {
|
||||
local exit_code=$?
|
||||
# Remove partial archive on failure
|
||||
if [[ "$exit_code" -ne 0 && -n "$ARCHIVE_PATH" && -f "$ARCHIVE_PATH" ]]; then
|
||||
warn "Removing partial archive: $ARCHIVE_PATH"
|
||||
rm -f "$ARCHIVE_PATH" 2>/dev/null
|
||||
fi
|
||||
# Always restart container if it was running
|
||||
if [[ "$CONTAINER_WAS_RUNNING" == true && "$DRY_RUN" == false ]]; then
|
||||
local status
|
||||
status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \
|
||||
"$CONTAINER_NAME" 2>/dev/null)
|
||||
if [[ "$status" != "true" ]]; then
|
||||
warn "Restarting $CONTAINER_NAME (cleanup)..."
|
||||
timeout "$DOCKER_TIMEOUT" docker start "$CONTAINER_NAME" >/dev/null 2>&1 || \
|
||||
error "Failed to restart $CONTAINER_NAME — start it manually"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
trap cleanup_on_exit EXIT
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Stop Container ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_STOP Stop Container ━━━"
|
||||
|
||||
STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \
|
||||
"$CONTAINER_NAME" 2>/dev/null)
|
||||
|
||||
case "$STATUS" in
|
||||
true)
|
||||
CONTAINER_WAS_RUNNING=true
|
||||
log "Stopping $CONTAINER_NAME for clean export..."
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
if timeout "$DOCKER_TIMEOUT" docker stop "$CONTAINER_NAME" >/dev/null 2>&1; then
|
||||
log "$CONTAINER_NAME stopped ✅"
|
||||
else
|
||||
error "Failed to stop $CONTAINER_NAME — aborting export"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
warn "DRY RUN — would stop $CONTAINER_NAME"
|
||||
fi
|
||||
;;
|
||||
false)
|
||||
log "$CONTAINER_NAME is not running — archiving as-is (was stopped state respected)"
|
||||
;;
|
||||
"")
|
||||
error "$CONTAINER_NAME not found — check container name"
|
||||
exit 1
|
||||
;;
|
||||
*)
|
||||
warn "$CONTAINER_NAME status: $STATUS — proceeding with caution"
|
||||
;;
|
||||
esac
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Archive ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Archive ━━━"
|
||||
|
||||
TIMESTAMP=$(date '+%Y-%m-%d_%H-%M')
|
||||
ARCHIVE_NAME="${CONTAINER_NAME}_${TIMESTAMP}.tar.gz"
|
||||
ARCHIVE_PATH="${OUTPUT_DIR}/${ARCHIVE_NAME}"
|
||||
|
||||
warn "Creating: $ARCHIVE_PATH"
|
||||
warn "Source: $APPDATA_PATH ($APPDATA_SIZE_H)"
|
||||
|
||||
START=$(date +%s)
|
||||
ARCHIVE_VERIFIED=false
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
if tar -czf "$ARCHIVE_PATH" \
|
||||
-C "$(dirname "$APPDATA_PATH")" \
|
||||
"$(basename "$APPDATA_PATH")" 2>/dev/null; then
|
||||
|
||||
ARCHIVE_SIZE=$(du -sh "$ARCHIVE_PATH" 2>/dev/null | cut -f1)
|
||||
warn "Archive created: $ARCHIVE_NAME ($ARCHIVE_SIZE)"
|
||||
|
||||
# Verify archive integrity before declaring success
|
||||
log "Verifying archive..."
|
||||
if tar --test-label -f "$ARCHIVE_PATH" 2>/dev/null || \
|
||||
tar -tzf "$ARCHIVE_PATH" >/dev/null 2>&1; then
|
||||
log "Archive verified ✅"
|
||||
ARCHIVE_VERIFIED=true
|
||||
else
|
||||
error "Archive verification FAILED — archive may be corrupt"
|
||||
error "Container will be restarted but DO NOT rely on this backup"
|
||||
notify "Container export archive corrupt — $CONTAINER_NAME backup may be unusable" \
|
||||
"Container Export" "warning"
|
||||
fi
|
||||
else
|
||||
error "tar failed — archive creation unsuccessful"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
warn "DRY RUN — would create: $ARCHIVE_PATH"
|
||||
ARCHIVE_VERIFIED=true
|
||||
fi
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Restart Container ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_START Restart Container ━━━"
|
||||
|
||||
RESTART_OK=false
|
||||
|
||||
if [[ "$CONTAINER_WAS_RUNNING" == true ]]; then
|
||||
log "Restarting $CONTAINER_NAME..."
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
if timeout "$DOCKER_TIMEOUT" docker start "$CONTAINER_NAME" >/dev/null 2>&1; then
|
||||
# Brief settle then verify
|
||||
sleep 3
|
||||
POST_STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
|
||||
'{{.State.Running}}' "$CONTAINER_NAME" 2>/dev/null)
|
||||
if [[ "$POST_STATUS" == "true" ]]; then
|
||||
log "$CONTAINER_NAME restarted and running ✅"
|
||||
RESTART_OK=true
|
||||
else
|
||||
error "$CONTAINER_NAME started but crashed immediately — check container logs"
|
||||
notify "$CONTAINER_NAME failed to stay running after export on $(hostname)" \
|
||||
"Container Export" "warning"
|
||||
fi
|
||||
else
|
||||
error "Failed to restart $CONTAINER_NAME — start it manually"
|
||||
notify "$CONTAINER_NAME failed to restart after export on $(hostname)" \
|
||||
"Container Export" "warning"
|
||||
fi
|
||||
else
|
||||
warn "DRY RUN — would restart $CONTAINER_NAME"
|
||||
RESTART_OK=true
|
||||
fi
|
||||
else
|
||||
log "$CONTAINER_NAME was not running — leaving stopped (state respected) ✅"
|
||||
RESTART_OK=true
|
||||
fi
|
||||
|
||||
# Clear trap — clean exit, cleanup_on_exit no longer needed
|
||||
trap - EXIT
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY CONTAINER EXPORT SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_CONTAINERS Container: $CONTAINER_NAME"
|
||||
echo "$ICON_DISK Appdata: $APPDATA_PATH ($APPDATA_SIZE_H)"
|
||||
echo "$ICON_SYNC Archive: ${ARCHIVE_NAME:-DRY RUN} ${ARCHIVE_SIZE:+($ARCHIVE_SIZE)}"
|
||||
echo "$ICON_SHIELD Verified: $([[ "$ARCHIVE_VERIFIED" == true ]] && echo "✅" || echo "❌ FAILED")"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no changes made"
|
||||
elif [[ "$ARCHIVE_VERIFIED" == true && "$RESTART_OK" == true ]]; then
|
||||
echo "$ICON_DONE Status: done — $ARCHIVE_NAME"
|
||||
elif [[ "$ARCHIVE_VERIFIED" == false ]]; then
|
||||
echo "$ICON_ERROR Status: archive verification FAILED — check backup before relying on it"
|
||||
else
|
||||
warn "Status: complete with warnings — check restart status above"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ "$ARCHIVE_VERIFIED" == false ]] && exit 1
|
||||
exit 0
|
||||
+336
@@ -0,0 +1,336 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ================================= Ramdisk Stop ===============================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Safely stops the transcode ramdisk: redirects the transcode symlink to the
|
||||
# SSD fallback before unmounting so Emby continues writing without interruption,
|
||||
# then unmounts the tmpfs and updates the state file.
|
||||
#
|
||||
# Primary use case: stopping the current ramdisk before re-running
|
||||
# ramdisk_setup.sh with new size or threshold values (setup is idempotent —
|
||||
# if the ramdisk is mounted, it skips the mount and reports status, so you
|
||||
# must stop it first to change the size).
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Executes in safe order:
|
||||
# 1. Validate — ramdisk mounted, SSD fallback exists
|
||||
# 2. Redirect symlink → SSD (Emby immediately writes to SSD instead)
|
||||
# 3. Warn if active transcode files still on ramdisk (informational — not a blocker)
|
||||
# 4. Unmount ramdisk tmpfs
|
||||
# 5. Update /tmp/transcode_state.db → current_target=TRANSCODE_SSD
|
||||
#
|
||||
# The symlink redirect happens before unmount so there is no window where Emby
|
||||
# has nowhere to write. Existing in-progress transcode files on the ramdisk are
|
||||
# lost on unmount — warn the user but proceed (this is expected for maintenance).
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Root Required
|
||||
# umount requires root.
|
||||
#
|
||||
# Single Instance Lock
|
||||
# acquire_lock prevents concurrent stop attempts.
|
||||
#
|
||||
# Mounted Check
|
||||
# Exits cleanly if ramdisk is not mounted — nothing to do.
|
||||
#
|
||||
# Symlink-First Order
|
||||
# Symlink is redirected before unmount — Emby never sees a broken path.
|
||||
#
|
||||
# transcode_manager Warning
|
||||
# Warns if transcode_manager.sh is running — it may flip the symlink back
|
||||
# to ramdisk on its next cycle. Stop transcode_manager before running this
|
||||
# if you need the SSD redirect to hold.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# host*.conf
|
||||
# HOST*_TRANSCODE_SSD SSD fallback directory — redirect target during stop.
|
||||
# Aliased by detect_hosts() → TRANSCODE_SSD.
|
||||
#
|
||||
# master.conf
|
||||
# TRANSCODE_LINK Symlink Emby uses. Must match Emby's transcode path setting.
|
||||
# RAMDISK_PATH tmpfs mount point.
|
||||
# TRANSCODE_STATE_FILE Override state file path (default: ${STATE_DIR}/transcode_state.db).
|
||||
#
|
||||
# ==============================================================================================
|
||||
# STATE FILES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# /tmp/transcode_state.db — updated to current_target=TRANSCODE_SSD after stop.
|
||||
# transcode_manager.sh reads this on its next cycle.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# ramdisk_stop.sh
|
||||
# Stop the ramdisk: redirect symlink → SSD, unmount, update state.
|
||||
#
|
||||
# ramdisk_stop.sh --dry-run
|
||||
# Show what would happen without making any changes.
|
||||
#
|
||||
# ramdisk_stop.sh --status
|
||||
# Show current mount state, symlink target, active files on ramdisk. Exit.
|
||||
#
|
||||
# ramdisk_stop.sh --log
|
||||
# Verbose output for each step.
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root — umount requires root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
acquire_lock
|
||||
|
||||
detect_hosts
|
||||
|
||||
STATE_FILE="${TRANSCODE_STATE_FILE:-${STATE_DIR:-/tmp}/transcode_state.db}"
|
||||
|
||||
log "Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
log "Ramdisk: $RAMDISK_PATH"
|
||||
log "Fallback: $TRANSCODE_SSD"
|
||||
log "Symlink: $TRANSCODE_LINK"
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_RAM Ramdisk path: $RAMDISK_PATH"
|
||||
echo "$ICON_DISK SSD fallback: $TRANSCODE_SSD"
|
||||
echo "$ICON_LINK Symlink: $TRANSCODE_LINK"
|
||||
echo ""
|
||||
|
||||
if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
|
||||
USAGE=$(df -BG "$RAMDISK_PATH" | awk 'NR==2 {print $3}')
|
||||
AVAIL=$(df -BG "$RAMDISK_PATH" | awk 'NR==2 {print $4}')
|
||||
echo " $ICON_RAM Ramdisk: mounted — $USAGE used / $AVAIL available ✅"
|
||||
FILE_COUNT=$(find "$RAMDISK_PATH" -type f 2>/dev/null | wc -l)
|
||||
echo " $ICON_RAM Active files on ramdisk: $FILE_COUNT"
|
||||
else
|
||||
echo " $ICON_RAM Ramdisk: NOT mounted"
|
||||
fi
|
||||
|
||||
if [[ -L "$TRANSCODE_LINK" ]]; then
|
||||
TARGET=$(readlink "$TRANSCODE_LINK")
|
||||
echo " $ICON_LINK Symlink: $TRANSCODE_LINK → $TARGET"
|
||||
else
|
||||
echo " $ICON_LINK Symlink: not set"
|
||||
fi
|
||||
|
||||
if [[ -f "$STATE_FILE" ]]; then
|
||||
echo ""
|
||||
echo " State file ($STATE_FILE):"
|
||||
while IFS='=' read -r key value; do
|
||||
[[ -z "$key" ]] && continue
|
||||
echo " $key = $value"
|
||||
done < "$STATE_FILE"
|
||||
else
|
||||
echo " $ICON_INFO State file: not found (ramdisk never started this boot)"
|
||||
fi
|
||||
|
||||
if pgrep -f "transcode_manager.sh" >/dev/null 2>&1; then
|
||||
echo ""
|
||||
warn "transcode_manager.sh is currently RUNNING"
|
||||
fi
|
||||
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Preflight ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Preflight ━━━"
|
||||
|
||||
# Bail if not mounted — nothing to do
|
||||
if ! mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
|
||||
warn "Ramdisk is not mounted at $RAMDISK_PATH — nothing to stop"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
log "Ramdisk is mounted ✅"
|
||||
|
||||
# Warn if transcode_manager is running — it may flip symlink back on next cycle
|
||||
if pgrep -f "transcode_manager.sh" >/dev/null 2>&1; then
|
||||
warn "transcode_manager.sh is currently RUNNING"
|
||||
warn "It may flip the symlink back to ramdisk on its next cycle"
|
||||
warn "Stop transcode_manager.sh first if you need the SSD redirect to hold"
|
||||
echo ""
|
||||
fi
|
||||
|
||||
# Confirm SSD fallback exists
|
||||
if [[ ! -d "$TRANSCODE_SSD" ]]; then
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — SSD fallback does not exist: $TRANSCODE_SSD"
|
||||
warn "DRY RUN — would create it before redirecting symlink"
|
||||
else
|
||||
warn "SSD fallback does not exist — creating: $TRANSCODE_SSD"
|
||||
mkdir -p "$TRANSCODE_SSD" || {
|
||||
error "Failed to create SSD fallback: $TRANSCODE_SSD"
|
||||
error "Cannot safely redirect symlink — aborting"
|
||||
exit 1
|
||||
}
|
||||
log "SSD fallback created ✅"
|
||||
fi
|
||||
else
|
||||
log "SSD fallback exists: $TRANSCODE_SSD ✅"
|
||||
fi
|
||||
|
||||
START=$(date +%s)
|
||||
STOP_SUCCESS=true
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Redirect Symlink → SSD ━━━
|
||||
# ==============================================================================================
|
||||
# Redirect BEFORE unmount — Emby continues writing to SSD with no broken path window.
|
||||
echo ""
|
||||
echo "━━━ $ICON_LINK Redirect Symlink → SSD ━━━"
|
||||
|
||||
if [[ -L "$TRANSCODE_LINK" ]]; then
|
||||
CURRENT_TARGET=$(readlink "$TRANSCODE_LINK")
|
||||
if [[ "$CURRENT_TARGET" == "$TRANSCODE_SSD" ]]; then
|
||||
echo "Symlink already points to SSD — no change needed ✅"
|
||||
else
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would redirect: $TRANSCODE_LINK → $TRANSCODE_SSD"
|
||||
else
|
||||
ln -sfn "$TRANSCODE_SSD" "$TRANSCODE_LINK" && \
|
||||
warn "Symlink redirected: $TRANSCODE_LINK → $TRANSCODE_SSD ✅" || {
|
||||
error "Failed to redirect symlink"
|
||||
STOP_SUCCESS=false
|
||||
}
|
||||
fi
|
||||
fi
|
||||
elif [[ -e "$TRANSCODE_LINK" ]]; then
|
||||
warn "$TRANSCODE_LINK exists but is not a symlink — leaving as-is"
|
||||
else
|
||||
warn "Symlink $TRANSCODE_LINK does not exist — nothing to redirect"
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Active Files Warning ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_RAM Active Files Check ━━━"
|
||||
|
||||
FILE_COUNT=$(find "$RAMDISK_PATH" -type f 2>/dev/null | wc -l)
|
||||
if [[ "$FILE_COUNT" -gt 0 ]]; then
|
||||
warn "⚠️ $FILE_COUNT file(s) still on ramdisk — will be lost on unmount"
|
||||
warn "Active transcode sessions should be stopped before unmounting"
|
||||
warn "Proceeding regardless (this is expected for maintenance)"
|
||||
if [[ "$LOG" == true ]]; then
|
||||
find "$RAMDISK_PATH" -type f 2>/dev/null | while read -r f; do
|
||||
log " $f"
|
||||
done
|
||||
fi
|
||||
else
|
||||
log "No active files on ramdisk ✅"
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Unmount ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_RAM Unmount Ramdisk ━━━"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would unmount: $RAMDISK_PATH"
|
||||
else
|
||||
if umount "$RAMDISK_PATH" 2>/dev/null; then
|
||||
warn "Ramdisk unmounted: $RAMDISK_PATH ✅"
|
||||
else
|
||||
# Regular unmount failed — check if only directory handles are open (no active writes)
|
||||
OPEN_FILES=$(lsof +D "$RAMDISK_PATH" 2>/dev/null | awk 'NR>1 && $5 != "DIR"' | wc -l)
|
||||
if [[ "$OPEN_FILES" -eq 0 ]]; then
|
||||
warn "Busy — only directory handles open, no active writes — trying lazy unmount"
|
||||
if umount -l "$RAMDISK_PATH"; then
|
||||
warn "Ramdisk lazy-unmounted: $RAMDISK_PATH ✅"
|
||||
warn "Handles will release when owning processes next check the directory"
|
||||
else
|
||||
error "Lazy unmount also failed — $RAMDISK_PATH"
|
||||
STOP_SUCCESS=false
|
||||
fi
|
||||
else
|
||||
error "Failed to unmount $RAMDISK_PATH — $OPEN_FILES file(s) still open for writing"
|
||||
error "Stop active transcode sessions and retry"
|
||||
STOP_SUCCESS=false
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Update State File ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR State File ━━━"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would update $STATE_FILE: current_target=$TRANSCODE_SSD"
|
||||
elif [[ "$STOP_SUCCESS" == true ]]; then
|
||||
NOW=$(date +%s)
|
||||
cat > "$STATE_FILE" <<EOF
|
||||
current_target=$TRANSCODE_SSD
|
||||
last_flip_time=$NOW
|
||||
flip_count_hour=0
|
||||
flip_hour_start=$NOW
|
||||
EOF
|
||||
log "State file updated: current_target=$TRANSCODE_SSD"
|
||||
else
|
||||
warn "Skipping state file update — stop had errors"
|
||||
fi
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY RAMDISK STOP SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_RAM Ramdisk: $RAMDISK_PATH"
|
||||
echo "$ICON_DISK Fallback: $TRANSCODE_SSD"
|
||||
echo "$ICON_LINK Symlink: $TRANSCODE_LINK → $(readlink "$TRANSCODE_LINK" 2>/dev/null || echo "not set")"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no changes made"
|
||||
elif [[ "$STOP_SUCCESS" == true ]]; then
|
||||
echo "$ICON_DONE Status: done ✅"
|
||||
echo "Run ramdisk_setup.sh to remount with new configuration"
|
||||
else
|
||||
echo "$ICON_ERROR Status: STOP HAD ERRORS"
|
||||
notify "Ramdisk stop errors on $(hostname) ($MY_ID) — check output" \
|
||||
"Ramdisk Stop" "warning"
|
||||
exit 1
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
+676
@@ -0,0 +1,676 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ============================= Partnership Onboard ============================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Runs once on both servers to establish a new partnership. Role is detected
|
||||
# automatically via detect_hosts() — no flags needed to declare which side you are.
|
||||
# Run on the mirror first (generates its SSH key), then on the owner to complete
|
||||
# setup remotely.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# MIRROR PATH (1 step)
|
||||
# Step 1: SSH key setup — generate keypair, copy to owner, update conf
|
||||
# Owner completes the rest remotely. Mirror is done.
|
||||
#
|
||||
# OWNER PATH (10 steps)
|
||||
# Step 1: SSH key setup — generate keypair, install on mirror, update conf
|
||||
# Step 2: Stop mirror auth — stop mirror's existing auth containers before replacing
|
||||
# Step 3: Deploy auth stack — push XMLs, pull images, create + start on mirror
|
||||
# Mariadb/Redis health-checked before Authelia deploys
|
||||
# Step 4: Stop mirror arr — stop mirror's existing arr containers before replacing
|
||||
# Step 5: Deploy arr stack — push arr XMLs, pull images, create + start on mirror
|
||||
# Step 6: Stop mirror services — stop mirror's existing services containers before replacing
|
||||
# Step 7: Deploy services stack — push Emby/Jellyfin/Seerr XMLs, pull images, create + start
|
||||
# Step 8: Partnership onboard — configure WebUIs → owner IP, write state, Emby
|
||||
# Step 9: Arr bootstrap — bidirectional library sync (arr_sync.sh)
|
||||
# Step 10: Conf push — push master.conf + setup state to all listed hosts
|
||||
#
|
||||
# ==============================================================================================
|
||||
# DESIGN PRINCIPLES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Credentials never in SSH command strings
|
||||
# Auth stack containers hold API keys, DB passwords, etc. The deploy script is written
|
||||
# locally, SCPed to the remote, and executed there. Command-line args are never used
|
||||
# to pass credentials — they'd appear in `ps` output and shell history on both servers.
|
||||
#
|
||||
# XML templates are the single source of truth for deployed containers
|
||||
# The owner's templates-user/ XMLs define every container deployed on the mirror.
|
||||
# The same XMLs that Unraid's Docker Manager uses are what get SCPed — the mirror's
|
||||
# Docker Manager can manage the containers after onboard without additional config.
|
||||
#
|
||||
# Dependency ordering in the auth stack is owner-enforced
|
||||
# PARTNERSHIP_AUTH_STACK order matters: Mariadb and Redis must come before Authelia.
|
||||
# The array is ordered correctly in host1.conf. After each Mariadb/Redis deploy,
|
||||
# the script waits for the container to be healthy before continuing. This is a remote
|
||||
# health check — the container must be running (or report healthy) before the next
|
||||
# dependent is deployed.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Root check
|
||||
# All operations run as root — SSH key management, docker operations, conf updates.
|
||||
#
|
||||
# SSH timeout on all remote calls
|
||||
# Every ssh/scp call uses SSH_TIMEOUT. No operation hangs indefinitely on a
|
||||
# slow or unreachable mirror.
|
||||
#
|
||||
# --dry-run shows exact actions without executing
|
||||
# Every step prints what it would do. SCP, deploy, plugin install, arr sync —
|
||||
# all dry-run safe.
|
||||
#
|
||||
# Step skip flags for partial re-runs
|
||||
# --skip-ssh, --skip-auth-stack, --skip-arr-stack, --skip-arr-sync allow
|
||||
# resuming after a partial failure without re-running completed steps.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# host*.conf
|
||||
#
|
||||
# HOST*_PARTNERSHIP_AUTH_STACK
|
||||
# XML filenames (from this server's templates-user/) to push and deploy on the
|
||||
# mirror as its auth stack. Order matters: database deps before Authelia.
|
||||
# Aliased by detect_hosts() → PARTNERSHIP_AUTH_STACK
|
||||
#
|
||||
# HOST*_PARTNERSHIP_REPLACE_CONTAINERS
|
||||
# Containers to stop on the mirror before deploying the auth stack.
|
||||
# Defined in the MIRROR's own conf (host*.conf on HOST2) — never in HOST1's conf.
|
||||
# Read live from the mirror via SSH during Step 3 (sources mirror's load_config.sh at
|
||||
# the same $SCRIPTS_ROOT path — convention: both servers use the same repo location).
|
||||
# Leave empty on HOST2 if no conflicting containers exist (fresh mirror: nothing to stop).
|
||||
# Aliased by detect_hosts() → PARTNERSHIP_REPLACE_CONTAINERS (on the mirror)
|
||||
#
|
||||
# HOST*_PARTNERSHIP_ARR_STACK
|
||||
# XML filenames to push and deploy on the mirror as its arr stack.
|
||||
# Leave empty to skip arr stack deploy.
|
||||
# Aliased by detect_hosts() → PARTNERSHIP_ARR_STACK
|
||||
#
|
||||
# HOST*_PARTNERSHIP_ARR_REPLACE_CONTAINERS
|
||||
# Arr containers to stop on the mirror before deploying the arr stack.
|
||||
# Same rule as PARTNERSHIP_REPLACE_CONTAINERS: defined in mirror's own conf, never HOST1's.
|
||||
# Aliased by detect_hosts() → PARTNERSHIP_ARR_REPLACE_CONTAINERS (on the mirror)
|
||||
#
|
||||
# HOST*_PARTNERSHIP_SERVICES_STACK
|
||||
# XML filenames to push and deploy on the mirror as its shared services stack.
|
||||
# Includes Emby, Jellyfin, Seerr, SeerrFin. Leave empty to skip services stack deploy.
|
||||
# Aliased by detect_hosts() → PARTNERSHIP_SERVICES_STACK
|
||||
#
|
||||
# HOST*_PARTNERSHIP_SERVICES_REPLACE_CONTAINERS
|
||||
# Services containers to stop on the mirror before deploying the services stack.
|
||||
# Same rule as PARTNERSHIP_REPLACE_CONTAINERS: defined in mirror's own conf, never HOST1's.
|
||||
# Aliased by detect_hosts() → PARTNERSHIP_SERVICES_REPLACE_CONTAINERS (on the mirror)
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Partnership/partnership_onboard.sh
|
||||
# Full onboard — role detected automatically
|
||||
#
|
||||
# Partnership/partnership_onboard.sh --dry-run
|
||||
# Preview all steps without making changes
|
||||
#
|
||||
# Partnership/partnership_onboard.sh --log
|
||||
# Verbose per-step output
|
||||
#
|
||||
# Partnership/partnership_onboard.sh --skip-ssh
|
||||
# Skip SSH key setup (key already in place)
|
||||
#
|
||||
# Partnership/partnership_onboard.sh --skip-auth-stack
|
||||
# Skip auth stack stop + deploy (Steps 3-4)
|
||||
#
|
||||
# Partnership/partnership_onboard.sh --skip-arr-stack
|
||||
# Skip arr stack stop + deploy (Steps 4-5)
|
||||
#
|
||||
# Partnership/partnership_onboard.sh --skip-services-stack
|
||||
# Skip services stack stop + deploy (Steps 6-7)
|
||||
#
|
||||
# Partnership/partnership_onboard.sh --skip-arr-sync
|
||||
# Skip arr library bootstrap (Step 9)
|
||||
#
|
||||
# Partnership/partnership_onboard.sh --phase1-only
|
||||
# OWNER only: SSH key exchange + conf push. Safe to run before HOST2 has Varaverk.
|
||||
# Writes HOST2_PHASE1_DONE=true to varaverk_setup.db.
|
||||
#
|
||||
# Partnership/partnership_onboard.sh --phase2-only
|
||||
# OWNER only: container deploy + arr + onboard (skips SSH). Triggered automatically
|
||||
# by HOST2 after it completes its Mirror-path onboard. Can also be run manually.
|
||||
# Writes HOST2_PHASE2_DONE=true to varaverk_setup.db.
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
SCRIPTS_ROOT="$SCRIPT_DIR/.."
|
||||
SSH_TIMEOUT=15
|
||||
|
||||
source "$SCRIPTS_ROOT/load_config.sh"
|
||||
source "$SCRIPTS_ROOT/Plugin/$PLATFORM/Partnership/containers.sh"
|
||||
|
||||
# ── Parse flags ───────────────────────────────────────────────────────────────────────────────
|
||||
SKIP_SSH=false
|
||||
SKIP_AUTH_STACK=false
|
||||
SKIP_ARR_STACK=false
|
||||
SKIP_SERVICES_STACK=false
|
||||
SKIP_ARR_SYNC=false
|
||||
PHASE1_ONLY=false # OWNER: SSH + conf push only (HOST2 not yet installed)
|
||||
PHASE2_ONLY=false # OWNER: containers/arr/onboard only (triggered by HOST2 after it onboards)
|
||||
FILTERED_ARGS=()
|
||||
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--skip-ssh) SKIP_SSH=true ;;
|
||||
--skip-auth-stack) SKIP_AUTH_STACK=true ;;
|
||||
--skip-arr-stack) SKIP_ARR_STACK=true ;;
|
||||
--skip-services-stack) SKIP_SERVICES_STACK=true ;;
|
||||
--skip-arr-sync) SKIP_ARR_SYNC=true ;;
|
||||
--phase1-only) PHASE1_ONLY=true ;;
|
||||
--phase2-only) PHASE2_ONLY=true; SKIP_SSH=true ;;
|
||||
*) FILTERED_ARGS+=("$arg") ;;
|
||||
esac
|
||||
done
|
||||
|
||||
parse_args "${FILTERED_ARGS[@]}"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
[[ "$EUID" -ne 0 ]] && { error "Must be run as root"; exit 1; }
|
||||
|
||||
acquire_lock
|
||||
|
||||
if ! command -v docker &>/dev/null; then
|
||||
error "Docker command not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
detect_hosts
|
||||
|
||||
OWNER_ID="${PARTNERSHIP_OWNER_HOST:-HOST1}"
|
||||
MIRROR_ID=$( [[ "$OWNER_ID" == "HOST1" ]] && echo "HOST2" || echo "HOST1" )
|
||||
OWNER="${!OWNER_ID}"
|
||||
MIRROR="${!MIRROR_ID}"
|
||||
# SSH_KEY (set by detect_hosts) is this server's own private key.
|
||||
# The remote accepts it because this server's PUBLIC key was installed there via ssh_setup.sh.
|
||||
# HOST{N}_SSH_KEY lives in host{N}.conf — with sparse checkout, the other server's
|
||||
# conf is never present here. Always use SSH_KEY (local private key) for outbound SSH.
|
||||
MIRROR_SSH_KEY="$SSH_KEY"
|
||||
|
||||
AM_OWNER=false
|
||||
AM_MIRROR=false
|
||||
[[ "$MY_ID" == "$OWNER_ID" ]] && AM_OWNER=true
|
||||
[[ "$MY_ID" == "$MIRROR_ID" ]] && AM_MIRROR=true
|
||||
|
||||
EXTRA_FLAGS=()
|
||||
[[ "$DRY_RUN" == true ]] && EXTRA_FLAGS+=("--dry-run")
|
||||
[[ "$LOG_MODE" == true ]] && EXTRA_FLAGS+=("--log")
|
||||
|
||||
START=$(date +%s)
|
||||
|
||||
# ── Helper: write phase completion flag to setup.db + push to remotes ─────────────────────────
|
||||
write_onboard_phase() {
|
||||
local target_id="$1" phase="$2"
|
||||
local key="${target_id}_PHASE${phase}_DONE"
|
||||
local state_file="$(platform_setup_db_path)"
|
||||
[[ "$DRY_RUN" == true ]] && { warn "DRY RUN — would write ${key}=true"; return 0; }
|
||||
if grep -q "^${key}=" "$state_file" 2>/dev/null; then
|
||||
sed -i "s|^${key}=.*|${key}=true|" "$state_file"
|
||||
else
|
||||
echo "${key}=true" >> "$state_file"
|
||||
fi
|
||||
platform_push_setup_state
|
||||
}
|
||||
|
||||
echo ""
|
||||
echo "━━━ $ICON_FALLBACK Partnership Onboard — $MY_ID ($LOCAL_SERVER_NAME) — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo ""
|
||||
echo " Role: $( [[ "$AM_OWNER" == true ]] && echo "OWNER" || echo "MIRROR" )"
|
||||
echo " This: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo " Partner: $( [[ "$AM_OWNER" == true ]] && echo "$MIRROR_ID ($MIRROR)" || echo "$OWNER_ID ($OWNER)" )"
|
||||
echo ""
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no permanent changes will be made"
|
||||
|
||||
# ==============================================================================================
|
||||
# ── HELPER: stop containers on the mirror by reading its own conf via SSH ────────────────────
|
||||
#
|
||||
# SSHes to the mirror, sources its load_config.sh at the same $SCRIPTS_ROOT path (both servers
|
||||
# use the same convention), and reads the named config array from the mirror's own conf.
|
||||
# HOST2's container list stays in HOST2's host2.conf — not duplicated in HOST1's conf.
|
||||
# Fails gracefully if scripts aren't present yet or the array is empty (nothing to stop).
|
||||
#
|
||||
# deploy_container_from_xml() already stops/removes containers with the same name as what's
|
||||
# being deployed. This step handles containers with DIFFERENT names that conflict.
|
||||
# ==============================================================================================
|
||||
stop_mirror_stack() {
|
||||
local config_var="$1" label="$2"
|
||||
local -a to_stop=()
|
||||
|
||||
mapfile -t to_stop < <(
|
||||
timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$MIRROR_IP" \
|
||||
"source '$SCRIPTS_ROOT/load_config.sh' 2>/dev/null
|
||||
detect_hosts 2>/dev/null
|
||||
printf '%s\n' \"\${${config_var}[@]:-}\"" 2>/dev/null | grep -v '^$'
|
||||
)
|
||||
|
||||
if [[ ${#to_stop[@]} -eq 0 ]]; then
|
||||
log "No $label containers to stop on $MIRROR — skipping"
|
||||
return 0
|
||||
fi
|
||||
|
||||
log "Stopping $label on $MIRROR: ${to_stop[*]}"
|
||||
for container in "${to_stop[@]}"; do
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn " DRY RUN — would stop + rm $container on $MIRROR"
|
||||
continue
|
||||
fi
|
||||
timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" root@"$MIRROR_IP" \
|
||||
"docker stop '$container' 2>/dev/null
|
||||
docker rm '$container' 2>/dev/null && echo removed" 2>/dev/null | \
|
||||
grep -q removed && \
|
||||
log " $container removed ✅" || \
|
||||
log " $container not found on $MIRROR — skipping"
|
||||
done
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ── MIRROR PATH ───────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
if [[ "$AM_MIRROR" == true ]]; then
|
||||
echo "━━━ Step 1/2 — SSH Key Setup (Mirror) ━━━"
|
||||
echo ""
|
||||
echo " Mirror sets up SSH keys, then notifies Owner to run Phase 2."
|
||||
echo ""
|
||||
|
||||
if [[ "$SKIP_SSH" == true ]]; then
|
||||
warn "Skipping SSH setup (--skip-ssh)"
|
||||
elif bash "$SCRIPT_DIR/ssh_setup.sh" "${EXTRA_FLAGS[@]}"; then
|
||||
log "SSH key ready ✅"
|
||||
else
|
||||
error "SSH key setup failed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "━━━ Step 2/2 — Notify Owner to Run Phase 2 ━━━"
|
||||
echo ""
|
||||
|
||||
OWNER_IP=$(resolve_tailscale_ip "$OWNER" 2>/dev/null || true)
|
||||
PHASE2_TRIGGERED=false
|
||||
|
||||
if [[ -n "$OWNER_IP" ]]; then
|
||||
# Read OWNER's SCRIPTS_DIR via platform probe command — don't assume same path as mirror
|
||||
_probe_cmd=$(platform_scripts_dir_probe_cmd)
|
||||
OWNER_SCRIPTS_DIR=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$OWNER_IP" \
|
||||
"$_probe_cmd" 2>/dev/null | tr -d '[:space:]')
|
||||
OWNER_SCRIPTS_DIR="${OWNER_SCRIPTS_DIR:-$SCRIPTS_DIR}"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would SSH to $OWNER ($OWNER_IP) and trigger Phase 2"
|
||||
PHASE2_TRIGGERED=true
|
||||
elif timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$OWNER_IP" \
|
||||
"nohup bash '${OWNER_SCRIPTS_DIR}/Partnership/partnership_onboard.sh' --phase2-only > /tmp/vv_phase2_onboard.log 2>&1 & echo triggered" \
|
||||
2>/dev/null | grep -q triggered; then
|
||||
log "Phase 2 triggered on $OWNER ✅"
|
||||
log "Watch progress on $OWNER: tail -f /tmp/vv_phase2_onboard.log"
|
||||
PHASE2_TRIGGERED=true
|
||||
else
|
||||
warn "Could not auto-trigger Phase 2 on $OWNER"
|
||||
fi
|
||||
else
|
||||
warn "Cannot resolve $OWNER Tailscale IP"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY MIRROR SETUP COMPLETE ━━━━━"
|
||||
echo " SSH key: ready"
|
||||
echo " Phase 2 on $OWNER: $( [[ "$PHASE2_TRIGGERED" == true ]] && echo "triggered ✅" || echo "needs manual trigger ⚠" )"
|
||||
if [[ "$PHASE2_TRIGGERED" == false ]]; then
|
||||
echo ""
|
||||
echo " Run manually on $OWNER:"
|
||||
echo " bash Partnership/partnership_onboard.sh --phase2-only"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ── OWNER PATH ────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
MIRROR_IP=$(resolve_tailscale_ip "$MIRROR")
|
||||
[[ -z "$MIRROR_IP" ]] && { error "Cannot resolve $MIRROR Tailscale IP — is Tailscale running?"; exit 1; }
|
||||
log "Mirror: $MIRROR ($MIRROR_IP)"
|
||||
[[ "$PHASE1_ONLY" == true ]] && log "Mode: Phase 1 only (SSH + conf push)"
|
||||
[[ "$PHASE2_ONLY" == true ]] && log "Mode: Phase 2 only (containers + arr + onboard)"
|
||||
echo ""
|
||||
|
||||
STEP_SSH_OK=false
|
||||
STEP_STOP_AUTH_OK=true
|
||||
STEP_AUTH_OK=true
|
||||
AUTH_DEPLOYED=0
|
||||
AUTH_FAILED=0
|
||||
STEP_STOP_ARR_OK=true
|
||||
STEP_ARR_OK=true
|
||||
ARR_DEPLOYED=0
|
||||
ARR_FAILED=0
|
||||
STEP_STOP_SERVICES_OK=true
|
||||
STEP_SERVICES_OK=true
|
||||
SERVICES_DEPLOYED=0
|
||||
SERVICES_FAILED=0
|
||||
ONBOARD_OK=false
|
||||
ARR_SYNC_OK=false
|
||||
MASTER_PUSH_OK=false
|
||||
|
||||
# ── Step 1: SSH ───────────────────────────────────────────────────────────────────────────────
|
||||
# Skipped when --phase2-only (SSH was already done in Phase 1).
|
||||
echo "━━━ Step 1 — SSH Key Setup ━━━"
|
||||
|
||||
if [[ "$SKIP_SSH" == true ]]; then
|
||||
warn "Skipping (--skip-ssh)"
|
||||
STEP_SSH_OK=true
|
||||
elif [[ "$PHASE1_ONLY" == true ]]; then
|
||||
# Phase 1 in background: test if SSH already works first — avoids ssh-copy-id
|
||||
# hanging for a password prompt with no TTY.
|
||||
if timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$MIRROR_IP" exit 0 2>/dev/null; then
|
||||
log "SSH to $MIRROR already works ✅ — skipping key install"
|
||||
STEP_SSH_OK=true
|
||||
else
|
||||
# Key not yet on HOST2 — try ssh_setup.sh (works interactively, may fail in background)
|
||||
if bash "$SCRIPT_DIR/ssh_setup.sh" "${EXTRA_FLAGS[@]}"; then
|
||||
log "SSH keys ready ✅"
|
||||
STEP_SSH_OK=true
|
||||
else
|
||||
# Soft-fail: generate key locally if not present, then tell user to install manually
|
||||
warn "Could not install key on $MIRROR automatically (no terminal for password prompt)"
|
||||
if [[ -f "$SSH_KEY" ]]; then
|
||||
log "Local key exists at: $SSH_KEY"
|
||||
else
|
||||
bash "$SCRIPT_DIR/ssh_setup.sh" --key-only "${EXTRA_FLAGS[@]}" 2>/dev/null || true
|
||||
fi
|
||||
if [[ -f "${SSH_KEY}.pub" ]]; then
|
||||
echo ""
|
||||
echo " Install this key on $MIRROR to complete SSH setup:"
|
||||
echo " ┌─────────────────────────────────────────────────────"
|
||||
cat "${SSH_KEY}.pub" | sed 's/^/ │ /'
|
||||
echo " └─────────────────────────────────────────────────────"
|
||||
echo " Run on a terminal: ssh-copy-id -i ${SSH_KEY}.pub root@${MIRROR_IP}"
|
||||
echo " Then click 'Push Conf' in the Partnership tab."
|
||||
# Write key-ready flag so UI can show the manual-install state
|
||||
[[ "$DRY_RUN" == false ]] && {
|
||||
local kflag="${MIRROR_ID}_KEY_READY"
|
||||
local _setup_f="$(platform_setup_db_path)"
|
||||
grep -q "^${kflag}=" "$_setup_f" 2>/dev/null \
|
||||
&& sed -i "s|^${kflag}=.*|${kflag}=true|" "$_setup_f" \
|
||||
|| echo "${kflag}=true" >> "$_setup_f"
|
||||
}
|
||||
fi
|
||||
STEP_SSH_OK=false
|
||||
fi
|
||||
fi
|
||||
elif bash "$SCRIPT_DIR/ssh_setup.sh" "${EXTRA_FLAGS[@]}"; then
|
||||
log "SSH keys ready ✅"
|
||||
STEP_SSH_OK=true
|
||||
else
|
||||
error "SSH key setup failed — aborting"
|
||||
error "Re-run or use --skip-ssh if key is already set up"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ── Phase 1 exit point ────────────────────────────────────────────────────────────────────────
|
||||
# --phase1-only: SSH + conf push is all HOST1 needs to do before HOST2 installs Varaverk.
|
||||
# HOST2's wizard will detect the pushed master.conf + state file and take the correct path.
|
||||
if [[ "$PHASE1_ONLY" == true ]]; then
|
||||
if [[ "$STEP_SSH_OK" == false ]]; then
|
||||
# SSH key not yet installed on HOST2 — can't push conf, but local setup still runs.
|
||||
# UI will show "key ready, install manually" state via HOST2_KEY_READY flag.
|
||||
echo ""
|
||||
echo "━━━ Phase 1 — HOST1 Local Setup (SSH pending) ━━━"
|
||||
bash "$SCRIPT_DIR/partnership_manager.sh" --onboard --local-only "${EXTRA_FLAGS[@]}" || \
|
||||
warn "Local setup had issues — check partnership_manager.sh output above"
|
||||
|
||||
END=$(date +%s)
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY PHASE 1 — SSH PENDING ━━━━━"
|
||||
echo " SSH keys: key generated ✅ — NOT yet installed on $MIRROR ⚠"
|
||||
echo " Conf push: skipped (needs SSH access to $MIRROR)"
|
||||
echo " HOST1 setup: done ✅"
|
||||
echo " Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
echo " ACTION NEEDED: install the key on $MIRROR:"
|
||||
echo " ssh-copy-id -i ${SSH_KEY}.pub root@${MIRROR_IP}"
|
||||
echo " Then click 'Push Conf' in Partnership tab, or run:"
|
||||
echo " bash Partnership/partnership_onboard.sh --phase1-only --skip-ssh"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "━━━ Phase 1 — Conf Push ━━━"
|
||||
|
||||
CONF_PUSH_OK=false
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would push master.conf + state file to $MIRROR"
|
||||
CONF_PUSH_OK=true
|
||||
else
|
||||
push_output=$(platform_push_conf)
|
||||
push_rc=$?
|
||||
[[ -n "$push_output" ]] && echo "$push_output"
|
||||
platform_push_setup_state
|
||||
if [[ $push_rc -eq 0 ]]; then
|
||||
log "Conf push complete ✅"
|
||||
CONF_PUSH_OK=true
|
||||
else
|
||||
warn "Conf push had failures — retry via Scheduler → master.conf → Save Conf"
|
||||
fi
|
||||
fi
|
||||
|
||||
# HOST1 local setup — runs immediately without needing HOST2
|
||||
echo ""
|
||||
echo "━━━ Phase 1 — HOST1 Local Setup ━━━"
|
||||
bash "$SCRIPT_DIR/partnership_manager.sh" --onboard --local-only "${EXTRA_FLAGS[@]}" || \
|
||||
warn "Local setup had issues — check partnership_manager.sh output above"
|
||||
|
||||
[[ "$DRY_RUN" == false ]] && write_onboard_phase "$MIRROR_ID" 1
|
||||
|
||||
END=$(date +%s)
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY PHASE 1 COMPLETE ━━━━━"
|
||||
echo " SSH keys: $( [[ "$STEP_SSH_OK" == true ]] && echo "ready ✅" || echo "skipped" )"
|
||||
echo " Conf push: $( [[ "$CONF_PUSH_OK" == true ]] && echo "done ✅" || echo "⚠ manual needed" )"
|
||||
echo " HOST1 setup: done ✅"
|
||||
echo " Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
echo " HOST1 is fully set up. HOST2 ($MIRROR) can now install the Varaverk plugin."
|
||||
echo " The wizard will detect the pushed conf and take the correct path."
|
||||
echo " When HOST2 completes its onboard, it will automatically trigger Phase 2 here."
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ── Step 2: Stop mirror's existing auth stack ─────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ Step 2 — Stop Mirror Auth Stack ━━━"
|
||||
|
||||
if [[ "$SKIP_AUTH_STACK" == true ]]; then
|
||||
warn "Skipping (--skip-auth-stack)"
|
||||
else
|
||||
stop_mirror_stack "PARTNERSHIP_REPLACE_CONTAINERS" "auth stack"
|
||||
fi
|
||||
|
||||
# ── Step 4: Deploy auth stack on mirror ───────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ Step 3 — Deploy Auth Stack on Mirror ━━━"
|
||||
|
||||
if [[ "$SKIP_AUTH_STACK" == true ]]; then
|
||||
warn "Skipping (--skip-auth-stack)"
|
||||
elif [[ ${#PARTNERSHIP_AUTH_STACK[@]} -eq 0 ]]; then
|
||||
warn "PARTNERSHIP_AUTH_STACK not set in ${MY_ID} conf — skipping auth stack deploy"
|
||||
warn "Add HOST${MY_ID: -1}_PARTNERSHIP_AUTH_STACK to host${MY_ID: -1}.conf"
|
||||
STEP_AUTH_OK=false
|
||||
else
|
||||
deploy_xml_stack PARTNERSHIP_AUTH_STACK
|
||||
AUTH_DEPLOYED=$_STACK_DEPLOYED
|
||||
AUTH_FAILED=$_STACK_FAILED
|
||||
echo "Auth stack: $AUTH_DEPLOYED deployed, $AUTH_FAILED failed"
|
||||
[[ "$AUTH_FAILED" -gt 0 ]] && STEP_AUTH_OK=false
|
||||
fi
|
||||
|
||||
# ── Step 5: Stop mirror's existing arr stack ──────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ Step 4 — Stop Mirror Arr Stack ━━━"
|
||||
|
||||
if [[ "$SKIP_ARR_STACK" == true ]]; then
|
||||
warn "Skipping (--skip-arr-stack)"
|
||||
elif [[ ${#PARTNERSHIP_ARR_STACK[@]} -eq 0 ]]; then
|
||||
log "PARTNERSHIP_ARR_STACK not configured — skipping arr stack deploy"
|
||||
SKIP_ARR_STACK=true
|
||||
else
|
||||
stop_mirror_stack "PARTNERSHIP_ARR_REPLACE_CONTAINERS" "arr stack"
|
||||
fi
|
||||
|
||||
# ── Step 5: Deploy arr stack on mirror ───────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ Step 5 — Deploy Arr Stack on Mirror ━━━"
|
||||
|
||||
if [[ "$SKIP_ARR_STACK" == true ]]; then
|
||||
warn "Skipping (--skip-arr-stack)"
|
||||
else
|
||||
deploy_xml_stack PARTNERSHIP_ARR_STACK
|
||||
ARR_DEPLOYED=$_STACK_DEPLOYED
|
||||
ARR_FAILED=$_STACK_FAILED
|
||||
echo "Arr stack: $ARR_DEPLOYED deployed, $ARR_FAILED failed"
|
||||
[[ "$ARR_FAILED" -gt 0 ]] && STEP_ARR_OK=false
|
||||
fi
|
||||
|
||||
# ── Step 6: Stop mirror's existing services stack ─────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ Step 6 — Stop Mirror Services Stack ━━━"
|
||||
|
||||
if [[ "$SKIP_SERVICES_STACK" == true ]]; then
|
||||
warn "Skipping (--skip-services-stack)"
|
||||
elif [[ ${#PARTNERSHIP_SERVICES_STACK[@]} -eq 0 ]]; then
|
||||
log "PARTNERSHIP_SERVICES_STACK not configured — skipping services stack deploy"
|
||||
SKIP_SERVICES_STACK=true
|
||||
else
|
||||
stop_mirror_stack "PARTNERSHIP_SERVICES_REPLACE_CONTAINERS" "services stack"
|
||||
fi
|
||||
|
||||
# ── Step 7: Deploy services stack on mirror ───────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ Step 7 — Deploy Services Stack on Mirror ━━━"
|
||||
|
||||
if [[ "$SKIP_SERVICES_STACK" == true ]]; then
|
||||
warn "Skipping (--skip-services-stack)"
|
||||
else
|
||||
deploy_xml_stack PARTNERSHIP_SERVICES_STACK
|
||||
SERVICES_DEPLOYED=$_STACK_DEPLOYED
|
||||
SERVICES_FAILED=$_STACK_FAILED
|
||||
echo "Services stack: $SERVICES_DEPLOYED deployed, $SERVICES_FAILED failed"
|
||||
[[ "$SERVICES_FAILED" -gt 0 ]] && STEP_SERVICES_OK=false
|
||||
fi
|
||||
|
||||
# ── Step 8: Partnership onboard ───────────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ Step 8 — Partnership Onboard ━━━"
|
||||
|
||||
if bash "$SCRIPTS_ROOT/Partnership/partnership_manager.sh" --onboard "${EXTRA_FLAGS[@]}"; then
|
||||
echo "Partnership onboard complete ✅"
|
||||
ONBOARD_OK=true
|
||||
else
|
||||
error "Partnership onboard failed"
|
||||
ONBOARD_OK=false
|
||||
fi
|
||||
|
||||
# ── Step 9: Arr library bootstrap ─────────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ Step 9 — Arr Library Bootstrap ━━━"
|
||||
|
||||
if [[ "$ONBOARD_OK" == false ]]; then
|
||||
warn "Skipping — onboard did not complete"
|
||||
elif [[ "$SKIP_ARR_SYNC" == true ]]; then
|
||||
warn "Skipping (--skip-arr-sync)"
|
||||
elif [[ ! -f "$SCRIPTS_ROOT/Media/arr_sync.sh" ]]; then
|
||||
warn "arr_sync.sh not found — run Media/arr_sync.sh manually once arrs are live"
|
||||
elif bash "$SCRIPTS_ROOT/Media/arr_sync.sh" "${EXTRA_FLAGS[@]}"; then
|
||||
echo "Arr bootstrap complete ✅"
|
||||
ARR_SYNC_OK=true
|
||||
else
|
||||
warn "Arr sync had errors — partnership still valid"
|
||||
warn "Re-run Media/arr_sync.sh once all arr containers are live"
|
||||
fi
|
||||
|
||||
# ── Step 10: Push master.conf to all listed hosts ─────────────────────────────────────────────
|
||||
# SSH is now established and all partners have the plugin installed.
|
||||
# Push the authoritative master.conf so every listed host is in sync immediately.
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Step 10 — master.conf Push ━━━"
|
||||
|
||||
if [[ "$ONBOARD_OK" == false ]]; then
|
||||
warn "Skipping — onboard did not complete"
|
||||
elif [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would push master.conf to all listed hosts"
|
||||
MASTER_PUSH_OK=true
|
||||
else
|
||||
push_output=$(platform_push_conf)
|
||||
push_rc=$?
|
||||
[[ -n "$push_output" ]] && echo "$push_output"
|
||||
platform_push_setup_state
|
||||
if [[ $push_rc -eq 0 ]]; then
|
||||
echo "master.conf sync complete ✅"
|
||||
MASTER_PUSH_OK=true
|
||||
else
|
||||
warn "master.conf push had failures — retry via Scheduler → master.conf → Save Conf"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Write Phase 2 completion state ────────────────────────────────────────────────────────────
|
||||
[[ "$ONBOARD_OK" == true && "$DRY_RUN" == false ]] && write_onboard_phase "$MIRROR_ID" 2
|
||||
|
||||
# ── Summary ───────────────────────────────────────────────────────────────────────────────────
|
||||
END=$(date +%s)
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY ONBOARD SUMMARY ━━━━━"
|
||||
echo " Owner: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo " Mirror: $MIRROR ($MIRROR_IP)"
|
||||
[[ "$PHASE2_ONLY" == true ]] && echo " Mode: Phase 2 (triggered by HOST2 notification)"
|
||||
echo " Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
_ok() { [[ "$1" == true ]] && echo "✅" || echo "❌"; }
|
||||
_skip() { [[ "$1" == true ]] && echo "skipped" || echo "$(_ok "$2")"; }
|
||||
|
||||
echo " Step 1 — SSH keys: $(_skip "$SKIP_SSH" "$STEP_SSH_OK")"
|
||||
echo " Step 2 — Stop auth: $(_skip "$SKIP_AUTH_STACK" "$STEP_STOP_AUTH_OK")"
|
||||
echo " Step 3 — Auth stack: $( [[ "$SKIP_AUTH_STACK" == true ]] && echo "skipped" || echo "${AUTH_DEPLOYED} deployed, ${AUTH_FAILED} failed" )"
|
||||
echo " Step 4 — Stop arr: $(_skip "$SKIP_ARR_STACK" "$STEP_STOP_ARR_OK")"
|
||||
echo " Step 5 — Arr stack: $( [[ "$SKIP_ARR_STACK" == true ]] && echo "skipped" || echo "${ARR_DEPLOYED} deployed, ${ARR_FAILED} failed" )"
|
||||
echo " Step 6 — Stop services: $(_skip "$SKIP_SERVICES_STACK" "$STEP_STOP_SERVICES_OK")"
|
||||
echo " Step 7 — Services stack: $( [[ "$SKIP_SERVICES_STACK" == true ]] && echo "skipped" || echo "${SERVICES_DEPLOYED} deployed, ${SERVICES_FAILED} failed" )"
|
||||
echo " Step 8 — Onboard: $(_ok "$ONBOARD_OK")"
|
||||
echo " Step 9 — Arr bootstrap: $( [[ "$SKIP_ARR_SYNC" == true || "$ONBOARD_OK" == false ]] && echo "skipped" || echo "$(_ok "$ARR_SYNC_OK")" )"
|
||||
echo " Step 10 — Conf push: $( [[ "$ONBOARD_OK" == false ]] && echo "skipped" || echo "$(_ok "$MASTER_PUSH_OK")" )"
|
||||
echo ""
|
||||
|
||||
if [[ "$ONBOARD_OK" == true ]]; then
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes made" || \
|
||||
echo "$ICON_DONE DONE — partnership established ✅"
|
||||
echo "Verify with: Partnership/partnership_manager.sh --status"
|
||||
else
|
||||
error "Setup incomplete — resolve errors above and re-run"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
[[ "$ONBOARD_OK" == false ]] && exit 1
|
||||
exit 0
|
||||
+697
@@ -0,0 +1,697 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ============================= Partnership Onboard ============================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Runs once on both servers to establish a new partnership. Role is detected
|
||||
# automatically via detect_hosts() — no flags needed to declare which side you are.
|
||||
# Run on the mirror first (generates its SSH key), then on the owner to complete
|
||||
# setup remotely.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# MIRROR PATH (1 step)
|
||||
# Step 1: SSH key setup — generate keypair, copy to owner, update conf
|
||||
# Owner completes the rest remotely. Mirror is done.
|
||||
#
|
||||
# OWNER PATH (10 steps)
|
||||
# Step 1: SSH key setup — generate keypair, install on mirror, update conf
|
||||
# Step 2: Stop mirror auth — stop mirror's existing auth containers before replacing
|
||||
# Step 3: Deploy auth stack — push XMLs, pull images, create + start on mirror
|
||||
# Mariadb/Redis health-checked before Authelia deploys
|
||||
# Step 4: Stop mirror arr — stop mirror's existing arr containers before replacing
|
||||
# Step 5: Deploy arr stack — push arr XMLs, pull images, create + start on mirror
|
||||
# Step 6: Stop mirror services — stop mirror's existing services containers before replacing
|
||||
# Step 7: Deploy services stack — push Emby/Jellyfin/Seerr XMLs, pull images, create + start
|
||||
# Step 8: Partnership onboard — configure WebUIs → owner IP, write state, Emby
|
||||
# Step 9: Arr bootstrap — bidirectional library sync (arr_sync.sh)
|
||||
# Step 10: Conf push — push master.conf + setup state to all listed hosts
|
||||
#
|
||||
# ==============================================================================================
|
||||
# DESIGN PRINCIPLES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Credentials never in SSH command strings
|
||||
# Auth stack containers hold API keys, DB passwords, etc. The deploy script is written
|
||||
# locally, SCPed to the remote, and executed there. Command-line args are never used
|
||||
# to pass credentials — they'd appear in `ps` output and shell history on both servers.
|
||||
#
|
||||
# XML templates are the single source of truth for deployed containers
|
||||
# The owner's templates-user/ XMLs define every container deployed on the mirror.
|
||||
# The same XMLs that Unraid's Docker Manager uses are what get SCPed — the mirror's
|
||||
# Docker Manager can manage the containers after onboard without additional config.
|
||||
#
|
||||
# Dependency ordering in the auth stack is owner-enforced
|
||||
# PARTNERSHIP_AUTH_STACK order matters: Mariadb and Redis must come before Authelia.
|
||||
# The array is ordered correctly in host1.conf. After each Mariadb/Redis deploy,
|
||||
# the script waits for the container to be healthy before continuing. This is a remote
|
||||
# health check — the container must be running (or report healthy) before the next
|
||||
# dependent is deployed.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Root check
|
||||
# All operations run as root — SSH key management, docker operations, conf updates.
|
||||
#
|
||||
# SSH timeout on all remote calls
|
||||
# Every ssh/scp call uses SSH_TIMEOUT. No operation hangs indefinitely on a
|
||||
# slow or unreachable mirror.
|
||||
#
|
||||
# --dry-run shows exact actions without executing
|
||||
# Every step prints what it would do. SCP, deploy, plugin install, arr sync —
|
||||
# all dry-run safe.
|
||||
#
|
||||
# Step skip flags for partial re-runs
|
||||
# --skip-ssh, --skip-auth-stack, --skip-arr-stack, --skip-arr-sync allow
|
||||
# resuming after a partial failure without re-running completed steps.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# host*.conf
|
||||
#
|
||||
# HOST*_PARTNERSHIP_AUTH_STACK
|
||||
# XML filenames (from this server's templates-user/) to push and deploy on the
|
||||
# mirror as its auth stack. Order matters: database deps before Authelia.
|
||||
# Aliased by detect_hosts() → PARTNERSHIP_AUTH_STACK
|
||||
#
|
||||
# HOST*_PARTNERSHIP_REPLACE_CONTAINERS
|
||||
# Containers to stop on the mirror before deploying the auth stack.
|
||||
# Defined in the MIRROR's own conf (host*.conf on HOST2) — never in HOST1's conf.
|
||||
# Read live from the mirror via SSH during Step 3 (sources mirror's load_config.sh at
|
||||
# the same $SCRIPTS_ROOT path — convention: both servers use the same repo location).
|
||||
# Leave empty on HOST2 if no conflicting containers exist (fresh mirror: nothing to stop).
|
||||
# Aliased by detect_hosts() → PARTNERSHIP_REPLACE_CONTAINERS (on the mirror)
|
||||
#
|
||||
# HOST*_PARTNERSHIP_ARR_STACK
|
||||
# XML filenames to push and deploy on the mirror as its arr stack.
|
||||
# Leave empty to skip arr stack deploy.
|
||||
# Aliased by detect_hosts() → PARTNERSHIP_ARR_STACK
|
||||
#
|
||||
# HOST*_PARTNERSHIP_ARR_REPLACE_CONTAINERS
|
||||
# Arr containers to stop on the mirror before deploying the arr stack.
|
||||
# Same rule as PARTNERSHIP_REPLACE_CONTAINERS: defined in mirror's own conf, never HOST1's.
|
||||
# Aliased by detect_hosts() → PARTNERSHIP_ARR_REPLACE_CONTAINERS (on the mirror)
|
||||
#
|
||||
# HOST*_PARTNERSHIP_SERVICES_STACK
|
||||
# XML filenames to push and deploy on the mirror as its shared services stack.
|
||||
# Includes Emby, Jellyfin, Seerr, SeerrFin. Leave empty to skip services stack deploy.
|
||||
# Aliased by detect_hosts() → PARTNERSHIP_SERVICES_STACK
|
||||
#
|
||||
# HOST*_PARTNERSHIP_SERVICES_REPLACE_CONTAINERS
|
||||
# Services containers to stop on the mirror before deploying the services stack.
|
||||
# Same rule as PARTNERSHIP_REPLACE_CONTAINERS: defined in mirror's own conf, never HOST1's.
|
||||
# Aliased by detect_hosts() → PARTNERSHIP_SERVICES_REPLACE_CONTAINERS (on the mirror)
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Partnership/partnership_onboard.sh
|
||||
# Full onboard — role detected automatically
|
||||
#
|
||||
# Partnership/partnership_onboard.sh --dry-run
|
||||
# Preview all steps without making changes
|
||||
#
|
||||
# Partnership/partnership_onboard.sh --log
|
||||
# Verbose per-step output
|
||||
#
|
||||
# Partnership/partnership_onboard.sh --skip-ssh
|
||||
# Skip SSH key setup (key already in place)
|
||||
#
|
||||
# Partnership/partnership_onboard.sh --skip-auth-stack
|
||||
# Skip auth stack stop + deploy (Steps 3-4)
|
||||
#
|
||||
# Partnership/partnership_onboard.sh --skip-arr-stack
|
||||
# Skip arr stack stop + deploy (Steps 4-5)
|
||||
#
|
||||
# Partnership/partnership_onboard.sh --skip-services-stack
|
||||
# Skip services stack stop + deploy (Steps 6-7)
|
||||
#
|
||||
# Partnership/partnership_onboard.sh --skip-arr-sync
|
||||
# Skip arr library bootstrap (Step 9)
|
||||
#
|
||||
# Partnership/partnership_onboard.sh --phase1-only
|
||||
# OWNER only: SSH key exchange + conf push. Safe to run before HOST2 has Varaverk.
|
||||
# Writes HOST2_PHASE1_DONE=true to varaverk_setup.db.
|
||||
#
|
||||
# Partnership/partnership_onboard.sh --phase2-only
|
||||
# OWNER only: container deploy + arr + onboard (skips SSH). Triggered automatically
|
||||
# by HOST2 after it completes its Mirror-path onboard. Can also be run manually.
|
||||
# Writes HOST2_PHASE2_DONE=true to varaverk_setup.db.
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
SCRIPTS_ROOT="$SCRIPT_DIR/.."
|
||||
SSH_TIMEOUT=15
|
||||
|
||||
source "$SCRIPTS_ROOT/load_config.sh"
|
||||
source "$SCRIPTS_ROOT/Plugin/$PLATFORM/Partnership/containers.sh"
|
||||
|
||||
# ── Parse flags ───────────────────────────────────────────────────────────────────────────────
|
||||
SKIP_SSH=false
|
||||
SKIP_AUTH_STACK=false
|
||||
SKIP_ARR_STACK=false
|
||||
SKIP_SERVICES_STACK=false
|
||||
SKIP_ARR_SYNC=false
|
||||
PHASE1_ONLY=false # OWNER: SSH + conf push only (HOST2 not yet installed)
|
||||
PHASE2_ONLY=false # OWNER: containers/arr/onboard only (triggered by HOST2 after it onboards)
|
||||
FILTERED_ARGS=()
|
||||
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--skip-ssh) SKIP_SSH=true ;;
|
||||
--skip-auth-stack) SKIP_AUTH_STACK=true ;;
|
||||
--skip-arr-stack) SKIP_ARR_STACK=true ;;
|
||||
--skip-services-stack) SKIP_SERVICES_STACK=true ;;
|
||||
--skip-arr-sync) SKIP_ARR_SYNC=true ;;
|
||||
--phase1-only) PHASE1_ONLY=true ;;
|
||||
--phase2-only) PHASE2_ONLY=true; SKIP_SSH=true ;;
|
||||
*) FILTERED_ARGS+=("$arg") ;;
|
||||
esac
|
||||
done
|
||||
|
||||
parse_args "${FILTERED_ARGS[@]}"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
[[ "$EUID" -ne 0 ]] && { error "Must be run as root"; exit 1; }
|
||||
|
||||
acquire_lock
|
||||
|
||||
if ! command -v docker &>/dev/null; then
|
||||
error "Docker command not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
detect_hosts
|
||||
|
||||
OWNER_ID="${PARTNERSHIP_OWNER_HOST:-HOST1}"
|
||||
MIRROR_ID=$( [[ "$OWNER_ID" == "HOST1" ]] && echo "HOST2" || echo "HOST1" )
|
||||
OWNER="${!OWNER_ID}"
|
||||
MIRROR="${!MIRROR_ID}"
|
||||
# SSH_KEY (set by detect_hosts) is this server's own private key.
|
||||
# The remote accepts it because this server's PUBLIC key was installed there via ssh_setup.sh.
|
||||
# HOST{N}_SSH_KEY lives in host{N}.conf — with sparse checkout, the other server's
|
||||
# conf is never present here. Always use SSH_KEY (local private key) for outbound SSH.
|
||||
MIRROR_SSH_KEY="$SSH_KEY"
|
||||
|
||||
AM_OWNER=false
|
||||
AM_MIRROR=false
|
||||
[[ "$MY_ID" == "$OWNER_ID" ]] && AM_OWNER=true
|
||||
[[ "$MY_ID" == "$MIRROR_ID" ]] && AM_MIRROR=true
|
||||
|
||||
EXTRA_FLAGS=()
|
||||
[[ "$DRY_RUN" == true ]] && EXTRA_FLAGS+=("--dry-run")
|
||||
[[ "$LOG_MODE" == true ]] && EXTRA_FLAGS+=("--log")
|
||||
|
||||
START=$(date +%s)
|
||||
|
||||
# ── Helper: write phase completion flag to setup.db + push to remotes ─────────────────────────
|
||||
write_onboard_phase() {
|
||||
local target_id="$1" phase="$2"
|
||||
local key="${target_id}_PHASE${phase}_DONE"
|
||||
local state_file="$(platform_setup_db_path)"
|
||||
[[ "$DRY_RUN" == true ]] && { warn "DRY RUN — would write ${key}=true"; return 0; }
|
||||
if grep -q "^${key}=" "$state_file" 2>/dev/null; then
|
||||
sed -i "s|^${key}=.*|${key}=true|" "$state_file"
|
||||
else
|
||||
echo "${key}=true" >> "$state_file"
|
||||
fi
|
||||
platform_push_setup_state
|
||||
}
|
||||
|
||||
echo ""
|
||||
echo "━━━ $ICON_FALLBACK Partnership Onboard — $MY_ID ($LOCAL_SERVER_NAME) — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo ""
|
||||
echo " Role: $( [[ "$AM_OWNER" == true ]] && echo "OWNER" || echo "MIRROR" )"
|
||||
echo " This: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo " Partner: $( [[ "$AM_OWNER" == true ]] && echo "$MIRROR_ID ($MIRROR)" || echo "$OWNER_ID ($OWNER)" )"
|
||||
echo ""
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no permanent changes will be made"
|
||||
|
||||
# ==============================================================================================
|
||||
# ── HELPER: stop containers on the mirror by reading its own conf via SSH ────────────────────
|
||||
#
|
||||
# SSHes to the mirror, sources its load_config.sh at the same $SCRIPTS_ROOT path (both servers
|
||||
# use the same convention), and reads the named config array from the mirror's own conf.
|
||||
# HOST2's container list stays in HOST2's host2.conf — not duplicated in HOST1's conf.
|
||||
# Fails gracefully if scripts aren't present yet or the array is empty (nothing to stop).
|
||||
#
|
||||
# deploy_container_from_xml() already stops/removes containers with the same name as what's
|
||||
# being deployed. This step handles containers with DIFFERENT names that conflict.
|
||||
# ==============================================================================================
|
||||
stop_mirror_stack() {
|
||||
local config_var="$1" label="$2"
|
||||
local -a to_stop=()
|
||||
|
||||
mapfile -t to_stop < <(
|
||||
timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$MIRROR_IP" \
|
||||
"source '$SCRIPTS_ROOT/load_config.sh' 2>/dev/null
|
||||
detect_hosts 2>/dev/null
|
||||
printf '%s\n' \"\${${config_var}[@]:-}\"" 2>/dev/null | grep -v '^$'
|
||||
)
|
||||
|
||||
if [[ ${#to_stop[@]} -eq 0 ]]; then
|
||||
log "No $label containers to stop on $MIRROR — skipping"
|
||||
return 0
|
||||
fi
|
||||
|
||||
log "Stopping $label on $MIRROR: ${to_stop[*]}"
|
||||
for container in "${to_stop[@]}"; do
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn " DRY RUN — would stop + rm $container on $MIRROR"
|
||||
continue
|
||||
fi
|
||||
timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" root@"$MIRROR_IP" \
|
||||
"docker stop '$container' 2>/dev/null
|
||||
docker rm '$container' 2>/dev/null && echo removed" 2>/dev/null | \
|
||||
grep -q removed && \
|
||||
log " $container removed ✅" || \
|
||||
log " $container not found on $MIRROR — skipping"
|
||||
done
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ── MIRROR PATH ───────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
if [[ "$AM_MIRROR" == true ]]; then
|
||||
echo "━━━ Step 1/2 — SSH Key Setup (Mirror) ━━━"
|
||||
echo ""
|
||||
echo " Mirror sets up SSH keys, then notifies Owner to run Phase 2."
|
||||
echo ""
|
||||
|
||||
if [[ "$SKIP_SSH" == true ]]; then
|
||||
warn "Skipping SSH setup (--skip-ssh)"
|
||||
elif bash "$SCRIPT_DIR/ssh_setup.sh" "${EXTRA_FLAGS[@]}"; then
|
||||
log "SSH key ready ✅"
|
||||
else
|
||||
error "SSH key setup failed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "━━━ Step 2/2 — Notify Owner to Run Phase 2 ━━━"
|
||||
echo ""
|
||||
|
||||
OWNER_IP=$(resolve_tailscale_ip "$OWNER" 2>/dev/null || true)
|
||||
PHASE2_TRIGGERED=false
|
||||
|
||||
if [[ -n "$OWNER_IP" ]]; then
|
||||
# Read OWNER's SCRIPTS_DIR via platform probe command — don't assume same path as mirror
|
||||
_probe_cmd=$(platform_scripts_dir_probe_cmd)
|
||||
OWNER_SCRIPTS_DIR=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$OWNER_IP" \
|
||||
"$_probe_cmd" 2>/dev/null | tr -d '[:space:]')
|
||||
OWNER_SCRIPTS_DIR="${OWNER_SCRIPTS_DIR:-$SCRIPTS_DIR}"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would SSH to $OWNER ($OWNER_IP) and trigger Phase 2"
|
||||
PHASE2_TRIGGERED=true
|
||||
elif timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$OWNER_IP" \
|
||||
"nohup bash '${OWNER_SCRIPTS_DIR}/Partnership/partnership_onboard.sh' --phase2-only > /tmp/vv_phase2_onboard.log 2>&1 & echo triggered" \
|
||||
2>/dev/null | grep -q triggered; then
|
||||
log "Phase 2 triggered on $OWNER ✅"
|
||||
log "Watch progress on $OWNER: tail -f /tmp/vv_phase2_onboard.log"
|
||||
PHASE2_TRIGGERED=true
|
||||
else
|
||||
warn "Could not auto-trigger Phase 2 on $OWNER"
|
||||
fi
|
||||
else
|
||||
warn "Cannot resolve $OWNER Tailscale IP"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY MIRROR SETUP COMPLETE ━━━━━"
|
||||
echo " SSH key: ready"
|
||||
echo " Phase 2 on $OWNER: $( [[ "$PHASE2_TRIGGERED" == true ]] && echo "triggered ✅" || echo "needs manual trigger ⚠" )"
|
||||
if [[ "$PHASE2_TRIGGERED" == false ]]; then
|
||||
echo ""
|
||||
echo " Run manually on $OWNER:"
|
||||
echo " bash Partnership/partnership_onboard.sh --phase2-only"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ── OWNER PATH ────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
MIRROR_IP=$(resolve_tailscale_ip "$MIRROR")
|
||||
[[ -z "$MIRROR_IP" ]] && { error "Cannot resolve $MIRROR Tailscale IP — is Tailscale running?"; exit 1; }
|
||||
log "Mirror: $MIRROR ($MIRROR_IP)"
|
||||
[[ "$PHASE1_ONLY" == true ]] && log "Mode: Phase 1 only (SSH + conf push)"
|
||||
[[ "$PHASE2_ONLY" == true ]] && log "Mode: Phase 2 only (containers + arr + onboard)"
|
||||
echo ""
|
||||
|
||||
STEP_SSH_OK=false
|
||||
STEP_NETWORK_OK=false
|
||||
STEP_STOP_AUTH_OK=true
|
||||
STEP_AUTH_OK=true
|
||||
AUTH_DEPLOYED=0
|
||||
AUTH_FAILED=0
|
||||
STEP_STOP_ARR_OK=true
|
||||
STEP_ARR_OK=true
|
||||
ARR_DEPLOYED=0
|
||||
ARR_FAILED=0
|
||||
STEP_STOP_SERVICES_OK=true
|
||||
STEP_SERVICES_OK=true
|
||||
SERVICES_DEPLOYED=0
|
||||
SERVICES_FAILED=0
|
||||
ONBOARD_OK=false
|
||||
ARR_SYNC_OK=false
|
||||
MASTER_PUSH_OK=false
|
||||
|
||||
# ── Step 1: SSH ───────────────────────────────────────────────────────────────────────────────
|
||||
# Skipped when --phase2-only (SSH was already done in Phase 1).
|
||||
echo "━━━ Step 1 — SSH Key Setup ━━━"
|
||||
|
||||
if [[ "$SKIP_SSH" == true ]]; then
|
||||
warn "Skipping (--skip-ssh)"
|
||||
STEP_SSH_OK=true
|
||||
elif [[ "$PHASE1_ONLY" == true ]]; then
|
||||
# Phase 1 in background: test if SSH already works first — avoids ssh-copy-id
|
||||
# hanging for a password prompt with no TTY.
|
||||
if timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$MIRROR_IP" exit 0 2>/dev/null; then
|
||||
log "SSH to $MIRROR already works ✅ — skipping key install"
|
||||
STEP_SSH_OK=true
|
||||
else
|
||||
# Key not yet on HOST2 — try ssh_setup.sh (works interactively, may fail in background)
|
||||
if bash "$SCRIPT_DIR/ssh_setup.sh" "${EXTRA_FLAGS[@]}"; then
|
||||
log "SSH keys ready ✅"
|
||||
STEP_SSH_OK=true
|
||||
else
|
||||
# Soft-fail: generate key locally if not present, then tell user to install manually
|
||||
warn "Could not install key on $MIRROR automatically (no terminal for password prompt)"
|
||||
if [[ -f "$SSH_KEY" ]]; then
|
||||
log "Local key exists at: $SSH_KEY"
|
||||
else
|
||||
bash "$SCRIPT_DIR/ssh_setup.sh" --key-only "${EXTRA_FLAGS[@]}" 2>/dev/null || true
|
||||
fi
|
||||
if [[ -f "${SSH_KEY}.pub" ]]; then
|
||||
echo ""
|
||||
echo " Install this key on $MIRROR to complete SSH setup:"
|
||||
echo " ┌─────────────────────────────────────────────────────"
|
||||
cat "${SSH_KEY}.pub" | sed 's/^/ │ /'
|
||||
echo " └─────────────────────────────────────────────────────"
|
||||
echo " Run on a terminal: ssh-copy-id -i ${SSH_KEY}.pub root@${MIRROR_IP}"
|
||||
echo " Then click 'Push Conf' in the Partnership tab."
|
||||
# Write key-ready flag so UI can show the manual-install state
|
||||
[[ "$DRY_RUN" == false ]] && {
|
||||
local kflag="${MIRROR_ID}_KEY_READY"
|
||||
local _setup_f="$(platform_setup_db_path)"
|
||||
grep -q "^${kflag}=" "$_setup_f" 2>/dev/null \
|
||||
&& sed -i "s|^${kflag}=.*|${kflag}=true|" "$_setup_f" \
|
||||
|| echo "${kflag}=true" >> "$_setup_f"
|
||||
}
|
||||
fi
|
||||
STEP_SSH_OK=false
|
||||
fi
|
||||
fi
|
||||
elif bash "$SCRIPT_DIR/ssh_setup.sh" "${EXTRA_FLAGS[@]}"; then
|
||||
log "SSH keys ready ✅"
|
||||
STEP_SSH_OK=true
|
||||
else
|
||||
error "SSH key setup failed — aborting"
|
||||
error "Re-run or use --skip-ssh if key is already set up"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ── Phase 1 exit point ────────────────────────────────────────────────────────────────────────
|
||||
# --phase1-only: SSH + conf push is all HOST1 needs to do before HOST2 installs Varaverk.
|
||||
# HOST2's wizard will detect the pushed master.conf + state file and take the correct path.
|
||||
if [[ "$PHASE1_ONLY" == true ]]; then
|
||||
if [[ "$STEP_SSH_OK" == false ]]; then
|
||||
# SSH key not yet installed on HOST2 — can't push conf, but local setup still runs.
|
||||
# UI will show "key ready, install manually" state via HOST2_KEY_READY flag.
|
||||
echo ""
|
||||
echo "━━━ Phase 1 — HOST1 Local Setup (SSH pending) ━━━"
|
||||
bash "$SCRIPT_DIR/partnership_manager.sh" --onboard --local-only "${EXTRA_FLAGS[@]}" || \
|
||||
warn "Local setup had issues — check partnership_manager.sh output above"
|
||||
|
||||
END=$(date +%s)
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY PHASE 1 — SSH PENDING ━━━━━"
|
||||
echo " SSH keys: key generated ✅ — NOT yet installed on $MIRROR ⚠"
|
||||
echo " Conf push: skipped (needs SSH access to $MIRROR)"
|
||||
echo " HOST1 setup: done ✅"
|
||||
echo " Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
echo " ACTION NEEDED: install the key on $MIRROR:"
|
||||
echo " ssh-copy-id -i ${SSH_KEY}.pub root@${MIRROR_IP}"
|
||||
echo " Then click 'Push Conf' in Partnership tab, or run:"
|
||||
echo " bash Partnership/partnership_onboard.sh --phase1-only --skip-ssh"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "━━━ Phase 1 — Conf Push ━━━"
|
||||
|
||||
CONF_PUSH_OK=false
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would push master.conf + state file to $MIRROR"
|
||||
CONF_PUSH_OK=true
|
||||
else
|
||||
push_output=$(platform_push_conf)
|
||||
push_rc=$?
|
||||
[[ -n "$push_output" ]] && echo "$push_output"
|
||||
platform_push_setup_state
|
||||
if [[ $push_rc -eq 0 ]]; then
|
||||
log "Conf push complete ✅"
|
||||
CONF_PUSH_OK=true
|
||||
else
|
||||
warn "Conf push had failures — retry via Scheduler → master.conf → Save Conf"
|
||||
fi
|
||||
fi
|
||||
|
||||
# HOST1 local setup — runs immediately without needing HOST2
|
||||
echo ""
|
||||
echo "━━━ Phase 1 — HOST1 Local Setup ━━━"
|
||||
bash "$SCRIPT_DIR/partnership_manager.sh" --onboard --local-only "${EXTRA_FLAGS[@]}" || \
|
||||
warn "Local setup had issues — check partnership_manager.sh output above"
|
||||
|
||||
[[ "$DRY_RUN" == false ]] && write_onboard_phase "$MIRROR_ID" 1
|
||||
|
||||
END=$(date +%s)
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY PHASE 1 COMPLETE ━━━━━"
|
||||
echo " SSH keys: $( [[ "$STEP_SSH_OK" == true ]] && echo "ready ✅" || echo "skipped" )"
|
||||
echo " Conf push: $( [[ "$CONF_PUSH_OK" == true ]] && echo "done ✅" || echo "⚠ manual needed" )"
|
||||
echo " HOST1 setup: done ✅"
|
||||
echo " Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
echo " HOST1 is fully set up. HOST2 ($MIRROR) can now install the Varaverk plugin."
|
||||
echo " The wizard will detect the pushed conf and take the correct path."
|
||||
echo " When HOST2 completes its onboard, it will automatically trigger Phase 2 here."
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ── Step 1b: Ensure custom Docker network exists on mirror ────────────────────────────────────
|
||||
# Must run before any container deploy — docker create fails if the network is missing.
|
||||
echo ""
|
||||
echo "━━━ Step 1b — Docker Network (Mirror) ━━━"
|
||||
|
||||
_net_script="${SCRIPTS_ROOT}/Docker_Essentials/docker_network_connect.sh"
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would run docker_network_connect.sh on $MIRROR"
|
||||
STEP_NETWORK_OK=true
|
||||
elif timeout 60 ssh -i "$MIRROR_SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$MIRROR_IP" \
|
||||
"bash '$_net_script'" 2>/dev/null; then
|
||||
log "Docker network ready on $MIRROR ✅"
|
||||
STEP_NETWORK_OK=true
|
||||
else
|
||||
warn "docker_network_connect.sh failed on $MIRROR — containers may fail if network is missing"
|
||||
warn "Check ${_net_script} on $MIRROR and re-run with --skip-ssh if needed"
|
||||
fi
|
||||
|
||||
# ── Step 2: Stop mirror's existing auth stack ─────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ Step 2 — Stop Mirror Auth Stack ━━━"
|
||||
|
||||
if [[ "$SKIP_AUTH_STACK" == true ]]; then
|
||||
warn "Skipping (--skip-auth-stack)"
|
||||
else
|
||||
stop_mirror_stack "PARTNERSHIP_REPLACE_CONTAINERS" "auth stack"
|
||||
fi
|
||||
|
||||
# ── Step 4: Deploy auth stack on mirror ───────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ Step 3 — Deploy Auth Stack on Mirror ━━━"
|
||||
|
||||
if [[ "$SKIP_AUTH_STACK" == true ]]; then
|
||||
warn "Skipping (--skip-auth-stack)"
|
||||
elif [[ ${#PARTNERSHIP_AUTH_STACK[@]} -eq 0 ]]; then
|
||||
warn "PARTNERSHIP_AUTH_STACK not set in ${MY_ID} conf — skipping auth stack deploy"
|
||||
warn "Add HOST${MY_ID: -1}_PARTNERSHIP_AUTH_STACK to host${MY_ID: -1}.conf"
|
||||
STEP_AUTH_OK=false
|
||||
else
|
||||
deploy_xml_stack PARTNERSHIP_AUTH_STACK
|
||||
AUTH_DEPLOYED=$_STACK_DEPLOYED
|
||||
AUTH_FAILED=$_STACK_FAILED
|
||||
echo "Auth stack: $AUTH_DEPLOYED deployed, $AUTH_FAILED failed"
|
||||
[[ "$AUTH_FAILED" -gt 0 ]] && STEP_AUTH_OK=false
|
||||
fi
|
||||
|
||||
# ── Step 5: Stop mirror's existing arr stack ──────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ Step 4 — Stop Mirror Arr Stack ━━━"
|
||||
|
||||
if [[ "$SKIP_ARR_STACK" == true ]]; then
|
||||
warn "Skipping (--skip-arr-stack)"
|
||||
elif [[ ${#PARTNERSHIP_ARR_STACK[@]} -eq 0 ]]; then
|
||||
log "PARTNERSHIP_ARR_STACK not configured — skipping arr stack deploy"
|
||||
SKIP_ARR_STACK=true
|
||||
else
|
||||
stop_mirror_stack "PARTNERSHIP_ARR_REPLACE_CONTAINERS" "arr stack"
|
||||
fi
|
||||
|
||||
# ── Step 5: Deploy arr stack on mirror ───────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ Step 5 — Deploy Arr Stack on Mirror ━━━"
|
||||
|
||||
if [[ "$SKIP_ARR_STACK" == true ]]; then
|
||||
warn "Skipping (--skip-arr-stack)"
|
||||
else
|
||||
deploy_xml_stack PARTNERSHIP_ARR_STACK
|
||||
ARR_DEPLOYED=$_STACK_DEPLOYED
|
||||
ARR_FAILED=$_STACK_FAILED
|
||||
echo "Arr stack: $ARR_DEPLOYED deployed, $ARR_FAILED failed"
|
||||
[[ "$ARR_FAILED" -gt 0 ]] && STEP_ARR_OK=false
|
||||
fi
|
||||
|
||||
# ── Step 6: Stop mirror's existing services stack ─────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ Step 6 — Stop Mirror Services Stack ━━━"
|
||||
|
||||
if [[ "$SKIP_SERVICES_STACK" == true ]]; then
|
||||
warn "Skipping (--skip-services-stack)"
|
||||
elif [[ ${#PARTNERSHIP_SERVICES_STACK[@]} -eq 0 ]]; then
|
||||
log "PARTNERSHIP_SERVICES_STACK not configured — skipping services stack deploy"
|
||||
SKIP_SERVICES_STACK=true
|
||||
else
|
||||
stop_mirror_stack "PARTNERSHIP_SERVICES_REPLACE_CONTAINERS" "services stack"
|
||||
fi
|
||||
|
||||
# ── Step 7: Deploy services stack on mirror ───────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ Step 7 — Deploy Services Stack on Mirror ━━━"
|
||||
|
||||
if [[ "$SKIP_SERVICES_STACK" == true ]]; then
|
||||
warn "Skipping (--skip-services-stack)"
|
||||
else
|
||||
deploy_xml_stack PARTNERSHIP_SERVICES_STACK
|
||||
SERVICES_DEPLOYED=$_STACK_DEPLOYED
|
||||
SERVICES_FAILED=$_STACK_FAILED
|
||||
echo "Services stack: $SERVICES_DEPLOYED deployed, $SERVICES_FAILED failed"
|
||||
[[ "$SERVICES_FAILED" -gt 0 ]] && STEP_SERVICES_OK=false
|
||||
fi
|
||||
|
||||
# ── Step 8: Partnership onboard ───────────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ Step 8 — Partnership Onboard ━━━"
|
||||
|
||||
if bash "$SCRIPTS_ROOT/Partnership/partnership_manager.sh" --onboard "${EXTRA_FLAGS[@]}"; then
|
||||
echo "Partnership onboard complete ✅"
|
||||
ONBOARD_OK=true
|
||||
else
|
||||
error "Partnership onboard failed"
|
||||
ONBOARD_OK=false
|
||||
fi
|
||||
|
||||
# ── Step 9: Arr library bootstrap ─────────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ Step 9 — Arr Library Bootstrap ━━━"
|
||||
|
||||
if [[ "$ONBOARD_OK" == false ]]; then
|
||||
warn "Skipping — onboard did not complete"
|
||||
elif [[ "$SKIP_ARR_SYNC" == true ]]; then
|
||||
warn "Skipping (--skip-arr-sync)"
|
||||
elif [[ ! -f "$SCRIPTS_ROOT/Media/arr_sync.sh" ]]; then
|
||||
warn "arr_sync.sh not found — run Media/arr_sync.sh manually once arrs are live"
|
||||
elif bash "$SCRIPTS_ROOT/Media/arr_sync.sh" "${EXTRA_FLAGS[@]}"; then
|
||||
echo "Arr bootstrap complete ✅"
|
||||
ARR_SYNC_OK=true
|
||||
else
|
||||
warn "Arr sync had errors — partnership still valid"
|
||||
warn "Re-run Media/arr_sync.sh once all arr containers are live"
|
||||
fi
|
||||
|
||||
# ── Step 10: Push master.conf to all listed hosts ─────────────────────────────────────────────
|
||||
# SSH is now established and all partners have the plugin installed.
|
||||
# Push the authoritative master.conf so every listed host is in sync immediately.
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Step 10 — master.conf Push ━━━"
|
||||
|
||||
if [[ "$ONBOARD_OK" == false ]]; then
|
||||
warn "Skipping — onboard did not complete"
|
||||
elif [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would push master.conf to all listed hosts"
|
||||
MASTER_PUSH_OK=true
|
||||
else
|
||||
push_output=$(platform_push_conf)
|
||||
push_rc=$?
|
||||
[[ -n "$push_output" ]] && echo "$push_output"
|
||||
platform_push_setup_state
|
||||
if [[ $push_rc -eq 0 ]]; then
|
||||
echo "master.conf sync complete ✅"
|
||||
MASTER_PUSH_OK=true
|
||||
else
|
||||
warn "master.conf push had failures — retry via Scheduler → master.conf → Save Conf"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Write Phase 2 completion state ────────────────────────────────────────────────────────────
|
||||
[[ "$ONBOARD_OK" == true && "$DRY_RUN" == false ]] && write_onboard_phase "$MIRROR_ID" 2
|
||||
|
||||
# ── Summary ───────────────────────────────────────────────────────────────────────────────────
|
||||
END=$(date +%s)
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY ONBOARD SUMMARY ━━━━━"
|
||||
echo " Owner: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo " Mirror: $MIRROR ($MIRROR_IP)"
|
||||
[[ "$PHASE2_ONLY" == true ]] && echo " Mode: Phase 2 (triggered by HOST2 notification)"
|
||||
echo " Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
_ok() { [[ "$1" == true ]] && echo "✅" || echo "❌"; }
|
||||
_skip() { [[ "$1" == true ]] && echo "skipped" || echo "$(_ok "$2")"; }
|
||||
|
||||
echo " Step 1 — SSH keys: $(_skip "$SKIP_SSH" "$STEP_SSH_OK")"
|
||||
echo " Step 1b — Docker network: $(_ok "$STEP_NETWORK_OK")"
|
||||
echo " Step 2 — Stop auth: $(_skip "$SKIP_AUTH_STACK" "$STEP_STOP_AUTH_OK")"
|
||||
echo " Step 3 — Auth stack: $( [[ "$SKIP_AUTH_STACK" == true ]] && echo "skipped" || echo "${AUTH_DEPLOYED} deployed, ${AUTH_FAILED} failed" )"
|
||||
echo " Step 4 — Stop arr: $(_skip "$SKIP_ARR_STACK" "$STEP_STOP_ARR_OK")"
|
||||
echo " Step 5 — Arr stack: $( [[ "$SKIP_ARR_STACK" == true ]] && echo "skipped" || echo "${ARR_DEPLOYED} deployed, ${ARR_FAILED} failed" )"
|
||||
echo " Step 6 — Stop services: $(_skip "$SKIP_SERVICES_STACK" "$STEP_STOP_SERVICES_OK")"
|
||||
echo " Step 7 — Services stack: $( [[ "$SKIP_SERVICES_STACK" == true ]] && echo "skipped" || echo "${SERVICES_DEPLOYED} deployed, ${SERVICES_FAILED} failed" )"
|
||||
echo " Step 8 — Onboard: $(_ok "$ONBOARD_OK")"
|
||||
echo " Step 9 — Arr bootstrap: $( [[ "$SKIP_ARR_SYNC" == true || "$ONBOARD_OK" == false ]] && echo "skipped" || echo "$(_ok "$ARR_SYNC_OK")" )"
|
||||
echo " Step 10 — Conf push: $( [[ "$ONBOARD_OK" == false ]] && echo "skipped" || echo "$(_ok "$MASTER_PUSH_OK")" )"
|
||||
echo ""
|
||||
|
||||
if [[ "$ONBOARD_OK" == true ]]; then
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes made" || \
|
||||
echo "$ICON_DONE DONE — partnership established ✅"
|
||||
echo "Verify with: Partnership/partnership_manager.sh --status"
|
||||
else
|
||||
error "Setup incomplete — resolve errors above and re-run"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
[[ "$ONBOARD_OK" == false ]] && exit 1
|
||||
exit 0
|
||||
+158
@@ -0,0 +1,158 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ============================= Conf Cache Sync ================================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Maintains a RAM-resident conf cache at /tmp/.vv/config/cached/.confs/.
|
||||
# Credentials and partner keys live in RAM only — never on disk across hosts.
|
||||
#
|
||||
# On array start (default / --array-start):
|
||||
# 1. Copy own conf to local cache
|
||||
# 2. Pull each available partner's conf from their disk → local cache
|
||||
# 3. Push own conf to each available partner's /tmp/.vv/ cache
|
||||
#
|
||||
# On conf save (--push-only):
|
||||
# Fast path — push updated own conf to all partners' /tmp/.vv/ cache only.
|
||||
# No pulls, no local cache rebuild.
|
||||
#
|
||||
# Cache is /tmp (tmpfs) — cleared every reboot, repopulated by this script
|
||||
# on next array start. Scripts source from cache for partner vars; own vars
|
||||
# always come from disk (load_config.sh skips cached copy of own conf).
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# conf_sync.sh Full sync: pull from all partners + push to all partners
|
||||
# conf_sync.sh --push-only Push own conf to all partners (fast, for conf-save hook)
|
||||
# conf_sync.sh --pull-only Pull partner confs into local cache only (for intermediate orch)
|
||||
# conf_sync.sh --dry-run Show what would happen, no changes
|
||||
# conf_sync.sh --log Verbose output
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
PUSH_ONLY=false
|
||||
PULL_ONLY=false
|
||||
FILTERED_ARGS=()
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--push-only) PUSH_ONLY=true ;;
|
||||
--pull-only) PULL_ONLY=true ;;
|
||||
*) FILTERED_ARGS+=("$arg") ;;
|
||||
esac
|
||||
done
|
||||
parse_args "${FILTERED_ARGS[@]}"
|
||||
detect_hosts
|
||||
|
||||
if [[ "${CONF_SYNC_ENABLED:-true}" == false ]]; then
|
||||
log "CONF_SYNC_ENABLED=false — skipping"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
CACHE_DIR="/tmp/.vv/config/cached/.confs"
|
||||
MY_CONF="$SCRIPTS_ROOT/Configurations/${MY_ID,,}.conf"
|
||||
SSH_TIMEOUT=10
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
|
||||
# ── Ensure cache dir exists ───────────────────────────────────────────────────
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
mkdir -p "$CACHE_DIR"
|
||||
fi
|
||||
|
||||
# ── Copy own conf into local cache ───────────────────────────────────────────
|
||||
if [[ "$PUSH_ONLY" == false ]] && [[ "$PULL_ONLY" == false ]]; then
|
||||
if [[ -f "$MY_CONF" ]]; then
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would copy $(basename "$MY_CONF") → $CACHE_DIR/"
|
||||
else
|
||||
cp "$MY_CONF" "$CACHE_DIR/${MY_ID,,}.conf" && \
|
||||
log "Own conf cached ✅" || warn "Failed to cache own conf"
|
||||
fi
|
||||
else
|
||||
warn "Own conf not found: $MY_CONF"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Per-partner sync ──────────────────────────────────────────────────────────
|
||||
PUSHED=0
|
||||
PULLED=0
|
||||
FAILED=0
|
||||
|
||||
for host_var in $(compgen -v | grep -E '^HOST[0-9]+$' | sort); do
|
||||
partner_host="${!host_var}"
|
||||
[[ -z "$partner_host" ]] && continue
|
||||
[[ "${host_var,,}" == "${MY_ID,,}" ]] && continue
|
||||
|
||||
partner_slot="${host_var,,}" # e.g. host2
|
||||
partner_ip=$(resolve_tailscale_ip "$partner_host" 2>/dev/null || true)
|
||||
|
||||
if [[ -z "$partner_ip" ]]; then
|
||||
warn "$partner_host — cannot resolve Tailscale IP, skipping"
|
||||
(( FAILED++ ))
|
||||
continue
|
||||
fi
|
||||
|
||||
# ── Pull: grab partner's conf from their disk → our local cache ──────────
|
||||
if [[ "$PUSH_ONLY" == false ]]; then
|
||||
remote_conf="${SCRIPTS_DIR}/Configurations/${partner_slot}.conf"
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would pull $partner_host:$remote_conf → $CACHE_DIR/${partner_slot}.conf"
|
||||
elif timeout "$SSH_TIMEOUT" scp -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \
|
||||
"root@${partner_ip}:${remote_conf}" \
|
||||
"$CACHE_DIR/${partner_slot}.conf" 2>/dev/null; then
|
||||
log "Pulled ${partner_slot}.conf from $partner_host ✅"
|
||||
(( PULLED++ ))
|
||||
else
|
||||
warn "Could not pull ${partner_slot}.conf from $partner_host"
|
||||
(( FAILED++ ))
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Push: send own conf to partner's /tmp/.vv/ cache ────────────────────
|
||||
if [[ "$PULL_ONLY" == true ]]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would push ${MY_ID,,}.conf → $partner_host:/tmp/.vv/config/cached/.confs/"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Ensure partner's cache dir exists, then SCP own conf into it
|
||||
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \
|
||||
"root@${partner_ip}" "mkdir -p '$CACHE_DIR'" 2>/dev/null
|
||||
|
||||
if timeout "$SSH_TIMEOUT" scp -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \
|
||||
"$MY_CONF" \
|
||||
"root@${partner_ip}:${CACHE_DIR}/${MY_ID,,}.conf" 2>/dev/null; then
|
||||
log "Pushed ${MY_ID,,}.conf to $partner_host ✅"
|
||||
(( PUSHED++ ))
|
||||
else
|
||||
warn "Could not push to $partner_host"
|
||||
(( FAILED++ ))
|
||||
fi
|
||||
done
|
||||
|
||||
# ── Summary ───────────────────────────────────────────────────────────────────
|
||||
if [[ "$PUSH_ONLY" == true ]]; then
|
||||
info "Conf push complete — pushed to $PUSHED host(s)${FAILED:+, $FAILED failed}"
|
||||
elif [[ "$PULL_ONLY" == true ]]; then
|
||||
info "Conf pull complete — pulled $PULLED partner conf(s)${FAILED:+, $FAILED failed}"
|
||||
else
|
||||
info "Conf sync complete — pulled $PULLED, pushed $PUSHED${FAILED:+, $FAILED failed}"
|
||||
fi
|
||||
|
||||
if [[ "$FAILED" -gt 0 ]]; then
|
||||
notify "Conf sync on $LOCAL_SERVER_NAME ($MY_ID) — $FAILED partner(s) failed. Partner config cache may be stale." \
|
||||
"Conf Sync" "warning"
|
||||
exit 1
|
||||
fi
|
||||
+365
@@ -0,0 +1,365 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ================================= Ramdisk Setup ==============================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Creates the tmpfs ramdisk, SSD fallback directory, transcode symlink, and
|
||||
# pre-creates transcoding-temp on the ramdisk. Run once at array start via
|
||||
# array_started.sh (System_Essentials/). Idempotent — already-mounted ramdisk
|
||||
# reports status and exits cleanly. Always resets the symlink to the ramdisk
|
||||
# on boot, ensuring a clean state regardless of what state it was in before
|
||||
# shutdown.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Creates four things in order:
|
||||
# 1. RAMDISK_PATH — tmpfs mount (size: HOST*_RAMDISK_SIZE ceiling, not a reservation)
|
||||
# 2. TRANSCODE_SSD — SSD fallback directory and transcoding-temp inside it
|
||||
# 3. TRANSCODE_LINK — symlink reset to RAMDISK_PATH (clean state at every boot)
|
||||
# 4. transcoding-temp/ inside RAMDISK_PATH — pre-created before Emby starts
|
||||
#
|
||||
# The transcoding-temp pre-creation is critical: if it doesn't exist on the ramdisk
|
||||
# when Emby starts, Emby searches all accessible paths for an existing one and finds
|
||||
# the SSD fallback version — routing all sessions there until Emby restarts.
|
||||
#
|
||||
# Initialises /tmp/transcode_state.db with current target and flip counters.
|
||||
# /tmp resets on reboot — correct, transcode state should not persist across boots.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Root Required
|
||||
# mount and symlink creation require root.
|
||||
#
|
||||
# Single Instance Lock
|
||||
# acquire_lock prevents duplicate runs at array start.
|
||||
#
|
||||
# Idempotent Mount Check
|
||||
# If RAMDISK_PATH is already a mountpoint, reports status and exits cleanly
|
||||
# without attempting to remount or changing anything.
|
||||
#
|
||||
# Notification Validated
|
||||
# platform_require_cmd confirms the notify script is present before use.
|
||||
#
|
||||
# Silent on Success
|
||||
# Startup script runs on every boot — no output when healthy.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# host*.conf
|
||||
#
|
||||
# HOST*_RAMDISK_SIZE
|
||||
# tmpfs ceiling (e.g. 10G). Must change together with WARN_GB and LOW_GB.
|
||||
# Aliased by detect_hosts() → RAMDISK_SIZE.
|
||||
#
|
||||
# HOST*_RAMDISK_WARN_GB
|
||||
# Usage level at which transcode_manager.sh flips symlink to SSD.
|
||||
#
|
||||
# HOST*_RAMDISK_LOW_GB
|
||||
# Usage level at which transcode_manager.sh flips back to ramdisk.
|
||||
#
|
||||
# HOST*_TRANSCODE_SSD
|
||||
# SSD fallback directory path.
|
||||
# Aliased by detect_hosts() → TRANSCODE_SSD.
|
||||
#
|
||||
# master.conf
|
||||
#
|
||||
# TRANSCODE_LINK
|
||||
# Symlink path Emby uses as its transcode directory. Must match the path
|
||||
# configured in Emby's transcoding settings.
|
||||
#
|
||||
# TRANSCODE_CHMOD / TRANSCODE_OWNER
|
||||
# Permissions applied to both ramdisk and SSD directories. (default: 755 / nobody:users)
|
||||
#
|
||||
# TRANSCODE_STATE_FILE
|
||||
# Override state file path. (default: ${STATE_DIR}/transcode_state.db)
|
||||
#
|
||||
# ==============================================================================================
|
||||
# STATE FILES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# TRANSCODE_STATE_FILE (default: ${STATE_DIR}/transcode_state.db)
|
||||
# Current symlink target + flip count tracking. Lives in STATE_DIR
|
||||
# (ephemeral on Unraid — resets on reboot correctly).
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# ramdisk_setup.sh
|
||||
# Normal setup run. Called by array_started.sh at boot.
|
||||
#
|
||||
# ramdisk_setup.sh --dry-run
|
||||
# Show what would be created without creating anything.
|
||||
#
|
||||
# ramdisk_setup.sh --status
|
||||
# Show current ramdisk mount state, symlink target, and SSD directory state.
|
||||
#
|
||||
# ramdisk_setup.sh --log
|
||||
# Verbose output showing each creation step.
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root — mount and symlink require root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
acquire_lock
|
||||
|
||||
# detect_hosts() sets MY_ID and aliases RAMDISK_SIZE, TRANSCODE_SSD etc.
|
||||
detect_hosts
|
||||
|
||||
log "Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
log "Ramdisk: $RAMDISK_PATH ($RAMDISK_SIZE)"
|
||||
log "Fallback: $TRANSCODE_SSD"
|
||||
log "Symlink: $TRANSCODE_LINK"
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_RAM Ramdisk path: $RAMDISK_PATH"
|
||||
echo "$ICON_RAM Ramdisk size: $RAMDISK_SIZE"
|
||||
echo "$ICON_RAM Warn at: ${RAMDISK_WARN_GB}GB"
|
||||
echo "$ICON_RAM Flip at: ${RAMDISK_LOW_GB}GB"
|
||||
echo "$ICON_DISK SSD fallback: $TRANSCODE_SSD"
|
||||
echo "$ICON_LINK Symlink: $TRANSCODE_LINK"
|
||||
echo "$ICON_GEAR Owner: $TRANSCODE_OWNER"
|
||||
echo "$ICON_GEAR Mode: $TRANSCODE_CHMOD"
|
||||
echo ""
|
||||
|
||||
if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
|
||||
USAGE=$(df -BG "$RAMDISK_PATH" | awk 'NR==2 {print $3}')
|
||||
AVAIL=$(df -BG "$RAMDISK_PATH" | awk 'NR==2 {print $4}')
|
||||
echo " $ICON_RAM Ramdisk: mounted — $USAGE used / $AVAIL available ✅"
|
||||
else
|
||||
echo " $ICON_RAM Ramdisk: NOT mounted"
|
||||
fi
|
||||
|
||||
if [[ -L "$TRANSCODE_LINK" ]]; then
|
||||
TARGET=$(readlink "$TRANSCODE_LINK")
|
||||
echo " $ICON_LINK Symlink: $TRANSCODE_LINK → $TARGET"
|
||||
else
|
||||
echo " $ICON_LINK Symlink: not set"
|
||||
fi
|
||||
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Ramdisk ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_RAM Ramdisk — $MY_ID ━━━"
|
||||
log "$ICON_RAM Path: $RAMDISK_PATH"
|
||||
log "$ICON_RAM Size: $RAMDISK_SIZE"
|
||||
echo ""
|
||||
|
||||
START=$(date +%s)
|
||||
SETUP_SUCCESS=true
|
||||
|
||||
if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
|
||||
USAGE=$(df -BG "$RAMDISK_PATH" | awk 'NR==2 {print $3}')
|
||||
AVAIL=$(df -BG "$RAMDISK_PATH" | awk 'NR==2 {print $4}')
|
||||
log "Ramdisk already mounted — $USAGE used / $AVAIL available"
|
||||
log "Skipping mount — verifying symlink and permissions"
|
||||
else
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would create $RAMDISK_PATH"
|
||||
warn "DRY RUN — would mount tmpfs ${RAMDISK_SIZE} at $RAMDISK_PATH"
|
||||
else
|
||||
log "Creating ramdisk mount point: $RAMDISK_PATH"
|
||||
mkdir -p "$RAMDISK_PATH" || {
|
||||
error "Failed to create $RAMDISK_PATH"
|
||||
notify "Ramdisk setup failed on $(hostname) ($MY_ID) — could not create mount point" \
|
||||
"Ramdisk Setup" "warning"
|
||||
exit 1
|
||||
}
|
||||
|
||||
log "Mounting tmpfs ${RAMDISK_SIZE} at $RAMDISK_PATH..."
|
||||
if mount -t tmpfs -o size="$RAMDISK_SIZE" tmpfs "$RAMDISK_PATH"; then
|
||||
warn "Ramdisk mounted — ${RAMDISK_SIZE} at $RAMDISK_PATH ✅"
|
||||
else
|
||||
error "Failed to mount ramdisk at $RAMDISK_PATH"
|
||||
notify "Ramdisk setup failed on $(hostname) ($MY_ID) — mount failed" \
|
||||
"Ramdisk Setup" "warning"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ SSD Fallback ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_DISK SSD Fallback ━━━"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would create SSD fallback: $TRANSCODE_SSD"
|
||||
else
|
||||
if [[ -d "$TRANSCODE_SSD" ]]; then
|
||||
log "SSD fallback already exists: $TRANSCODE_SSD"
|
||||
else
|
||||
log "Creating SSD fallback directory: $TRANSCODE_SSD"
|
||||
if mkdir -p "$TRANSCODE_SSD"; then
|
||||
log "SSD fallback created: $TRANSCODE_SSD ✅"
|
||||
else
|
||||
error "Failed to create SSD fallback: $TRANSCODE_SSD"
|
||||
SETUP_SUCCESS=false
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Symlink ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_LINK Symlink ━━━"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would set $TRANSCODE_LINK → $RAMDISK_PATH"
|
||||
else
|
||||
if [[ -L "$TRANSCODE_LINK" ]]; then
|
||||
CURRENT_TARGET=$(readlink "$TRANSCODE_LINK")
|
||||
if [[ "$CURRENT_TARGET" == "$RAMDISK_PATH" ]]; then
|
||||
log "Symlink already points to ramdisk — no change needed ✅"
|
||||
else
|
||||
log "Updating symlink: $CURRENT_TARGET → $RAMDISK_PATH"
|
||||
ln -sfn "$RAMDISK_PATH" "$TRANSCODE_LINK" || {
|
||||
error "Failed to update symlink"
|
||||
SETUP_SUCCESS=false
|
||||
}
|
||||
fi
|
||||
elif [[ -e "$TRANSCODE_LINK" ]]; then
|
||||
warn "$TRANSCODE_LINK exists but is not a symlink — removing and replacing"
|
||||
rm -rf "$TRANSCODE_LINK"
|
||||
ln -sfn "$RAMDISK_PATH" "$TRANSCODE_LINK" || {
|
||||
error "Failed to create symlink"
|
||||
SETUP_SUCCESS=false
|
||||
}
|
||||
else
|
||||
log "Creating symlink: $TRANSCODE_LINK → $RAMDISK_PATH"
|
||||
mkdir -p "$(dirname "$TRANSCODE_LINK")"
|
||||
ln -sfn "$RAMDISK_PATH" "$TRANSCODE_LINK" || {
|
||||
error "Failed to create symlink"
|
||||
SETUP_SUCCESS=false
|
||||
}
|
||||
fi
|
||||
|
||||
[[ "$SETUP_SUCCESS" == true ]] && log "Symlink: $TRANSCODE_LINK → $RAMDISK_PATH ✅"
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Transcoding-temp Directory ━━━
|
||||
# ==============================================================================================
|
||||
# Pre-created inside ramdisk so Emby always finds it there at session start.
|
||||
# Without this Emby creates it at its own first-writable path — which may be
|
||||
# SSD even when the symlink points at the ramdisk — locking all sessions onto SSD.
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Transcoding Temp Directory ━━━"
|
||||
|
||||
TRANSCODE_TEMP_DIR="${RAMDISK_PATH}/transcoding-temp"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would create $TRANSCODE_TEMP_DIR"
|
||||
else
|
||||
if [[ -d "$TRANSCODE_TEMP_DIR" ]]; then
|
||||
log "transcoding-temp already exists on ramdisk"
|
||||
else
|
||||
if mkdir -p "$TRANSCODE_TEMP_DIR"; then
|
||||
log "Created transcoding-temp on ramdisk ✅"
|
||||
else
|
||||
error "Failed to create transcoding-temp on ramdisk"
|
||||
SETUP_SUCCESS=false
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ -d "$TRANSCODE_TEMP_DIR" ]]; then
|
||||
chmod "$TRANSCODE_CHMOD" "$TRANSCODE_TEMP_DIR"
|
||||
chown "$TRANSCODE_OWNER" "$TRANSCODE_TEMP_DIR"
|
||||
log "Permissions set on transcoding-temp ($TRANSCODE_CHMOD $TRANSCODE_OWNER)"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Permissions ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Permissions ━━━"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would apply $TRANSCODE_CHMOD $TRANSCODE_OWNER to $RAMDISK_PATH and $TRANSCODE_SSD"
|
||||
else
|
||||
for path in "$RAMDISK_PATH" "$TRANSCODE_SSD"; do
|
||||
if [[ -d "$path" ]]; then
|
||||
chmod "$TRANSCODE_CHMOD" "$path"
|
||||
chown "$TRANSCODE_OWNER" "$path"
|
||||
log "Permissions set: $path ($TRANSCODE_CHMOD $TRANSCODE_OWNER)"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Initialise State File ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
STATE_FILE="${TRANSCODE_STATE_FILE:-${STATE_DIR:-/tmp}/transcode_state.db}"
|
||||
NOW=$(date +%s)
|
||||
cat > "$STATE_FILE" <<EOF
|
||||
current_target=$RAMDISK_PATH
|
||||
last_flip_time=$NOW
|
||||
flip_count_hour=0
|
||||
flip_hour_start=$NOW
|
||||
EOF
|
||||
log "State file initialised: $STATE_FILE"
|
||||
fi
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY RAMDISK SETUP SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_RAM Ramdisk: $RAMDISK_PATH ($RAMDISK_SIZE)"
|
||||
echo "$ICON_DISK Fallback: $TRANSCODE_SSD"
|
||||
echo "$ICON_LINK Symlink: $TRANSCODE_LINK"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no changes made"
|
||||
elif [[ "$SETUP_SUCCESS" == true ]]; then
|
||||
echo "$ICON_DONE Status: done ✅"
|
||||
else
|
||||
echo "$ICON_ERROR Status: SETUP HAD ERRORS"
|
||||
notify "Ramdisk setup errors on $(hostname) ($MY_ID) — check output" \
|
||||
"Ramdisk Setup" "warning"
|
||||
exit 1
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
+166
@@ -0,0 +1,166 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ================================= Unraid API Key Renewal ====================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Creates/overwrites the Varaverk API key in the unraid-api service registry at
|
||||
# array start. The registry is ephemeral — OS updates and service restarts clear
|
||||
# it. This script re-registers the key every boot so Varaverk's enhanced
|
||||
# monitoring self-heals without manual intervention.
|
||||
#
|
||||
# Also updates HOST*_UNRAID_API_KEY in the local host conf so the partnership
|
||||
# page always reflects the live key value.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# unraid_api_key_renew.sh
|
||||
# Renew the key. Silent on success.
|
||||
#
|
||||
# unraid_api_key_renew.sh --dry-run
|
||||
# Show what would happen — no changes made.
|
||||
#
|
||||
# unraid_api_key_renew.sh --log
|
||||
# Verbose output.
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../../../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
acquire_lock
|
||||
detect_hosts
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────────
|
||||
CONF_FILE="$SCRIPT_DIR/../../../Configurations/${MY_ID,,}.conf"
|
||||
VAR_NAME="${MY_ID}_UNRAID_API_KEY"
|
||||
|
||||
# Key name: "Varaverk <hostname>" stripping any unraid- prefix
|
||||
# Space separator — unRAID API only allows letters, numbers, and spaces
|
||||
HOSTNAME_SUFFIX=$(hostname -s 2>/dev/null | sed 's/^[Uu][Nn][Rr][Aa][Ii][Dd]-//' || hostname -s)
|
||||
KEY_NAME="Varaverk ${HOSTNAME_SUFFIX}"
|
||||
|
||||
log "$ICON_GEAR Conf file: $CONF_FILE"
|
||||
log "$ICON_GEAR Key var: $VAR_NAME"
|
||||
log "$ICON_GEAR Key name: $KEY_NAME"
|
||||
|
||||
if [[ ! -f "$CONF_FILE" ]]; then
|
||||
error "Conf file not found: $CONF_FILE"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would check registry for $KEY_NAME, renew only if missing"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────────
|
||||
# Check if key already exists in the unraid-api registry before creating.
|
||||
# --overwrite generates a new key value every time, invalidating the old one.
|
||||
# Only renew if the registry has lost it.
|
||||
log "Checking unraid-api registry for $KEY_NAME..."
|
||||
EXISTING=$(timeout 5 /usr/local/sbin/unraid-api apikey --name "$KEY_NAME" --json </dev/null 2>/dev/null)
|
||||
KEY=$(echo "$EXISTING" | jq -r '.key // empty' 2>/dev/null)
|
||||
|
||||
if [[ -n "$KEY" ]]; then
|
||||
PREVIEW="${KEY:0:8}...${KEY: -4}"
|
||||
# Always sync registry key → conf, even if the key was already there.
|
||||
# Conf gets wiped on git pull / conf regeneration without touching the registry.
|
||||
CONF_HAS_KEY=$(grep -oP "(?<=^\s*${VAR_NAME}=\")[^\"]*" "$CONF_FILE" 2>/dev/null || true)
|
||||
if [[ "$CONF_HAS_KEY" == "$KEY" ]]; then
|
||||
echo "API key valid ✅ — $VAR_NAME = $PREVIEW"
|
||||
log "Key in registry and conf — no action needed"
|
||||
exit 0
|
||||
fi
|
||||
log "Key in registry but conf is stale — syncing..."
|
||||
if grep -q "^\s*${VAR_NAME}\s*=" "$CONF_FILE"; then
|
||||
sed -i "s|^\(\s*${VAR_NAME}\s*=\s*\)\"[^\"]*\"|\1\"${KEY}\"|" "$CONF_FILE"
|
||||
else
|
||||
echo " ${VAR_NAME}=\"${KEY}\"" >> "$CONF_FILE"
|
||||
fi
|
||||
echo "API key synced to conf ✅ — $VAR_NAME = $PREVIEW"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
log "Key not found in registry — creating new key..."
|
||||
|
||||
RAW=$(timeout 10 /usr/local/sbin/unraid-api apikey \
|
||||
--name "$KEY_NAME" --create --overwrite \
|
||||
--description "Varaverk plugin" --roles ADMIN --json </dev/null 2>&1)
|
||||
|
||||
if [[ -z "$RAW" ]]; then
|
||||
error "unraid-api returned no output"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
KEY=$(echo "$RAW" | jq -r '.key // empty' 2>/dev/null)
|
||||
if [[ -z "$KEY" ]]; then
|
||||
error "No key in unraid-api response: ${RAW:0:200}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────────
|
||||
if grep -q "^\s*${VAR_NAME}\s*=" "$CONF_FILE"; then
|
||||
sed -i "s|^\(\s*${VAR_NAME}\s*=\s*\)\"[^\"]*\"|\1\"${KEY}\"|" "$CONF_FILE"
|
||||
else
|
||||
echo " ${VAR_NAME}=\"${KEY}\"" >> "$CONF_FILE"
|
||||
fi
|
||||
|
||||
PREVIEW="${KEY:0:8}...${KEY: -4}"
|
||||
log "Writing new key to: $CONF_FILE"
|
||||
warn "API key renewed ✅ — $VAR_NAME = $PREVIEW (registry had lost it)"
|
||||
|
||||
# ── Push renewed key into each partner's OWN conf ─────────────────────────────
|
||||
# Each host's conf is its complete keychest — no cross-host conf files needed.
|
||||
# SSH_KEY is set by detect_hosts() — this server's outbound private key.
|
||||
if [[ -z "$SSH_KEY" ]]; then
|
||||
log "No SSH key configured — skipping partner push"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
for host_var in $(compgen -v | grep -E '^HOST[0-9]+$'); do
|
||||
partner_host="${!host_var}"
|
||||
[[ -z "$partner_host" ]] && continue
|
||||
[[ "${host_var,,}" == "${MY_ID,,}" ]] && continue
|
||||
|
||||
partner_slot="${host_var,,}" # e.g. host2
|
||||
partner_ip=$(resolve_tailscale_ip "$partner_host" 2>/dev/null || true)
|
||||
[[ -z "$partner_ip" ]] && { log "Cannot resolve IP for $partner_host — skipping"; continue; }
|
||||
|
||||
# Target is the partner's OWN conf on their machine
|
||||
partner_conf="/boot/config/plugins/varaverk/Configurations/${partner_slot}.conf"
|
||||
tmp=$(mktemp /tmp/vv_kp_XXXXXX.sh)
|
||||
remote="/tmp/vv_kp_${RANDOM}.sh"
|
||||
chmod 700 "$tmp"
|
||||
|
||||
# Key stays in the temp file — never appears in SSH command args
|
||||
cat > "$tmp" <<PUSHSCRIPT
|
||||
#!/bin/sh
|
||||
target='${partner_conf}'
|
||||
if grep -q "\b${VAR_NAME}\b" "\$target" 2>/dev/null; then
|
||||
sed -i 's|^\(\\s*${VAR_NAME}\\s*=\\s*\)"[^"]*"|\1"${KEY}"|' "\$target"
|
||||
else
|
||||
printf ' ${VAR_NAME}="%s"\n' '${KEY}' >> "\$target"
|
||||
fi
|
||||
echo ok
|
||||
PUSHSCRIPT
|
||||
|
||||
if timeout 10 scp -i "$SSH_KEY" -o ConnectTimeout=10 -o BatchMode=yes \
|
||||
-o StrictHostKeyChecking=no "$tmp" "root@${partner_ip}:${remote}" 2>/dev/null; then
|
||||
if timeout 10 ssh -i "$SSH_KEY" -o ConnectTimeout=10 -o BatchMode=yes \
|
||||
-o StrictHostKeyChecking=no "root@${partner_ip}" \
|
||||
"bash '${remote}'; rc=\$?; rm -f '${remote}'; exit \$rc" 2>/dev/null | grep -q ok; then
|
||||
log "Key pushed to $partner_host ✅"
|
||||
else
|
||||
warn "Key push to $partner_host failed — they can create their own copy"
|
||||
fi
|
||||
else
|
||||
warn "SCP to $partner_host failed — skipping"
|
||||
fi
|
||||
rm -f "$tmp"
|
||||
done
|
||||
+1124
File diff suppressed because it is too large
Load Diff
+317
@@ -0,0 +1,317 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ================================= Network Watchdog ===========================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Services-layer connectivity — checks that the outside world can actually reach
|
||||
# what it needs to reach. Silent when everything is reachable. Only fires when
|
||||
# something in the connectivity chain has broken.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Check 1 — Internet Connectivity
|
||||
# curl external endpoint → fail = alert + skip all remaining checks.
|
||||
# Internet down means DDNS and NPM checks would false-positive — gating prevents noise.
|
||||
#
|
||||
# Check 2 — DDNS (Cloudflare)
|
||||
# Public IP via ifconfig.me vs DNS record via dig @1.1.1.1.
|
||||
# Match → pass (silent). Mismatch → restart DDNS container + notify.
|
||||
# Container restart triggers an immediate Cloudflare record update.
|
||||
#
|
||||
# Check 3 — Tailscale
|
||||
# tailscale status → Running → pass (silent). Not running → notify.
|
||||
# Notify only — no restart attempt. Tailscale state issues warrant human review.
|
||||
#
|
||||
# Check 4 — NPM Proxy (external check)
|
||||
# curl external URL → 2-strike system before restarting NginxProxyManager.
|
||||
# Strike 1: warn + notify. Strike 2: restart NPM + notify + clear strikes.
|
||||
# Strikes auto-clear when the external URL becomes reachable again.
|
||||
# External check — verifies the full stack (DNS → NPM → backend), not just NPM running.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Root Required
|
||||
# docker restart requires root.
|
||||
#
|
||||
# Single Instance Lock
|
||||
# acquire_lock prevents concurrent runs.
|
||||
#
|
||||
# Internet Gates All Checks
|
||||
# If internet is down, DDNS and NPM checks are skipped — no cascade of false positives.
|
||||
#
|
||||
# Strike Before Acting on NPM
|
||||
# Single curl failure could be transient DNS hiccup or CDN blip.
|
||||
# Two consecutive failures confirms NPM is the problem.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# master.conf
|
||||
#
|
||||
# NETWORK_WATCHDOG_ENABLED toggle entire watchdog (default: true)
|
||||
# NETWORK_WATCHDOG_INTERNET_URL endpoint for internet reachability check
|
||||
# NETWORK_WATCHDOG_INTERNET_TIMEOUT curl timeout in seconds for internet check
|
||||
# NETWORK_WATCHDOG_CHECK_TAILSCALE toggle tailscale check (default: true)
|
||||
# NETWORK_WATCHDOG_NPM_TIMEOUT curl timeout for NPM external check
|
||||
# NETWORK_WATCHDOG_NPM_STRIKE_LIMIT consecutive failures before NPM restart
|
||||
# NETWORK_WATCHDOG_NPM_STATE_FILE strike count persistence (/tmp — resets on reboot)
|
||||
#
|
||||
# host*.conf (host-specific)
|
||||
#
|
||||
# HOST*_NETWORK_WATCHDOG_DDNS_DOMAIN domain to resolve and compare to public IP
|
||||
# HOST*_NETWORK_WATCHDOG_DDNS_CONTAINER container to restart on DDNS mismatch
|
||||
# HOST*_NETWORK_WATCHDOG_NPM_URL external URL to test full proxy stack
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# network_watchdog.sh
|
||||
# Run all connectivity checks. Silent when healthy.
|
||||
#
|
||||
# network_watchdog.sh --dry-run
|
||||
# Run all checks without restarting any containers.
|
||||
#
|
||||
# network_watchdog.sh --status
|
||||
# Show configuration, current public IP, DNS record, NPM strike state.
|
||||
#
|
||||
# network_watchdog.sh --log
|
||||
# Verbose output — show each check result even when passing.
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Resolve host-specific config ━━━
|
||||
# ==============================================================================================
|
||||
_ddns_domain_var="${MY_ID}_NETWORK_WATCHDOG_DDNS_DOMAIN"
|
||||
_ddns_container_var="${MY_ID}_NETWORK_WATCHDOG_DDNS_CONTAINER"
|
||||
_npm_url_var="${MY_ID}_NETWORK_WATCHDOG_NPM_URL"
|
||||
|
||||
DDNS_DOMAIN="${!_ddns_domain_var:-}"
|
||||
DDNS_CONTAINER="${!_ddns_container_var:-}"
|
||||
NPM_URL="${!_npm_url_var:-}"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
acquire_lock
|
||||
|
||||
detect_hosts
|
||||
|
||||
[[ "${NETWORK_WATCHDOG_ENABLED:-true}" != "true" ]] && echo "Network watchdog disabled" && exit 0
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be restarted"
|
||||
|
||||
log "$ICON_GEAR Config: internet=${NETWORK_WATCHDOG_INTERNET_URL:-https://1.1.1.1} timeout=${NETWORK_WATCHDOG_INTERNET_TIMEOUT:-5}s tailscale=${NETWORK_WATCHDOG_CHECK_TAILSCALE:-true} npm-strikes=${NETWORK_WATCHDOG_NPM_STRIKE_LIMIT:-2}"
|
||||
log "$ICON_NET DDNS: ${DDNS_DOMAIN:-not configured} → ${DDNS_CONTAINER:-no container} NPM: ${NPM_URL:-not configured}"
|
||||
|
||||
touch "${NETWORK_WATCHDOG_NPM_STATE_FILE}" 2>/dev/null
|
||||
|
||||
# ━━━ Strike helpers ━━━
|
||||
get_strikes() { grep "^${1}:" "${2}" 2>/dev/null | cut -d: -f2 || echo "0"; }
|
||||
set_strikes() {
|
||||
local key="$1" count="$2" file="$3"
|
||||
if grep -q "^${key}:" "$file" 2>/dev/null; then
|
||||
sed -i "s|^${key}:.*|${key}:${count}|" "$file"
|
||||
else
|
||||
echo "${key}:${count}" >> "$file"
|
||||
fi
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY NETWORK WATCHDOG STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_GEAR Internet URL: ${NETWORK_WATCHDOG_INTERNET_URL:-https://1.1.1.1}"
|
||||
echo "$ICON_GEAR DDNS domain: ${DDNS_DOMAIN:-not configured}"
|
||||
echo "$ICON_GEAR DDNS container: ${DDNS_CONTAINER:-not configured}"
|
||||
echo "$ICON_GEAR Tailscale: ${NETWORK_WATCHDOG_CHECK_TAILSCALE:-true}"
|
||||
echo "$ICON_GEAR NPM URL: ${NPM_URL:-not configured}"
|
||||
echo "$ICON_GEAR NPM strikes: $(get_strikes "npm" "${NETWORK_WATCHDOG_NPM_STATE_FILE}") / ${NETWORK_WATCHDOG_NPM_STRIKE_LIMIT:-2}"
|
||||
echo ""
|
||||
echo "── Current State ──"
|
||||
|
||||
if curl -sf --max-time "${NETWORK_WATCHDOG_INTERNET_TIMEOUT:-5}" \
|
||||
"${NETWORK_WATCHDOG_INTERNET_URL:-https://1.1.1.1}" >/dev/null 2>&1; then
|
||||
echo " $ICON_SUCCESS Internet: reachable"
|
||||
else
|
||||
echo " $ICON_ERROR Internet: NOT reachable"
|
||||
fi
|
||||
|
||||
if [[ -n "$DDNS_DOMAIN" ]]; then
|
||||
_pub=$(curl -sf --max-time 5 https://ifconfig.me 2>/dev/null | tr -d '[:space:]')
|
||||
_dns=$(dig +short "$DDNS_DOMAIN" @1.1.1.1 2>/dev/null | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | head -1)
|
||||
echo " $ICON_GEAR Public IP: ${_pub:-unknown}"
|
||||
echo " $ICON_GEAR DNS record: ${_dns:-unknown}"
|
||||
[[ "$_pub" == "$_dns" ]] && \
|
||||
echo " $ICON_SUCCESS DDNS: in sync" || \
|
||||
echo " $ICON_ERROR DDNS: MISMATCH — public=$_pub dns=$_dns"
|
||||
else
|
||||
echo " $ICON_GEAR DDNS: not configured for $MY_ID"
|
||||
fi
|
||||
|
||||
if [[ "${NETWORK_WATCHDOG_CHECK_TAILSCALE:-true}" == "true" ]] && command -v tailscale >/dev/null 2>&1; then
|
||||
if tailscale status --json 2>/dev/null | grep -qE '"BackendState":\s*"Running"'; then
|
||||
echo " $ICON_SUCCESS Tailscale: running"
|
||||
else
|
||||
echo " $ICON_ERROR Tailscale: NOT running"
|
||||
fi
|
||||
else
|
||||
echo " $ICON_GEAR Tailscale: check disabled or not installed"
|
||||
fi
|
||||
|
||||
if [[ -n "$NPM_URL" ]]; then
|
||||
if curl -sf --max-time "${NETWORK_WATCHDOG_NPM_TIMEOUT:-10}" "$NPM_URL" >/dev/null 2>&1; then
|
||||
echo " $ICON_SUCCESS NPM proxy: reachable ($NPM_URL)"
|
||||
else
|
||||
echo " $ICON_ERROR NPM proxy: NOT reachable ($NPM_URL)"
|
||||
fi
|
||||
else
|
||||
echo " $ICON_GEAR NPM proxy: not configured for $MY_ID"
|
||||
fi
|
||||
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Check 1 — Internet Connectivity ━━━
|
||||
# ==============================================================================================
|
||||
echo "━━━ $ICON_NET Network Watchdog — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
|
||||
ISSUES=0
|
||||
|
||||
if ! curl -sf --max-time "${NETWORK_WATCHDOG_INTERNET_TIMEOUT:-5}" \
|
||||
"${NETWORK_WATCHDOG_INTERNET_URL:-https://1.1.1.1}" >/dev/null 2>&1; then
|
||||
warn "$ICON_ERROR Internet not reachable — skipping DDNS, Tailscale, and NPM checks"
|
||||
notify "Network watchdog: internet not reachable on $(hostname) ($MY_ID)" \
|
||||
"Network Watchdog" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log "$ICON_SUCCESS Internet reachable"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Check 2 — DDNS ━━━
|
||||
# ==============================================================================================
|
||||
if [[ -n "$DDNS_DOMAIN" ]] && [[ -n "$DDNS_CONTAINER" ]]; then
|
||||
PUBLIC_IP=$(curl -sf --max-time 5 https://ifconfig.me 2>/dev/null | tr -d '[:space:]')
|
||||
DNS_IP=$(dig +short "$DDNS_DOMAIN" @1.1.1.1 2>/dev/null | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | head -1)
|
||||
|
||||
if [[ -z "$PUBLIC_IP" ]]; then
|
||||
warn "Could not determine public IP — skipping DDNS check"
|
||||
elif [[ -z "$DNS_IP" ]]; then
|
||||
warn "Could not resolve $DDNS_DOMAIN — skipping DDNS check"
|
||||
elif [[ "$PUBLIC_IP" == "$DNS_IP" ]]; then
|
||||
log "$ICON_SUCCESS DDNS in sync — $DDNS_DOMAIN → $DNS_IP"
|
||||
else
|
||||
warn "$ICON_ERROR DDNS mismatch — public=$PUBLIC_IP dns=$DNS_IP"
|
||||
(( ISSUES++ ))
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would restart $DDNS_CONTAINER"
|
||||
else
|
||||
warn "Restarting $DDNS_CONTAINER to trigger Cloudflare update..."
|
||||
if docker restart "$DDNS_CONTAINER" >/dev/null 2>&1; then
|
||||
warn "$DDNS_CONTAINER restarted ✅"
|
||||
notify "DDNS mismatch on $(hostname) ($MY_ID) — $DDNS_DOMAIN was $DNS_IP, public is $PUBLIC_IP — $DDNS_CONTAINER restarted" \
|
||||
"Network Watchdog" "warning"
|
||||
else
|
||||
warn "$DDNS_CONTAINER restart failed"
|
||||
notify "DDNS mismatch on $(hostname) ($MY_ID) — $DDNS_CONTAINER restart failed — manual intervention needed" \
|
||||
"Network Watchdog" "warning"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
else
|
||||
log "DDNS check not configured for $MY_ID — skipping"
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Check 3 — Tailscale ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "${NETWORK_WATCHDOG_CHECK_TAILSCALE:-true}" == "true" ]]; then
|
||||
if ! command -v tailscale >/dev/null 2>&1; then
|
||||
log "Tailscale not installed — skipping"
|
||||
elif tailscale status --json 2>/dev/null | grep -qE '"BackendState":\s*"Running"'; then
|
||||
log "$ICON_SUCCESS Tailscale running"
|
||||
else
|
||||
warn "$ICON_ERROR Tailscale not in Running state"
|
||||
notify "Tailscale not running on $(hostname) ($MY_ID) — manual check needed" \
|
||||
"Network Watchdog" "warning"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Check 4 — NPM Proxy (external) ━━━
|
||||
# ==============================================================================================
|
||||
if [[ -n "$NPM_URL" ]]; then
|
||||
NPM_STRIKES=$(get_strikes "npm" "${NETWORK_WATCHDOG_NPM_STATE_FILE}")
|
||||
NPM_STRIKE_LIMIT="${NETWORK_WATCHDOG_NPM_STRIKE_LIMIT:-2}"
|
||||
|
||||
if curl -sf --max-time "${NETWORK_WATCHDOG_NPM_TIMEOUT:-10}" "$NPM_URL" >/dev/null 2>&1; then
|
||||
log "$ICON_SUCCESS NPM proxy reachable — $NPM_URL"
|
||||
if [[ "$NPM_STRIKES" -gt 0 ]]; then
|
||||
log "NPM strikes cleared (was $NPM_STRIKES)"
|
||||
set_strikes "npm" 0 "${NETWORK_WATCHDOG_NPM_STATE_FILE}"
|
||||
fi
|
||||
else
|
||||
NPM_STRIKES=$(( NPM_STRIKES + 1 ))
|
||||
set_strikes "npm" "$NPM_STRIKES" "${NETWORK_WATCHDOG_NPM_STATE_FILE}"
|
||||
(( ISSUES++ ))
|
||||
|
||||
if [[ "$NPM_STRIKES" -lt "$NPM_STRIKE_LIMIT" ]]; then
|
||||
warn "$ICON_ERROR NPM proxy not reachable — $NPM_URL (strike $NPM_STRIKES/$NPM_STRIKE_LIMIT)"
|
||||
notify "NPM proxy not reachable on $(hostname) ($MY_ID) — $NPM_URL (strike $NPM_STRIKES/$NPM_STRIKE_LIMIT)" \
|
||||
"Network Watchdog" "warning"
|
||||
else
|
||||
warn "$ICON_ERROR NPM proxy strike limit reached ($NPM_STRIKES) — restarting NginxProxyManager"
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would restart NginxProxyManager"
|
||||
else
|
||||
if docker restart NginxProxyManager >/dev/null 2>&1; then
|
||||
warn "NginxProxyManager restarted ✅"
|
||||
set_strikes "npm" 0 "${NETWORK_WATCHDOG_NPM_STATE_FILE}"
|
||||
notify "NPM proxy restarted on $(hostname) ($MY_ID) — $NPM_URL was unreachable for $NPM_STRIKES cycles" \
|
||||
"Network Watchdog" "warning"
|
||||
else
|
||||
warn "NginxProxyManager restart failed — manual intervention needed"
|
||||
notify "NPM proxy restart FAILED on $(hostname) ($MY_ID) — manual intervention needed" \
|
||||
"Network Watchdog" "warning"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
else
|
||||
log "NPM check not configured for $MY_ID — skipping"
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Exit ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$ISSUES" -gt 0 ]]; then
|
||||
exit 1
|
||||
else
|
||||
echo "Network healthy ✅ ($(date '+%H:%M:%S'))"
|
||||
exit 0
|
||||
fi
|
||||
+356
@@ -0,0 +1,356 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ============================= Daily Sync Maintenance =========================================
|
||||
# ==============================================================================================
|
||||
# Daily orchestrator — runs the full daily maintenance window in the correct order.
|
||||
# Schedule: 0 1 * * * (1am daily via User Scripts plugin)
|
||||
#
|
||||
# ── EXECUTION ORDER ───────────────────────────────────────────────────────────────────────────
|
||||
# Pre-sync:
|
||||
# git_pull_execute.sh — pull latest scripts first, always
|
||||
#
|
||||
# Arr Sync (arr_sync.sh):
|
||||
# Syncs Lidarr/Sonarr/Radarr libraries across all nodes bidirectionally.
|
||||
# All nodes agree on tracked library before any files are transferred.
|
||||
# Remote nodes that don't have an arr running are skipped gracefully.
|
||||
#
|
||||
# Rsync window (DAILY_SYNC_SHARES per host):
|
||||
# HOST*_DAILY_SYNC_SHARES — media shares spread to all nodes (no --delete)
|
||||
# HOST*_PERSONAL_SHARES — encrypted personal shares
|
||||
#
|
||||
# Post-sync maintenance (DAILY_MAINTENANCE_SCRIPTS):
|
||||
# media_shares_permissions.sh — fix ownership before arr cleanup
|
||||
# media_cleaner.sh anime — remove junk from anime shares
|
||||
# media_cleaner.sh media — remove junk from media shares
|
||||
# lidarr_cleanup.sh — remove orphaned music files (local arr = truth)
|
||||
# sonarr_cleanup.sh — remove orphaned TV files (local arr = truth)
|
||||
# radarr_cleanup.sh — remove orphaned movie files (local arr = truth)
|
||||
# docker_daily_restart.sh — restart containers needing daily restart
|
||||
#
|
||||
# ── WHY ORDER MATTERS ─────────────────────────────────────────────────────────────────────────
|
||||
# git pull first — maintenance runs on latest code, not yesterday's
|
||||
# arr sync before rsync — all nodes track the same library before files are spread;
|
||||
# prevents remote arrs from searching for content already owned
|
||||
# rsync before cleanup — cleanup sees fully spread state, rsync has no --delete
|
||||
# permissions before arr cleanup — arrs need correct ownership to delete/rename
|
||||
# arr cleanup after permissions — clean ownership = successful orphan deletion
|
||||
# docker restart last — containers already processed by cleanup
|
||||
#
|
||||
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
|
||||
# Bidirectional — same script runs on both servers, correct direction automatic.
|
||||
# detect_hosts() aliases DAILY_SYNC_SHARES and PERSONAL_SHARES from HOST*_ vars.
|
||||
# No manual HOST1/HOST2 comparisons — MY_ID routes correctly on any server.
|
||||
#
|
||||
# ── DRIVE TEMP HANDLING ───────────────────────────────────────────────────────────────────────
|
||||
# rsync.sh returns exit codes for temperature issues:
|
||||
# exit 1 = temp WARN — skip this share, continue to next
|
||||
# exit 2 = temp CRITICAL — abort ALL remaining syncs in this window
|
||||
# All other failures — skip share, continue to next
|
||||
#
|
||||
# ── SILENT WHEN HEALTHY ───────────────────────────────────────────────────────────────────────
|
||||
# Runs daily at 1am — clean run should produce minimal output.
|
||||
# Each job reports log() on success (silent), warn()/error() on failure (visible).
|
||||
# Summary always shown — gives window timing and share/job counts.
|
||||
# Notify only on failure — successful daily maintenance doesn't need notification.
|
||||
#
|
||||
# ── CONFIGURATION (master.conf + host*.conf) ───────────────────────────────────────────
|
||||
# HOST*_DAILY_SYNC_SHARES — shares pushed to mirror each day
|
||||
# HOST*_PERSONAL_SHARES — encrypted personal shares
|
||||
# DAILY_MAINTENANCE_SCRIPTS — maintenance jobs (permissions, cleanup, restart)
|
||||
# DAILY_RSYNC_ENABLED — enable/disable rsync section
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# daily_sync_maintenance.sh — normal run
|
||||
# daily_sync_maintenance.sh --dry-run — preview without syncing or changing
|
||||
# daily_sync_maintenance.sh --log — verbose per-share/per-job output
|
||||
# daily_sync_maintenance.sh --status — show configured shares and jobs
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
RSYNC_SCRIPT="$SCRIPT_DIR/../Rsync/rsync.sh"
|
||||
SCRIPTS_ROOT="$SCRIPT_DIR/.."
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
if ! command -v docker &>/dev/null; then
|
||||
error "Docker command not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
detect_hosts
|
||||
resolve_remote_ip
|
||||
|
||||
acquire_lock
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY DAILY SYNC STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_NET Remote: $REMOTE_ID ($REMOTE_SERVER_NAME)"
|
||||
echo "$ICON_SYNC Rsync enabled: ${RSYNC_ENABLED:-true}"
|
||||
echo "$ICON_SYNC Daily enabled: ${DAILY_RSYNC_ENABLED:-false}"
|
||||
echo ""
|
||||
echo "━━━ Daily Sync Shares ━━━"
|
||||
for share in "${DAILY_SYNC_SHARES[@]}"; do
|
||||
[[ -n "$share" ]] && echo " $ICON_SYNC $share"
|
||||
done
|
||||
for share in "${PERSONAL_SHARES[@]}"; do
|
||||
[[ -n "$share" ]] && echo " $ICON_SYNC $share (personal)"
|
||||
done
|
||||
echo ""
|
||||
echo "━━━ Daily Maintenance Scripts ━━━"
|
||||
for entry in "${DAILY_MAINTENANCE_SCRIPTS[@]}"; do
|
||||
[[ -z "$entry" ]] && continue
|
||||
echo " $ICON_GEAR ${entry##*/}"
|
||||
done
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ── BUILD SHARE LIST via detect_hosts aliases ─────────────────────────────────────────────────
|
||||
# detect_hosts() sets DAILY_SYNC_SHARES and PERSONAL_SHARES from HOST*_ vars
|
||||
# No manual HOST1/HOST2 comparison needed — aliased automatically per server
|
||||
# ==============================================================================================
|
||||
ALL_SHARES=()
|
||||
for share in "${DAILY_SYNC_SHARES[@]}"; do
|
||||
[[ -n "$share" ]] && ALL_SHARES+=("$share")
|
||||
done
|
||||
for share in "${PERSONAL_SHARES[@]}"; do
|
||||
[[ -n "$share" ]] && ALL_SHARES+=("$share")
|
||||
done
|
||||
SHARE_COUNT=${#ALL_SHARES[@]}
|
||||
|
||||
# ── Split maintenance scripts: git pull runs pre-sync, rest run post-sync ─────────────────────
|
||||
PRE_SYNC_SCRIPTS=()
|
||||
POST_SYNC_SCRIPTS=()
|
||||
for script_entry in "${DAILY_MAINTENANCE_SCRIPTS[@]}"; do
|
||||
[[ -z "$script_entry" ]] && continue
|
||||
script_name=$(basename "${script_entry%% *}")
|
||||
if [[ "$script_name" == "git_pull_execute.sh" ]]; then
|
||||
PRE_SYNC_SCRIPTS+=("$script_entry")
|
||||
else
|
||||
POST_SYNC_SCRIPTS+=("$script_entry")
|
||||
fi
|
||||
done
|
||||
|
||||
# Helper — run a maintenance script, track pass/fail
|
||||
run_job() {
|
||||
local script_entry="$1"
|
||||
local extra_dry=""
|
||||
[[ "$DRY_RUN" == true ]] && extra_dry="--dry-run"
|
||||
|
||||
read -r -a script_args <<< "$script_entry"
|
||||
local script_path="$SCRIPTS_ROOT/${script_args[0]}"
|
||||
local script_name
|
||||
script_name=$(basename "${script_args[0]}")
|
||||
local extra_args=("${script_args[@]:1}")
|
||||
|
||||
if [[ ! -f "$script_path" ]]; then
|
||||
error "$script_name — not found at $script_path"
|
||||
JOB_FAIL+=("$script_name")
|
||||
return 1
|
||||
fi
|
||||
|
||||
log "Running: $script_name ${extra_args[*]}"
|
||||
# shellcheck disable=SC2086
|
||||
if bash "$script_path" "${extra_args[@]}" $extra_dry; then
|
||||
log "$script_name — done ✅"
|
||||
JOB_PASS+=("$script_name ${extra_args[*]}")
|
||||
else
|
||||
error "$script_name — failed (exit $?)"
|
||||
JOB_FAIL+=("$script_name ${extra_args[*]}")
|
||||
fi
|
||||
}
|
||||
|
||||
WINDOW_START=$(date +%s)
|
||||
JOB_PASS=()
|
||||
JOB_FAIL=()
|
||||
PASS=()
|
||||
FAIL=()
|
||||
SHARE_TIMES=()
|
||||
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Daily Maintenance — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Pre-sync — git pull ━━━
|
||||
# ==============================================================================================
|
||||
if [[ ${#PRE_SYNC_SCRIPTS[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_GIT Pre-sync ━━━"
|
||||
for script_entry in "${PRE_SYNC_SCRIPTS[@]}"; do
|
||||
run_job "$script_entry"
|
||||
done
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Arr Sync — library reconciliation before file spreading ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Arr Sync ━━━"
|
||||
|
||||
ARR_SYNC_SCRIPT="$SCRIPTS_ROOT/Media/arr_sync.sh"
|
||||
if [[ "${ARR_SYNC_ENABLED:-true}" != "true" ]]; then
|
||||
echo "ARR_SYNC_ENABLED=false — skipping"
|
||||
elif [[ ! -f "$ARR_SYNC_SCRIPT" ]]; then
|
||||
warn "arr_sync.sh not found at $ARR_SYNC_SCRIPT — skipping"
|
||||
JOB_FAIL+=("arr_sync.sh")
|
||||
else
|
||||
_arr_sync_args=()
|
||||
[[ "$DRY_RUN" == true ]] && _arr_sync_args+=("--dry-run")
|
||||
if bash "$ARR_SYNC_SCRIPT" "${_arr_sync_args[@]}"; then
|
||||
echo "Arr sync complete ✅"
|
||||
JOB_PASS+=("arr_sync.sh")
|
||||
else
|
||||
warn "Arr sync completed with errors — continuing to rsync"
|
||||
JOB_FAIL+=("arr_sync.sh")
|
||||
fi
|
||||
unset _arr_sync_args
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Media Share Sync ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Media Share Sync — $SHARE_COUNT share(s) ━━━"
|
||||
|
||||
TOTAL_START=$(date +%s)
|
||||
SHARE_INDEX=0
|
||||
ABORT_ALL_SYNCS=false
|
||||
|
||||
if ! check_rsync_enabled "DAILY"; then
|
||||
warn "Daily rsync disabled — skipping all $SHARE_COUNT share syncs"
|
||||
warn "Proceeding to maintenance jobs..."
|
||||
elif [[ "$SHARE_COUNT" -eq 0 ]]; then
|
||||
warn "No shares configured for $MY_ID — check HOST*_DAILY_SYNC_SHARES in host*.conf"
|
||||
else
|
||||
# Pre-flight — connectivity then remote rootfs
|
||||
check_connectivity
|
||||
check_remote_rootfs
|
||||
|
||||
RSYNC_DRY=""
|
||||
[[ "$DRY_RUN" == true ]] && RSYNC_DRY="--dry-run"
|
||||
|
||||
for SHARE in "${ALL_SHARES[@]}"; do
|
||||
(( SHARE_INDEX++ ))
|
||||
SHARE_NAME=$(basename "$SHARE")
|
||||
SHARE_START=$(date +%s)
|
||||
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Share $SHARE_INDEX/$SHARE_COUNT: $SHARE_NAME ━━━"
|
||||
|
||||
if [[ "$ABORT_ALL_SYNCS" == true ]]; then
|
||||
warn "$SHARE_NAME — skipped (drive temps CRITICAL earlier in window)"
|
||||
FAIL+=("$SHARE_NAME:temp-critical")
|
||||
continue
|
||||
fi
|
||||
|
||||
bash "$RSYNC_SCRIPT" "$SHARE" $RSYNC_DRY
|
||||
RSYNC_EXIT=$?
|
||||
|
||||
SHARE_TIMES+=("$SHARE_NAME:$(( $(date +%s) - SHARE_START ))")
|
||||
|
||||
case "$RSYNC_EXIT" in
|
||||
0)
|
||||
PASS+=("$SHARE_NAME")
|
||||
log "$SHARE_NAME — done ✅"
|
||||
;;
|
||||
1)
|
||||
FAIL+=("$SHARE_NAME:temp-warn")
|
||||
warn "$SHARE_NAME skipped — drive temps too high"
|
||||
;;
|
||||
2)
|
||||
FAIL+=("$SHARE_NAME:temp-critical")
|
||||
ABORT_ALL_SYNCS=true
|
||||
error "$SHARE_NAME aborted — drive temps CRITICAL, stopping all remaining syncs"
|
||||
notify "Daily sync aborted on $(hostname) ($MY_ID) — drive temps CRITICAL during $SHARE_NAME" \
|
||||
"Daily Sync" "warning"
|
||||
;;
|
||||
*)
|
||||
FAIL+=("$SHARE_NAME")
|
||||
error "$SHARE_NAME failed (exit $RSYNC_EXIT) — continuing to next share"
|
||||
;;
|
||||
esac
|
||||
|
||||
done
|
||||
fi
|
||||
|
||||
TOTAL_END=$(date +%s)
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Post-sync Maintenance Jobs ━━━
|
||||
# ==============================================================================================
|
||||
if [[ ${#POST_SYNC_SCRIPTS[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_CLEAN Post-sync Maintenance ━━━"
|
||||
for script_entry in "${POST_SYNC_SCRIPTS[@]}"; do
|
||||
echo ""
|
||||
run_job "$script_entry"
|
||||
done
|
||||
fi
|
||||
|
||||
WINDOW_END=$(date +%s)
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY DAILY MAINTENANCE SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_TIME Window: $(date -d @"$WINDOW_START" '+%Y-%m-%d %H:%M:%S') → $(date -d @"$WINDOW_END" '+%H:%M:%S')"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( WINDOW_END - WINDOW_START )))"
|
||||
echo ""
|
||||
|
||||
echo "$ICON_SYNC Shares ($SHARE_COUNT):"
|
||||
for entry in "${SHARE_TIMES[@]}"; do
|
||||
sname="${entry%%:*}"
|
||||
sdur="${entry##*:}"
|
||||
if printf '%s\n' "${FAIL[@]}" | grep -q "^${sname}"; then
|
||||
echo " $ICON_ERROR $sname — $(format_duration "$sdur")"
|
||||
else
|
||||
echo " $ICON_DONE $sname — $(format_duration "$sdur")"
|
||||
fi
|
||||
done
|
||||
[[ "$SHARE_COUNT" -eq 0 || "${DAILY_RSYNC_ENABLED:-false}" == "false" ]] && \
|
||||
echo " (rsync disabled)"
|
||||
echo " Passed: ${#PASS[@]}/$SHARE_COUNT Failed: ${#FAIL[@]}/$SHARE_COUNT"
|
||||
echo ""
|
||||
|
||||
if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then
|
||||
echo "$ICON_GEAR Jobs:"
|
||||
for job in "${JOB_PASS[@]}"; do echo " $ICON_DONE $job"; done
|
||||
for job in "${JOB_FAIL[@]}"; do echo " $ICON_ERROR $job"; done
|
||||
echo ""
|
||||
fi
|
||||
|
||||
TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} ))
|
||||
|
||||
if [[ "$TOTAL_FAIL" -gt 0 ]]; then
|
||||
warn "Status: $TOTAL_FAIL failure(s)"
|
||||
notify "Daily maintenance completed with failures on $(hostname) ($MY_ID) — shares: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \
|
||||
"Daily Maintenance" "warning"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 1
|
||||
else
|
||||
echo "$ICON_DONE Status: all complete — ${#PASS[@]} share(s) synced, ${#JOB_PASS[@]} job(s) run"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
+308
@@ -0,0 +1,308 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ================================= ZFS Pool Scrub ============================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Triggers a ZFS scrub on all pools (or a specific pool), waits for completion,
|
||||
# and sends a notification with any errors found. Reads every block on every pool
|
||||
# and verifies checksums — catches silent corruption that would otherwise only
|
||||
# surface when the corrupted data is read (possibly after redundancy can no
|
||||
# longer help). Monthly recommended for all pools; quarterly minimum for large pools.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Starts a scrub on each pool, then polls every 60 seconds until all complete.
|
||||
# Progress is shown every poll — safe to leave running or interrupt. ZFS scrub
|
||||
# continues in the kernel even if the script is stopped — it does not depend on
|
||||
# this script remaining alive.
|
||||
#
|
||||
# Scrub is safe to run while the pool is in use. It does consume I/O bandwidth —
|
||||
# schedule during off-peak hours or maintenance windows.
|
||||
#
|
||||
# Pools in HOST*_ZFS_REPORT_IGNORE_POOLS are skipped automatically (single-disk
|
||||
# VM pools, temp pools, etc.). Specifying a pool by name bypasses the ignore list.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Single Instance Lock
|
||||
# acquire_lock prevents concurrent scrub starts on the same server.
|
||||
#
|
||||
# Scrub-in-Progress Check
|
||||
# Detects pools already scrubbing and skips them rather than erroring — safe
|
||||
# to run when a scrub may have been started by another path.
|
||||
#
|
||||
# SIGTERM Trap
|
||||
# The poll loop exits cleanly on signal. The ZFS scrub continues regardless.
|
||||
#
|
||||
# Tool Validation
|
||||
# platform_require_cmd confirms zpool and the notify script are present before use.
|
||||
#
|
||||
# Silent When Clean
|
||||
# Only errors produce visible output and a notification.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# host*.conf
|
||||
#
|
||||
# HOST*_ZFS_REPORT_IGNORE_POOLS
|
||||
# Pools to exclude from automatic scrub. Typically single-disk VM pools
|
||||
# or temporary pools that do not need integrity checking.
|
||||
# Aliased by detect_hosts() → ZFS_REPORT_IGNORE_POOLS.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# zfs_pool_scrub.sh
|
||||
# Scrub all pools not in ZFS_REPORT_IGNORE_POOLS. Wait for completion.
|
||||
#
|
||||
# zfs_pool_scrub.sh poolname
|
||||
# Scrub a specific pool by name. Bypasses the ignore list.
|
||||
#
|
||||
# zfs_pool_scrub.sh --status
|
||||
# Show current scrub status for all pools and exit.
|
||||
#
|
||||
# zfs_pool_scrub.sh --dry-run
|
||||
# Show which pools would be scrubbed. No scrub started.
|
||||
#
|
||||
# zfs_pool_scrub.sh --log
|
||||
# Verbose progress output every poll cycle.
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
TARGET_POOL="${PARSED_ARGS[0]:-}"
|
||||
SCRUB_RUNNING=true
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
platform_require_cmd \
|
||||
"$(command -v zpool 2>/dev/null || echo /sbin/zpool)" \
|
||||
"--version" "" \
|
||||
"zpool" || {
|
||||
error "ZFS not available on this system — zpool not found"
|
||||
exit 1
|
||||
}
|
||||
|
||||
|
||||
acquire_lock
|
||||
|
||||
# detect_hosts() sets MY_ID and aliases HOST*_ZFS_REPORT_IGNORE_POOLS
|
||||
detect_hosts
|
||||
|
||||
# Build ignore pool map
|
||||
declare -A IGNORE_MAP
|
||||
for pool in "${ZFS_REPORT_IGNORE_POOLS[@]:-}"; do
|
||||
[[ -n "$pool" ]] && IGNORE_MAP["$pool"]=1
|
||||
done
|
||||
|
||||
log "Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
log "Ignore pools: ${ZFS_REPORT_IGNORE_POOLS[*]:-none}"
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no scrubs will be started"
|
||||
|
||||
# SIGTERM trap — exit poll loop cleanly
|
||||
trap 'warn "ZFS scrub script interrupted — scrub continues in background"; SCRUB_RUNNING=false; exit 0' \
|
||||
SIGTERM SIGINT
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY ZFS SCRUB STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo ""
|
||||
while IFS= read -r pool; do
|
||||
[[ -z "$pool" ]] && continue
|
||||
SCAN=$(zpool status "$pool" 2>/dev/null | grep "scan:")
|
||||
IGNORED=""
|
||||
[[ -n "${IGNORE_MAP[$pool]:-}" ]] && IGNORED=" (ignored)"
|
||||
echo " $ICON_ZFS $pool${IGNORED} — ${SCAN:-no scan data}"
|
||||
done < <(zpool list -H -o name 2>/dev/null)
|
||||
echo ""
|
||||
echo " Ignored pools: ${ZFS_REPORT_IGNORE_POOLS[*]:-none}"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ── Build pool list ────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
POOLS_TO_SCRUB=()
|
||||
|
||||
if [[ -n "$TARGET_POOL" ]]; then
|
||||
# Specific pool — bypass ignore list, validate exists
|
||||
if ! zpool list "$TARGET_POOL" >/dev/null 2>&1; then
|
||||
error "Pool not found: $TARGET_POOL"
|
||||
exit 1
|
||||
fi
|
||||
POOLS_TO_SCRUB=("$TARGET_POOL")
|
||||
else
|
||||
# All pools — skip ignored ones
|
||||
while IFS= read -r pool; do
|
||||
[[ -z "$pool" ]] && continue
|
||||
if [[ -n "${IGNORE_MAP[$pool]:-}" ]]; then
|
||||
log "Skipping $pool (in ZFS_REPORT_IGNORE_POOLS)"
|
||||
continue
|
||||
fi
|
||||
POOLS_TO_SCRUB+=("$pool")
|
||||
done < <(zpool list -H -o name 2>/dev/null)
|
||||
fi
|
||||
|
||||
if [[ ${#POOLS_TO_SCRUB[@]} -eq 0 ]]; then
|
||||
warn "No pools to scrub — all pools may be on the ignore list"
|
||||
warn "Ignored: ${ZFS_REPORT_IGNORE_POOLS[*]:-none}"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
log "Pools to scrub: ${POOLS_TO_SCRUB[*]}"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Start Scrubs ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_ZFS Starting ZFS Scrubs — $MY_ID ━━━"
|
||||
START=$(date +%s)
|
||||
|
||||
STARTED=()
|
||||
SKIPPED_POOLS=()
|
||||
|
||||
for pool in "${POOLS_TO_SCRUB[@]}"; do
|
||||
|
||||
# Check if scrub already in progress
|
||||
ALREADY=$(zpool status "$pool" 2>/dev/null | grep "scan:" | grep -c "in progress" || true)
|
||||
if [[ "$ALREADY" -gt 0 ]]; then
|
||||
warn "$pool — scrub already in progress — joining existing scrub"
|
||||
STARTED+=("$pool")
|
||||
continue
|
||||
fi
|
||||
|
||||
log "Starting scrub on $pool..."
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would scrub: $pool"
|
||||
STARTED+=("$pool")
|
||||
elif zpool scrub "$pool" 2>/dev/null; then
|
||||
log "$pool scrub started ✅"
|
||||
STARTED+=("$pool")
|
||||
else
|
||||
error "Failed to start scrub on $pool"
|
||||
SKIPPED_POOLS+=("$pool")
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY ZFS SCRUB SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
warn "DRY RUN — no scrubs started"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ ${#STARTED[@]} -eq 0 ]]; then
|
||||
error "No scrubs were started — check pool status"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Poll Until Complete ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_TIME Waiting for Scrubs to Complete ━━━"
|
||||
log "Polling every 60 seconds — scrubs may take hours on large pools"
|
||||
log "Safe to interrupt — scrubs continue in background if script is stopped"
|
||||
|
||||
while [[ "$SCRUB_RUNNING" == true ]]; do
|
||||
sleep 60
|
||||
|
||||
STILL_RUNNING=false
|
||||
for pool in "${STARTED[@]}"; do
|
||||
IN_PROGRESS=$(zpool status "$pool" 2>/dev/null | \
|
||||
grep "scan:" | grep -c "in progress" || true)
|
||||
if [[ "$IN_PROGRESS" -gt 0 ]]; then
|
||||
STILL_RUNNING=true
|
||||
# Show progress — always visible so user knows it's running
|
||||
PROGRESS=$(zpool status "$pool" 2>/dev/null | \
|
||||
grep "scan:" | grep -oE "[0-9]+\.[0-9]+% done")
|
||||
REPAIRED=$(zpool status "$pool" 2>/dev/null | \
|
||||
grep "scan:" | grep -oE "[0-9]+ repaired")
|
||||
warn "$pool — scrub in progress ${PROGRESS:+$PROGRESS}${REPAIRED:+ ($REPAIRED)}"
|
||||
fi
|
||||
done
|
||||
|
||||
[[ "$STILL_RUNNING" == false ]] && SCRUB_RUNNING=false
|
||||
done
|
||||
|
||||
END=$(date +%s)
|
||||
warn "All scrubs complete — $(format_duration $(( END - START )))"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Results ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_ZFS Scrub Results ━━━"
|
||||
|
||||
POOLS_OK=()
|
||||
POOLS_ERRORS=()
|
||||
|
||||
for pool in "${STARTED[@]}"; do
|
||||
SCAN_LINE=$(zpool status "$pool" 2>/dev/null | grep "scan:")
|
||||
ERRORS=$(zpool status "$pool" 2>/dev/null | \
|
||||
grep "errors:" | grep -v "No known data errors")
|
||||
|
||||
if [[ -n "$ERRORS" ]]; then
|
||||
error "$pool — ERRORS FOUND"
|
||||
error " $SCAN_LINE"
|
||||
error " $ERRORS"
|
||||
POOLS_ERRORS+=("$pool")
|
||||
else
|
||||
echo "$pool — $SCAN_LINE"
|
||||
POOLS_OK+=("$pool")
|
||||
fi
|
||||
done
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY ZFS SCRUB SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_ZFS Pools: ${#POOLS_TO_SCRUB[@]} to scrub"
|
||||
echo "$ICON_SUCCESS Clean: ${#POOLS_OK[@]}"
|
||||
[[ ${#POOLS_ERRORS[@]} -gt 0 ]] && echo "$ICON_ERROR Errors: ${#POOLS_ERRORS[@]}"
|
||||
[[ ${#SKIPPED_POOLS[@]} -gt 0 ]] && warn "Failed start: ${SKIPPED_POOLS[*]}"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
|
||||
if [[ ${#POOLS_ERRORS[@]} -gt 0 ]]; then
|
||||
echo "$ICON_ERROR Status: ERRORS FOUND — ${POOLS_ERRORS[*]}"
|
||||
notify "ZFS scrub errors on $(hostname) ($MY_ID) — pools with errors: ${POOLS_ERRORS[*]}" \
|
||||
"ZFS Scrub" "warning"
|
||||
elif [[ ${#POOLS_OK[@]} -gt 0 ]]; then
|
||||
echo "$ICON_DONE Status: all ${#POOLS_OK[@]} pools clean ✅"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ ${#POOLS_ERRORS[@]} -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
+247
@@ -0,0 +1,247 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ============================= PHP-FPM Max Children ===========================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Raises PHP-FPM pm.max_children to prevent WebGUI slowdowns under load. Run
|
||||
# once at array start via ARRAY_START_SCRIPTS. Idempotent — silent when the
|
||||
# value is already correct, no restart on clean boot.
|
||||
#
|
||||
# unRAID's WebGUI runs through PHP-FPM. The default pm.max_children is very
|
||||
# low (4–8). Under load — multiple users, Docker operations, heavy dashboard
|
||||
# usage — all PHP workers saturate and new requests queue. The WebGUI becomes
|
||||
# slow or unresponsive.
|
||||
#
|
||||
# PHP_MAX_CHILDREN=250 is appropriate for 128GB RAM: ~2MB per worker = ~500MB
|
||||
# total. Too high wastes RAM; too low causes slowdowns.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# DESIGN PRINCIPLES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Idempotent Content Check
|
||||
# Reads the current pm.max_children value before writing. If already at
|
||||
# target → silent exit, no PHP-FPM restart. Restarting PHP-FPM unnecessarily
|
||||
# disrupts active WebGUI sessions on every boot.
|
||||
#
|
||||
# Pattern Match Before Write
|
||||
# Verifies the sed pattern finds pm.max_children in the config before
|
||||
# applying any change. Prevents silent failures where sed succeeds but
|
||||
# writes nothing because the key was missing or commented out.
|
||||
#
|
||||
# Apply Sequence
|
||||
# 1. Read current pm.max_children from PHP_CONF
|
||||
# 2. If already at target → exit silently
|
||||
# 3. Verify sed pattern matches before writing
|
||||
# 4. Apply sed replacement
|
||||
# 5. Restart PHP-FPM via rc.php-fpm
|
||||
# 6. Verify PHP-FPM process running after restart
|
||||
# 7. Read back config to confirm value applied
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Root Required
|
||||
# Writing to /etc/php83/ requires root.
|
||||
#
|
||||
# Single Instance Lock
|
||||
# acquire_lock prevents concurrent runs at array start.
|
||||
#
|
||||
# Process Verify
|
||||
# Confirms PHP-FPM running after restart — errors if it failed to start.
|
||||
#
|
||||
# Config Verify
|
||||
# Reads back config after restart to confirm the value was actually applied.
|
||||
#
|
||||
# Silent on Success
|
||||
# Runs every boot — no noise when already correct.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# master.conf
|
||||
#
|
||||
# PHP_MAX_CHILDREN
|
||||
# Target pm.max_children value. (default: 250)
|
||||
#
|
||||
# PHP_CONF
|
||||
# Path to PHP-FPM www.conf. (default: /etc/php83/php-fpm.d/www.conf)
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# php_fpm_max_children.sh
|
||||
# Read current value. Update and restart PHP-FPM only if changed. Silent if correct.
|
||||
#
|
||||
# php_fpm_max_children.sh --dry-run
|
||||
# Show current vs target value. No config write or restart.
|
||||
#
|
||||
# php_fpm_max_children.sh --status
|
||||
# Show current pm.max_children, target, and PHP-FPM process state.
|
||||
#
|
||||
# php_fpm_max_children.sh --log
|
||||
# Verbose output showing idempotent check, config write, and restart result.
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../../../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root — writing system config requires root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
platform_require_cmd \
|
||||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||||
"" "" \
|
||||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||
|
||||
validate_int PHP_MAX_CHILDREN "$PHP_MAX_CHILDREN"
|
||||
require_var PHP_CONF
|
||||
|
||||
acquire_lock
|
||||
|
||||
detect_hosts
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY PHP-FPM STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_GEAR Config file: $PHP_CONF"
|
||||
echo "$ICON_PHP Target: pm.max_children = $PHP_MAX_CHILDREN"
|
||||
echo ""
|
||||
|
||||
if [[ -f "$PHP_CONF" ]]; then
|
||||
CURRENT_VAL=$(grep -E "^pm\.max_children" "$PHP_CONF" 2>/dev/null | \
|
||||
awk '{print $NF}')
|
||||
if [[ "${CURRENT_VAL:-0}" -eq "$PHP_MAX_CHILDREN" ]]; then
|
||||
echo " $ICON_SUCCESS Current: pm.max_children = $CURRENT_VAL (correct ✅)"
|
||||
else
|
||||
echo " $ICON_WARN Current: pm.max_children = ${CURRENT_VAL:-not set} (would update)"
|
||||
fi
|
||||
else
|
||||
echo " $ICON_ERROR Config file not found: $PHP_CONF"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
if pgrep -f "php-fpm" >/dev/null 2>&1; then
|
||||
FPM_COUNT=$(pgrep -fc "php-fpm" 2>/dev/null || echo "?")
|
||||
echo " $ICON_SUCCESS PHP-FPM: running ($FPM_COUNT worker(s))"
|
||||
else
|
||||
echo " $ICON_ERROR PHP-FPM: NOT running"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ PHP-FPM Config ━━━
|
||||
# ==============================================================================================
|
||||
START=$(date +%s)
|
||||
|
||||
if [[ ! -f "$PHP_CONF" ]]; then
|
||||
error "PHP config file not found: $PHP_CONF"
|
||||
notify "PHP-FPM config not found on $(hostname) ($MY_ID) — $PHP_CONF missing" \
|
||||
"PHP-FPM" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ── Idempotent check ─────────────────────────────────────────────────────────────────────────
|
||||
CURRENT_VAL=$(grep -E "^pm\.max_children" "$PHP_CONF" 2>/dev/null | awk '{print $NF}')
|
||||
if [[ "${CURRENT_VAL:-0}" -eq "$PHP_MAX_CHILDREN" ]]; then
|
||||
echo "pm.max_children already $PHP_MAX_CHILDREN ✅"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
warn "pm.max_children: ${CURRENT_VAL:-not set} → $PHP_MAX_CHILDREN"
|
||||
|
||||
# ── Verify pattern exists before writing ─────────────────────────────────────────────────────
|
||||
if ! grep -qE "^pm\.max_children" "$PHP_CONF" 2>/dev/null; then
|
||||
error "pm.max_children not found in $PHP_CONF — cannot apply"
|
||||
error "Add 'pm.max_children = $PHP_MAX_CHILDREN' to $PHP_CONF manually"
|
||||
notify "PHP-FPM pm.max_children not found in config on $(hostname) ($MY_ID)" \
|
||||
"PHP-FPM" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would set pm.max_children = $PHP_MAX_CHILDREN in $PHP_CONF"
|
||||
warn "DRY RUN — would restart PHP-FPM"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ── Apply setting ─────────────────────────────────────────────────────────────────────────────
|
||||
log "Applying pm.max_children = $PHP_MAX_CHILDREN..."
|
||||
if ! sed -i "s/^pm\.max_children.*/pm.max_children = $PHP_MAX_CHILDREN/" "$PHP_CONF"; then
|
||||
error "Failed to update $PHP_CONF"
|
||||
notify "PHP-FPM config update failed on $(hostname) ($MY_ID)" \
|
||||
"PHP-FPM" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log "Config updated"
|
||||
|
||||
# ── Restart PHP-FPM ──────────────────────────────────────────────────────────────────────────
|
||||
log "Restarting PHP-FPM..."
|
||||
if ! platform_restart_service php-fpm; then
|
||||
error "PHP-FPM restart command failed"
|
||||
notify "PHP-FPM restart failed on $(hostname) ($MY_ID)" \
|
||||
"PHP-FPM" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
sleep 3 # Allow PHP-FPM workers to initialise
|
||||
|
||||
# ── Verify process running ────────────────────────────────────────────────────────────────────
|
||||
if ! pgrep -f "php-fpm" >/dev/null 2>&1; then
|
||||
error "PHP-FPM not running after restart — WebGUI may be broken"
|
||||
notify "PHP-FPM failed to start after config update on $(hostname) ($MY_ID)" \
|
||||
"PHP-FPM" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ── Verify config reflects target ────────────────────────────────────────────────────────────
|
||||
APPLIED_VAL=$(grep -E "^pm\.max_children" "$PHP_CONF" 2>/dev/null | awk '{print $NF}')
|
||||
if [[ "${APPLIED_VAL:-0}" -ne "$PHP_MAX_CHILDREN" ]]; then
|
||||
warn "Config reads pm.max_children = ${APPLIED_VAL:-unknown} — expected $PHP_MAX_CHILDREN"
|
||||
warn "Check $PHP_CONF manually"
|
||||
else
|
||||
log "Verified: pm.max_children = $APPLIED_VAL ✅"
|
||||
fi
|
||||
|
||||
FPM_WORKERS=$(pgrep -fc "php-fpm" 2>/dev/null || echo "?")
|
||||
log "$ICON_PHP Workers running: $FPM_WORKERS"
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY PHP-FPM SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_GEAR Config file: $PHP_CONF"
|
||||
echo "$ICON_PHP Applied: pm.max_children = $PHP_MAX_CHILDREN"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
echo "$ICON_DONE Status: done ✅"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
exit 0
|
||||
+334
@@ -0,0 +1,334 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ============================= Emby Session Report ============================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Emby usage report via the Emby API. Scheduled weekly (Sunday 11am). Queries
|
||||
# activity logs and session history to produce a summary of what was watched,
|
||||
# by whom, and how often. No persistent state — queries fresh on every run.
|
||||
#
|
||||
# Reports: server info and uptime, active sessions and transcode ratio, library
|
||||
# counts (movies/episodes/songs), activity history for the last EMBY_REPORT_DAYS
|
||||
# days, top EMBY_REPORT_TOP_N content items, most active users, and ramdisk
|
||||
# transcode status.
|
||||
#
|
||||
# Notifies only if transcoding exceeds 80% of streams — may indicate a client
|
||||
# configuration issue. Silent on clean runs.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Single Instance Lock
|
||||
# acquire_lock prevents duplicate reports running simultaneously.
|
||||
#
|
||||
# API Connectivity Check
|
||||
# check_api() verifies Emby is reachable before any queries. API failure in
|
||||
# one section does not abort the others — each section guards itself.
|
||||
#
|
||||
# Tool Validation
|
||||
# Checks for curl and jq at startup — exits with a clear error if either is missing.
|
||||
#
|
||||
# Per-Host Credentials
|
||||
# detect_hosts() aliases HOST*_EMBY_URL and HOST*_EMBY_API_KEY → EMBY_URL / EMBY_API_KEY.
|
||||
# Each server reports on its own Emby instance automatically.
|
||||
#
|
||||
# Notification Validated
|
||||
# platform_require_cmd confirms the notify script is present before use.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# host*.conf
|
||||
#
|
||||
# HOST*_EMBY_URL
|
||||
# Emby server URL for this host. Aliased by detect_hosts() → EMBY_URL.
|
||||
#
|
||||
# HOST*_EMBY_API_KEY
|
||||
# Emby API key for this host. Aliased by detect_hosts() → EMBY_API_KEY.
|
||||
# Generate via Emby UI → Settings → API Keys → New API Key.
|
||||
#
|
||||
# master.conf
|
||||
#
|
||||
# EMBY_REPORT_DAYS
|
||||
# Number of days to include in the activity history section. (default: 7)
|
||||
#
|
||||
# EMBY_REPORT_TOP_N
|
||||
# Number of top content items to show in the report. (default: 10)
|
||||
#
|
||||
# RAMDISK_PATH
|
||||
# Ramdisk mount path — used for transcode status reporting.
|
||||
#
|
||||
# TRANSCODE_LINK
|
||||
# Symlink path — used to determine current transcode location.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# emby_session_report.sh
|
||||
# Generate and send Emby usage report.
|
||||
#
|
||||
# emby_session_report.sh --dry-run
|
||||
# Test API connectivity and generate report output. No notification sent.
|
||||
#
|
||||
# emby_session_report.sh --status
|
||||
# Show Emby URL, API key (masked), and report configuration. Then exit.
|
||||
#
|
||||
# emby_session_report.sh --log
|
||||
# Verbose per-section output during report generation.
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v curl >/dev/null 2>&1; then
|
||||
error "curl not found — required for Emby API calls"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v jq >/dev/null 2>&1; then
|
||||
error "jq not found — required for JSON parsing"
|
||||
notify "Emby report failed on $(hostname) — jq not installed" "Emby Report" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
acquire_lock
|
||||
|
||||
# detect_hosts() sets MY_ID and aliases EMBY_URL, EMBY_API_KEY
|
||||
detect_hosts
|
||||
|
||||
require_var EMBY_URL
|
||||
require_var EMBY_API_KEY
|
||||
|
||||
log "$ICON_GEAR Config: url=${EMBY_URL} period=${EMBY_REPORT_DAYS}d top=${EMBY_REPORT_TOP_N}"
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — API queried but no notification sent"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_EMBY Emby URL: $EMBY_URL"
|
||||
echo "$ICON_TIME Period: Last ${EMBY_REPORT_DAYS} days"
|
||||
echo "$ICON_EMBY Top N: ${EMBY_REPORT_TOP_N} items"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ── API HELPER ────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
emby_api() {
|
||||
local endpoint="$1"
|
||||
local response http_code body
|
||||
|
||||
response=$(curl -sf \
|
||||
--max-time 15 \
|
||||
-H "X-Emby-Token: $EMBY_API_KEY" \
|
||||
-w "\n%{http_code}" \
|
||||
"${EMBY_URL}/${endpoint}" 2>/dev/null)
|
||||
|
||||
http_code=$(echo "$response" | tail -1)
|
||||
body=$(echo "$response" | head -n -1)
|
||||
|
||||
if [[ "$http_code" != "200" ]]; then
|
||||
error "Emby API HTTP $http_code for: $endpoint"
|
||||
return 1
|
||||
fi
|
||||
echo "$body"
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Emby Session Report ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_EMBY Emby Session Report — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_TIME Period: Last ${EMBY_REPORT_DAYS} days"
|
||||
echo ""
|
||||
|
||||
START=$(date +%s)
|
||||
|
||||
# ── Connectivity and server info ──────────────────────────────────────────────────────────────
|
||||
if ! check_api "$EMBY_URL" "Emby" 10; then
|
||||
notify "Emby report failed on $(hostname) — cannot connect to Emby at $EMBY_URL" \
|
||||
"Emby Report" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
SYSTEM_INFO=$(emby_api "System/Info" 2>/dev/null) || {
|
||||
error "Cannot connect to Emby at $EMBY_URL"
|
||||
exit 1
|
||||
}
|
||||
|
||||
SERVER_NAME=$(echo "$SYSTEM_INFO" | jq -r '.ServerName // "Unknown"' 2>/dev/null)
|
||||
SERVER_VERSION=$(echo "$SYSTEM_INFO" | jq -r '.Version // "Unknown"' 2>/dev/null)
|
||||
log "$ICON_EMBY Connected to: $SERVER_NAME (v$SERVER_VERSION) ✅"
|
||||
|
||||
# ── Active Sessions ───────────────────────────────────────────────────────────────────────────
|
||||
echo "━━━ $ICON_EMBY Active Sessions ━━━"
|
||||
SESSIONS=$(emby_api "Sessions" 2>/dev/null) || { warn "Could not fetch sessions"; SESSIONS="[]"; }
|
||||
|
||||
ACTIVE_COUNT=$(echo "$SESSIONS" | \
|
||||
jq '[.[] | select(.NowPlayingItem != null)] | length' 2>/dev/null || echo 0)
|
||||
TRANSCODE_NOW=$(echo "$SESSIONS" | \
|
||||
jq '[.[] | select(.NowPlayingItem != null) | select(.TranscodingInfo != null)] | length' \
|
||||
2>/dev/null || echo 0)
|
||||
DIRECT_NOW=$(( ACTIVE_COUNT - TRANSCODE_NOW ))
|
||||
|
||||
echo " $ICON_EMBY Active streams: $ACTIVE_COUNT"
|
||||
echo " $ICON_EMBY Direct play: $DIRECT_NOW"
|
||||
echo " $ICON_EMBY Transcoding: $TRANSCODE_NOW"
|
||||
|
||||
if [[ "$ACTIVE_COUNT" -gt 0 ]]; then
|
||||
echo ""
|
||||
echo " Now playing:"
|
||||
echo "$SESSIONS" | jq -r '
|
||||
.[] |
|
||||
select(.NowPlayingItem != null) |
|
||||
" \(.UserName // "Unknown") → \(.NowPlayingItem.Name // "Unknown") [\(if .TranscodingInfo != null then "transcode" else "direct" end)]"
|
||||
' 2>/dev/null || true
|
||||
fi
|
||||
log "$ICON_EMBY Sessions: $ACTIVE_COUNT active ($DIRECT_NOW direct / $TRANSCODE_NOW transcode)"
|
||||
echo ""
|
||||
|
||||
# ── Library Stats ─────────────────────────────────────────────────────────────────────────────
|
||||
echo "━━━ $ICON_EMBY Library ━━━"
|
||||
ITEMS=$(emby_api "Items/Counts" 2>/dev/null) || { warn "Could not fetch library counts"; ITEMS="{}"; }
|
||||
|
||||
MOVIE_COUNT=$(echo "$ITEMS" | jq '.MovieCount // 0' 2>/dev/null || echo 0)
|
||||
EPISODE_COUNT=$(echo "$ITEMS" | jq '.EpisodeCount // 0' 2>/dev/null || echo 0)
|
||||
SONG_COUNT=$(echo "$ITEMS" | jq '.SongCount // 0' 2>/dev/null || echo 0)
|
||||
|
||||
echo " $ICON_EMBY Movies: $MOVIE_COUNT"
|
||||
echo " $ICON_EMBY Episodes: $EPISODE_COUNT"
|
||||
echo " $ICON_EMBY Songs: $SONG_COUNT"
|
||||
log "$ICON_EMBY Library: $MOVIE_COUNT movies $EPISODE_COUNT episodes $SONG_COUNT songs"
|
||||
echo ""
|
||||
|
||||
# ── Activity History ──────────────────────────────────────────────────────────────────────────
|
||||
# Query activity log for the configured period
|
||||
echo "━━━ $ICON_EMBY Activity — Last ${EMBY_REPORT_DAYS} Days ━━━"
|
||||
|
||||
REPORT_START=$(date -d "${EMBY_REPORT_DAYS} days ago" '+%Y-%m-%dT00:00:00.000Z')
|
||||
|
||||
ACTIVITY=$(emby_api "System/ActivityLog/Entries?MinDate=${REPORT_START}&Limit=1000" \
|
||||
2>/dev/null) || { warn "Could not fetch activity log"; ACTIVITY="{}"; }
|
||||
|
||||
TOTAL_PLAYS=$(echo "$ACTIVITY" | \
|
||||
jq '[.Items // [] | .[] | select(.Type == "VideoPlayback" or .Type == "AudioPlayback")] | length' \
|
||||
2>/dev/null || echo 0)
|
||||
|
||||
TRANSCODE_PLAYS=$(echo "$ACTIVITY" | \
|
||||
jq '[.Items // [] | .[] | select(.Type == "VideoPlaybackUnplugged" or
|
||||
(.Type == "VideoPlayback" and (.Overview // "" | contains("Transcode"))))] | length' \
|
||||
2>/dev/null || echo 0)
|
||||
|
||||
echo " $ICON_EMBY Total play events: $TOTAL_PLAYS"
|
||||
|
||||
if [[ "$TOTAL_PLAYS" -gt 0 ]]; then
|
||||
TRANSCODE_PCT=$(awk "BEGIN {printf \"%.0f\", ($TRANSCODE_PLAYS / $TOTAL_PLAYS) * 100}")
|
||||
DIRECT_PCT=$(( 100 - TRANSCODE_PCT ))
|
||||
echo " $ICON_EMBY Direct play: ~${DIRECT_PCT}%"
|
||||
echo " $ICON_EMBY Transcoded: ~${TRANSCODE_PCT}%"
|
||||
log "$ICON_EMBY Activity: $TOTAL_PLAYS plays — ~${DIRECT_PCT}% direct / ~${TRANSCODE_PCT}% transcode"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# ── Top Content ───────────────────────────────────────────────────────────────────────────────
|
||||
echo "━━━ $ICON_EMBY Top ${EMBY_REPORT_TOP_N} Content ━━━"
|
||||
|
||||
TOP_ITEMS=$(emby_api "Items?SortBy=DatePlayed&SortOrder=Descending&Limit=${EMBY_REPORT_TOP_N}&Recursive=true&Fields=Overview&IncludeItemTypes=Movie,Episode" \
|
||||
2>/dev/null) || { warn "Could not fetch top content"; TOP_ITEMS="{}"; }
|
||||
|
||||
TOP_COUNT=$(echo "$TOP_ITEMS" | jq '.Items // [] | length' 2>/dev/null || echo 0)
|
||||
if [[ "$TOP_COUNT" -gt 0 ]]; then
|
||||
echo "$TOP_ITEMS" | jq -r '
|
||||
.Items // [] |
|
||||
to_entries[] |
|
||||
" \(.key + 1). \(.value.Name // "Unknown") [\(.value.Type // "")]"
|
||||
' 2>/dev/null || warn "Could not parse top content"
|
||||
else
|
||||
echo " No recent play history found"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# ── Most Active Users ─────────────────────────────────────────────────────────────────────────
|
||||
echo "━━━ $ICON_EMBY Most Active Users ━━━"
|
||||
USERS=$(emby_api "Users" 2>/dev/null) || { warn "Could not fetch users"; USERS="[]"; }
|
||||
|
||||
USER_COUNT=$(echo "$USERS" | jq 'length' 2>/dev/null || echo 0)
|
||||
echo " $ICON_EMBY Total users: $USER_COUNT"
|
||||
|
||||
if [[ "$USER_COUNT" -gt 0 ]]; then
|
||||
echo "$USERS" | jq -r '
|
||||
sort_by(.LastActivityDate // "0") |
|
||||
reverse |
|
||||
.[:5][] |
|
||||
" \(.Name // "Unknown") — last active: \(.LastActivityDate // "never" | split("T")[0])"
|
||||
' 2>/dev/null || true
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# ── Ramdisk / Transcode Status ────────────────────────────────────────────────────────────────
|
||||
echo "━━━ $ICON_RAM Transcode Status ━━━"
|
||||
if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
|
||||
RAMDISK_USED_KB=$(df "$RAMDISK_PATH" --output=used 2>/dev/null | tail -1 | tr -d ' ')
|
||||
RAMDISK_USED_GB=$(awk "BEGIN {printf \"%.2f\", ${RAMDISK_USED_KB:-0} / 1048576}")
|
||||
SYMLINK=$(readlink "$TRANSCODE_LINK" 2>/dev/null || echo "unknown")
|
||||
echo " $ICON_RAM Ramdisk usage: ${RAMDISK_USED_GB}GB / ${RAMDISK_SIZE:-8G}"
|
||||
echo " $ICON_LINK Symlink target: $SYMLINK"
|
||||
if [[ "$SYMLINK" == *"ssd"* ]] || [[ "$SYMLINK" == *"cache"* ]]; then
|
||||
warn "Transcode link pointing at SSD — ramdisk may be full"
|
||||
fi
|
||||
else
|
||||
warn "Ramdisk not mounted at $RAMDISK_PATH"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo "━━━━━ $ICON_SUMMARY EMBY REPORT SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_EMBY Server: $SERVER_NAME (v$SERVER_VERSION)"
|
||||
echo "$ICON_EMBY Active: $ACTIVE_COUNT streams ($DIRECT_NOW direct / $TRANSCODE_NOW transcode)"
|
||||
echo "$ICON_EMBY Library: $MOVIE_COUNT movies $EPISODE_COUNT episodes $SONG_COUNT songs"
|
||||
echo "$ICON_EMBY Period: $TOTAL_PLAYS play events in last ${EMBY_REPORT_DAYS} days"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
# Only notify on issues — high transcode rate may indicate config problem
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
if [[ "$TOTAL_PLAYS" -gt 10 && "${TRANSCODE_PCT:-0}" -gt 80 ]]; then
|
||||
notify "Emby report on $(hostname) — high transcode rate: ${TRANSCODE_PCT}% of $TOTAL_PLAYS plays — check direct play config" \
|
||||
"Emby Report" "warning"
|
||||
fi
|
||||
fi
|
||||
|
||||
exit 0
|
||||
+229
@@ -0,0 +1,229 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ============================= Bulk Permissions Repair ========================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Applies correct ownership and permissions to one or more specific paths.
|
||||
# Targeted repair — faster than media_shares_permissions.sh, which processes
|
||||
# every configured share. Use after failed transfers that left root:root ownership,
|
||||
# containers writing as root before PUID/PGID was fixed, manual file copies, or
|
||||
# new shares that need permissions applied before the next nightly run.
|
||||
#
|
||||
# Counts files with wrong ownership before fixing. A high count on a recently
|
||||
# written share means a container has wrong PUID/PGID — add PUID=99 PGID=100
|
||||
# to its Docker template. Common culprits: SABnzbd, qBittorrent, slskd.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# DESIGN PRINCIPLES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Permissions Model
|
||||
# Directories (PERMISSIONS_DIR_MODE, default 755):
|
||||
# Owner (nobody) — rwx enter, list, create files
|
||||
# Group (users) — r-x enter and list
|
||||
# Others — r-x Samba guests can browse
|
||||
# Files (PERMISSIONS_FILE_MODE, default 664):
|
||||
# Owner (nobody) — rw read + write
|
||||
# Group (users) — rw arrs can import and rename
|
||||
# Others — r Samba guests can read
|
||||
# No execute bit — media files are never executable
|
||||
#
|
||||
# Separate Passes
|
||||
# Directories and files are chmod'd in separate find passes. A combined pass
|
||||
# with mode 664 would wrongly strip the execute bit from directories, making
|
||||
# them untraversable.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Root Required
|
||||
# chown requires root — exits immediately if not running as root.
|
||||
#
|
||||
# Path Existence Check
|
||||
# Each path is verified before processing — missing paths log an error and
|
||||
# are skipped rather than silently passing.
|
||||
#
|
||||
# Notification Validated
|
||||
# platform_require_cmd confirms the notify script is present before use.
|
||||
#
|
||||
# Silent on Success
|
||||
# Only failures and the wrong-owner diagnostic produce visible output.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# master.conf
|
||||
#
|
||||
# PERMISSIONS_OWNER
|
||||
# Owner applied to all paths. (default: nobody:users)
|
||||
#
|
||||
# PERMISSIONS_DIR_MODE
|
||||
# chmod mode for directories. (default: 755)
|
||||
#
|
||||
# PERMISSIONS_FILE_MODE
|
||||
# chmod mode for files. (default: 664)
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# bulk_permissions_repair.sh /path/to/share [/another/path ...]
|
||||
# Apply ownership and permissions to each specified path.
|
||||
#
|
||||
# bulk_permissions_repair.sh /path/to/share --dry-run
|
||||
# Show wrong-owner count per path. No chown or chmod applied.
|
||||
#
|
||||
# bulk_permissions_repair.sh /path/to/share --log
|
||||
# Verbose output including per-path file counts and modes applied.
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root — chown requires root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
acquire_lock
|
||||
|
||||
|
||||
# detect_hosts() sets MY_ID — used in summary
|
||||
detect_hosts
|
||||
|
||||
if [[ ${#PARSED_ARGS[@]} -eq 0 ]]; then
|
||||
error "No paths specified"
|
||||
error "Usage: bulk_permissions_repair.sh /path/to/share [/another/path] [--dry-run]"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log "Dir mode: ${PERMISSIONS_DIR_MODE:-755}"
|
||||
log "File mode: ${PERMISSIONS_FILE_MODE:-664}"
|
||||
log "Owner: $PERMISSIONS_OWNER"
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no permissions will be changed"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Apply Permissions ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_PERMS Permissions Repair — $MY_ID ━━━"
|
||||
|
||||
START=$(date +%s)
|
||||
PASS=()
|
||||
FAIL=()
|
||||
TOTAL_WRONG_OWNER=0
|
||||
|
||||
for share_path in "${PARSED_ARGS[@]}"; do
|
||||
[[ -z "$share_path" ]] && continue
|
||||
|
||||
echo ""
|
||||
echo "━━━ $ICON_PERMS $(basename "$share_path") ━━━"
|
||||
|
||||
if [[ ! -d "$share_path" ]]; then
|
||||
error "$share_path — not found"
|
||||
FAIL+=("$share_path")
|
||||
continue
|
||||
fi
|
||||
|
||||
# Count files for context — warn level so user knows what they're in for on large shares
|
||||
FILE_COUNT=$(find "$share_path" -type f 2>/dev/null | wc -l)
|
||||
DIR_COUNT=$(find "$share_path" -type d 2>/dev/null | wc -l)
|
||||
SIZE=$(du -sh "$share_path" 2>/dev/null | cut -f1)
|
||||
warn "$share_path — $FILE_COUNT files, $DIR_COUNT dirs ($SIZE)"
|
||||
|
||||
# Count files with wrong ownership before fixing — diagnostic
|
||||
WRONG_OWNER=$(find "$share_path" \( ! -user nobody -o ! -group users \) \
|
||||
2>/dev/null | wc -l)
|
||||
if [[ "$WRONG_OWNER" -gt 0 ]]; then
|
||||
warn "$WRONG_OWNER file(s) with wrong ownership — fixing..."
|
||||
if [[ "$WRONG_OWNER" -gt 500 ]]; then
|
||||
warn "High wrong-owner count — check container PUID/PGID settings (should be PUID=99 PGID=100)"
|
||||
warn "Common culprits: SABnzbd, qBittorrent, slskd"
|
||||
fi
|
||||
TOTAL_WRONG_OWNER=$(( TOTAL_WRONG_OWNER + WRONG_OWNER ))
|
||||
else
|
||||
log "Ownership already correct — applying mode only"
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would apply:"
|
||||
warn " chown -R $PERMISSIONS_OWNER $share_path"
|
||||
warn " find -type d → chmod ${PERMISSIONS_DIR_MODE:-755}"
|
||||
warn " find -type f → chmod ${PERMISSIONS_FILE_MODE:-664}"
|
||||
PASS+=("$(basename "$share_path")")
|
||||
continue
|
||||
fi
|
||||
|
||||
CHOWN_OK=true
|
||||
CHMOD_DIR_OK=true
|
||||
CHMOD_FILE_OK=true
|
||||
|
||||
# Apply ownership first
|
||||
log "Applying ownership: $PERMISSIONS_OWNER..."
|
||||
chown -R "$PERMISSIONS_OWNER" "$share_path" 2>/dev/null || CHOWN_OK=false
|
||||
|
||||
# Apply directory permissions — separate pass (dirs need execute bit)
|
||||
log "Applying directory permissions: ${PERMISSIONS_DIR_MODE:-755}..."
|
||||
find "$share_path" -type d \
|
||||
-exec chmod "${PERMISSIONS_DIR_MODE:-755}" {} + 2>/dev/null || CHMOD_DIR_OK=false
|
||||
|
||||
# Apply file permissions — no execute bit on media files
|
||||
log "Applying file permissions: ${PERMISSIONS_FILE_MODE:-664}..."
|
||||
find "$share_path" -type f \
|
||||
-exec chmod "${PERMISSIONS_FILE_MODE:-664}" {} + 2>/dev/null || CHMOD_FILE_OK=false
|
||||
|
||||
if [[ "$CHOWN_OK" == true && "$CHMOD_DIR_OK" == true && "$CHMOD_FILE_OK" == true ]]; then
|
||||
log "$ICON_UNLOCKED $(basename "$share_path") — permissions applied ✅"
|
||||
PASS+=("$(basename "$share_path")")
|
||||
else
|
||||
error "$(basename "$share_path") — repair failed"
|
||||
error " chown: $CHOWN_OK chmod dirs: $CHMOD_DIR_OK chmod files: $CHMOD_FILE_OK"
|
||||
FAIL+=("$(basename "$share_path")")
|
||||
fi
|
||||
done
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY PERMISSIONS REPAIR SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_PERMS Dir mode: ${PERMISSIONS_DIR_MODE:-755}"
|
||||
echo "$ICON_PERMS File mode: ${PERMISSIONS_FILE_MODE:-664}"
|
||||
echo "$ICON_PERMS Owner: $PERMISSIONS_OWNER"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
[[ ${#PASS[@]} -gt 0 ]] && log "Pass: ${PASS[*]}"
|
||||
[[ ${#FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Fail: ${FAIL[*]}"
|
||||
|
||||
if [[ "$TOTAL_WRONG_OWNER" -gt 0 ]]; then
|
||||
warn "Total wrong-owner files fixed: $TOTAL_WRONG_OWNER"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no changes made"
|
||||
elif [[ ${#FAIL[@]} -gt 0 ]]; then
|
||||
echo "$ICON_ERROR Status: SOME REPAIRS FAILED"
|
||||
notify "Permissions repair failed on $(hostname) — ${FAIL[*]}" \
|
||||
"Permissions Repair" "warning"
|
||||
else
|
||||
echo "$ICON_DONE Status: done — ${#PASS[@]} path(s) repaired"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ ${#FAIL[@]} -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
+588
@@ -0,0 +1,588 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ================================= Resource Manager ===========================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Pressure reduction layer — detects rising system load and reduces it before
|
||||
# things break. Called by watchdog_orchestrator.sh every 15 minutes as a
|
||||
# single-pass run. The middle layer between docker_watchdog.sh (fixes broken
|
||||
# containers) and stability_watchdog.sh (reboots). Does neither of those things.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Three-Level Pressure Response
|
||||
#
|
||||
# Level 1 — SOFT (RAM < RW_RAM_SOFT_GB OR load > RW_LOAD_SOFT_MULTIPLIER × cores):
|
||||
# Throttle SABnzbd download speed to RW_SABNZBD_SPEED_SOFT.
|
||||
# Throttle qBittorrent download to RW_QBIT_DL_SOFT KB/s.
|
||||
#
|
||||
# Level 2 — MEDIUM (RAM < RW_RAM_MEDIUM_GB OR load > RW_LOAD_MEDIUM_MULTIPLIER × cores):
|
||||
# Further throttle SABnzbd + qBittorrent to medium limits.
|
||||
# docker pause RW_PAUSE_CONTAINERS — suspend without losing state, instantly reversible.
|
||||
#
|
||||
# Level 3 — HARD (RAM < RW_RAM_HARD_GB):
|
||||
# docker stop RW_STOP_CONTAINERS — optional/heavy services (games, LocalAI, etc.).
|
||||
# Write mem_shutdown_active=true → signals docker_watchdog to defer container restarts.
|
||||
#
|
||||
# Recovery
|
||||
# Pressure must stay below current threshold for RW_RECOVER_CYCLES consecutive
|
||||
# runs before restoring. De-escalates one level at a time — prevents re-triggering
|
||||
# immediately after recovery. Level 3 additionally requires RAM >= RW_RAM_RECOVER_GB
|
||||
# before containers are un-stopped.
|
||||
#
|
||||
# Coordination with docker_watchdog.sh
|
||||
# At level 3, writes mem_shutdown_active=true to RW_STATE_FILE.
|
||||
# docker_watchdog.sh reads this and defers all container restart logic.
|
||||
# Without this, docker_watchdog would immediately restart containers that were
|
||||
# just stopped to free RAM — defeating the purpose of level 3.
|
||||
# Cleared when pressure resolves and containers are restarted.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Root Required
|
||||
# docker pause/stop require root.
|
||||
#
|
||||
# Single Instance Lock
|
||||
# acquire_lock prevents concurrent runs from racing on state file writes.
|
||||
#
|
||||
# RW_CRITICAL_CONTAINERS
|
||||
# Containers listed here are never paused or stopped regardless of pressure level.
|
||||
#
|
||||
# RW_ENABLED Flag
|
||||
# Set RW_ENABLED=false to disable the entire script without removing it from
|
||||
# the orchestrator schedule.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# master.conf
|
||||
# RW_ENABLED, RW_STATE_FILE
|
||||
# RW_RAM_SOFT_GB, RW_RAM_MEDIUM_GB, RW_RAM_HARD_GB, RW_RAM_RECOVER_GB
|
||||
# RW_LOAD_SOFT_MULTIPLIER, RW_LOAD_MEDIUM_MULTIPLIER
|
||||
# RW_RECOVER_CYCLES
|
||||
# RW_SABNZBD_ENABLED, RW_SABNZBD_SPEED_SOFT, RW_SABNZBD_SPEED_MEDIUM
|
||||
# RW_QBIT_ENABLED, RW_QBIT_DL_SOFT, RW_QBIT_DL_MEDIUM
|
||||
# RW_CRITICAL_CONTAINERS — never paused or stopped regardless of pressure
|
||||
#
|
||||
# host*.conf (aliased by detect_hosts())
|
||||
# HOST*_RW_PAUSE_CONTAINERS — docker pause at medium pressure
|
||||
# HOST*_RW_STOP_CONTAINERS — docker stop at hard pressure
|
||||
# HOST*_SABNZBD_URL, HOST*_SABNZBD_API_KEY
|
||||
# HOST*_QBIT_URL, HOST*_QBIT_USERNAME, HOST*_QBIT_PASSWORD
|
||||
#
|
||||
# ==============================================================================================
|
||||
# STATE FILES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# RW_STATE_FILE
|
||||
# Pressure level, recovery cycle count, stopped container list, and the
|
||||
# mem_shutdown_active coordination flag read by docker_watchdog.sh.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# resource_watchdog.sh
|
||||
# Single-pass pressure check. Apply actions if threshold crossed. Silent if below.
|
||||
#
|
||||
# resource_watchdog.sh --dry-run
|
||||
# Show current pressure level and what would be throttled/paused/stopped. No changes.
|
||||
#
|
||||
# resource_watchdog.sh --status
|
||||
# Show current pressure level, active actions, recovery cycle count, stopped containers.
|
||||
#
|
||||
# resource_watchdog.sh --log
|
||||
# Verbose per-check output — show RAM, load, each threshold comparison, each action.
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ "${RW_ENABLED:-true}" != "true" ]]; then
|
||||
echo "Resource Manager disabled (RW_ENABLED=false)"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
acquire_lock
|
||||
|
||||
if ! command -v docker &>/dev/null; then
|
||||
error "Docker command not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
detect_hosts
|
||||
|
||||
DOCKER_TIMEOUT=15
|
||||
|
||||
log "$ICON_GEAR Config: soft=RAM<${RW_RAM_SOFT_GB}GB/load≥${RW_LOAD_SOFT_THRESH} medium=RAM<${RW_RAM_MEDIUM_GB}GB/load≥${RW_LOAD_MEDIUM_THRESH} hard=RAM<${RW_RAM_HARD_GB}GB recover=RAM≥${RW_RAM_RECOVER_GB}GB cycles=${RW_RECOVER_CYCLES}"
|
||||
log "$ICON_CONTAINERS Pause at medium: ${RW_PAUSE_CONTAINERS[*]:-none} Stop at hard: ${RW_STOP_CONTAINERS[*]:-none}"
|
||||
|
||||
touch "$RW_STATE_FILE" 2>/dev/null || {
|
||||
error "Cannot create state file: $RW_STATE_FILE"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# ── Exit Trap — restart containers stopped this run if script crashes ──────────────────────────
|
||||
declare -a _RW_TRAP_STOPPED=()
|
||||
|
||||
_rw_trap_restart_stopped() {
|
||||
[[ ${#_RW_TRAP_STOPPED[@]} -eq 0 ]] && return
|
||||
for c in "${_RW_TRAP_STOPPED[@]}"; do
|
||||
[[ -z "$c" ]] && continue
|
||||
if docker inspect "$c" >/dev/null 2>&1; then
|
||||
warn "Exit trap: restarting $c (stopped but state not persisted)"
|
||||
docker start "$c" >/dev/null 2>&1 || warn " Failed to restart $c"
|
||||
fi
|
||||
done
|
||||
}
|
||||
trap "_release_all_locks; _rw_trap_restart_stopped" EXIT
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ State Helpers ━━━
|
||||
# ==============================================================================================
|
||||
# rm_state_get/set use : separator for RM-internal state
|
||||
# rm_state_get_eq/set_eq use = separator for docker_watchdog coordination flags
|
||||
|
||||
rm_state_get() {
|
||||
grep -E "^${1}:" "$RW_STATE_FILE" 2>/dev/null | cut -d: -f2-
|
||||
}
|
||||
|
||||
rm_state_set() {
|
||||
local key="$1" val="$2"
|
||||
grep -vE "^${key}:" "$RW_STATE_FILE" 2>/dev/null > "${RW_STATE_FILE}.tmp"
|
||||
echo "${key}:${val}" >> "${RW_STATE_FILE}.tmp"
|
||||
mv "${RW_STATE_FILE}.tmp" "$RW_STATE_FILE"
|
||||
}
|
||||
|
||||
rm_state_get_eq() {
|
||||
grep -E "^${1}=" "$RW_STATE_FILE" 2>/dev/null | cut -d= -f2-
|
||||
}
|
||||
|
||||
rm_state_set_eq() {
|
||||
local key="$1" val="$2"
|
||||
grep -vE "^${key}=" "$RW_STATE_FILE" 2>/dev/null > "${RW_STATE_FILE}.tmp"
|
||||
echo "${key}=${val}" >> "${RW_STATE_FILE}.tmp"
|
||||
mv "${RW_STATE_FILE}.tmp" "$RW_STATE_FILE"
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Load State ━━━
|
||||
# ==============================================================================================
|
||||
CURRENT_LEVEL=$(rm_state_get "rm_action_level"); CURRENT_LEVEL=${CURRENT_LEVEL:-0}
|
||||
RECOVER_CYCLES=$(rm_state_get "rm_recover_cycles"); RECOVER_CYCLES=${RECOVER_CYCLES:-0}
|
||||
PAUSED_LIST=$(rm_state_get "rm_paused_containers"); PAUSED_LIST=${PAUSED_LIST:-""}
|
||||
STOPPED_LIST=$(rm_state_get "rm_stopped_containers"); STOPPED_LIST=${STOPPED_LIST:-""}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Pressure Calculation ━━━
|
||||
# ==============================================================================================
|
||||
TOTAL_CORES=$(nproc)
|
||||
MEM_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo)
|
||||
MEM_GB=$(( MEM_KB / 1024 / 1024 ))
|
||||
LOAD=$(awk '{print $1}' /proc/loadavg)
|
||||
LOAD_INT=$(printf "%.0f" "$LOAD")
|
||||
RW_LOAD_SOFT_THRESH=$(awk "BEGIN{printf \"%.0f\", $TOTAL_CORES * ${RW_LOAD_SOFT_MULTIPLIER:-2.0}}")
|
||||
RW_LOAD_MEDIUM_THRESH=$(awk "BEGIN{printf \"%.0f\", $TOTAL_CORES * ${RW_LOAD_MEDIUM_MULTIPLIER:-3.0}}")
|
||||
|
||||
TARGET_LEVEL=0
|
||||
TARGET_REASON=""
|
||||
if [[ "$MEM_GB" -lt "${RW_RAM_HARD_GB:-6}" ]]; then
|
||||
TARGET_LEVEL=3
|
||||
TARGET_REASON="RAM ${MEM_GB}GB < hard threshold ${RW_RAM_HARD_GB}GB"
|
||||
elif [[ "$MEM_GB" -lt "${RW_RAM_MEDIUM_GB:-8}" ]] || [[ "$LOAD_INT" -ge "$RW_LOAD_MEDIUM_THRESH" ]]; then
|
||||
TARGET_LEVEL=2
|
||||
[[ "$MEM_GB" -lt "${RW_RAM_MEDIUM_GB:-8}" ]] && TARGET_REASON="RAM ${MEM_GB}GB < medium threshold ${RW_RAM_MEDIUM_GB}GB"
|
||||
[[ "$LOAD_INT" -ge "$RW_LOAD_MEDIUM_THRESH" ]] && TARGET_REASON="${TARGET_REASON:+$TARGET_REASON, }load ${LOAD} >= medium threshold ${RW_LOAD_MEDIUM_THRESH}"
|
||||
elif [[ "$MEM_GB" -lt "${RW_RAM_SOFT_GB:-12}" ]] || [[ "$LOAD_INT" -ge "$RW_LOAD_SOFT_THRESH" ]]; then
|
||||
TARGET_LEVEL=1
|
||||
[[ "$MEM_GB" -lt "${RW_RAM_SOFT_GB:-12}" ]] && TARGET_REASON="RAM ${MEM_GB}GB < soft threshold ${RW_RAM_SOFT_GB}GB"
|
||||
[[ "$LOAD_INT" -ge "$RW_LOAD_SOFT_THRESH" ]] && TARGET_REASON="${TARGET_REASON:+$TARGET_REASON, }load ${LOAD} >= soft threshold ${RW_LOAD_SOFT_THRESH}"
|
||||
fi
|
||||
|
||||
LEVEL_NAMES=("normal" "soft" "medium" "hard")
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY RESOURCE MANAGER STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo ""
|
||||
echo "── Current State ──"
|
||||
echo " Action level: $CURRENT_LEVEL (${LEVEL_NAMES[$CURRENT_LEVEL]:-unknown})"
|
||||
echo " Target level: $TARGET_LEVEL (${LEVEL_NAMES[$TARGET_LEVEL]:-unknown})"
|
||||
echo " Recover cycles: $RECOVER_CYCLES / ${RW_RECOVER_CYCLES:-3}"
|
||||
[[ -n "$PAUSED_LIST" ]] && echo " Paused: $PAUSED_LIST"
|
||||
[[ -n "$STOPPED_LIST" ]] && echo " Stopped: $STOPPED_LIST"
|
||||
MEM_SHUTDOWN_ACTIVE=$(rm_state_get_eq "mem_shutdown_active")
|
||||
[[ "$MEM_SHUTDOWN_ACTIVE" == "true" ]] && warn " docker_watchdog DEFERRED (mem_shutdown_active=true)"
|
||||
echo ""
|
||||
echo "── System Pressure ──"
|
||||
echo " RAM free: ${MEM_GB}GB (soft:<${RW_RAM_SOFT_GB} medium:<${RW_RAM_MEDIUM_GB} hard:<${RW_RAM_HARD_GB} recover:>=${RW_RAM_RECOVER_GB})"
|
||||
echo " Load avg: ${LOAD} (soft:>=${RW_LOAD_SOFT_THRESH} medium:>=${RW_LOAD_MEDIUM_THRESH} cores:${TOTAL_CORES})"
|
||||
echo ""
|
||||
echo "── Configuration ──"
|
||||
echo " SABnzbd throttle: ${RW_SABNZBD_ENABLED:-true} soft=${RW_SABNZBD_SPEED_SOFT} medium=${RW_SABNZBD_SPEED_MEDIUM}"
|
||||
echo " qBit throttle: ${RW_QBIT_ENABLED:-true} soft=${RW_QBIT_DL_SOFT}KB/s medium=${RW_QBIT_DL_MEDIUM}KB/s"
|
||||
echo ""
|
||||
echo "── Container Lists (this host) ──"
|
||||
echo " Pause at medium: ${RW_PAUSE_CONTAINERS[*]:-none configured}"
|
||||
echo " Stop at hard: ${RW_STOP_CONTAINERS[*]:-none configured}"
|
||||
echo " Critical (never touched): ${RW_CRITICAL_CONTAINERS[*]:-none}"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Critical Container Guard ━━━
|
||||
# ==============================================================================================
|
||||
# Returns 0 if container is safe to pause/stop, 1 if it is critical
|
||||
is_critical() {
|
||||
local container="$1"
|
||||
for c in "${RW_CRITICAL_CONTAINERS[@]:-}"; do
|
||||
[[ "$c" == "$container" ]] && return 1
|
||||
done
|
||||
return 0
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ SABnzbd API ━━━
|
||||
# ==============================================================================================
|
||||
sabnzbd_set_speed() {
|
||||
local speed="$1"
|
||||
[[ "${RW_SABNZBD_ENABLED:-true}" != "true" ]] && return 0
|
||||
[[ -z "$SABNZBD_URL" || -z "$SABNZBD_API_KEY" ]] && return 0
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would set SABnzbd speed to $speed"
|
||||
return 0
|
||||
fi
|
||||
|
||||
curl -sf --max-time 10 \
|
||||
"${SABNZBD_URL}/api?mode=config&name=speedlimit&value=${speed}&apikey=${SABNZBD_API_KEY}" \
|
||||
>/dev/null 2>&1 && log "SABnzbd speed → $speed" || warn "SABnzbd API call failed"
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ qBittorrent API ━━━
|
||||
# ==============================================================================================
|
||||
QBIT_COOKIE="/tmp/rm_qbit_cookie.txt"
|
||||
|
||||
qbit_login() {
|
||||
[[ "${RW_QBIT_ENABLED:-true}" != "true" ]] && return 0
|
||||
[[ -z "$QBIT_URL" || -z "$QBIT_USERNAME" || -z "$QBIT_PASSWORD" ]] && return 0
|
||||
|
||||
curl -sf --max-time 10 -c "$QBIT_COOKIE" \
|
||||
-X POST "${QBIT_URL}/api/v2/auth/login" \
|
||||
-d "username=${QBIT_USERNAME}&password=${QBIT_PASSWORD}" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
qbit_set_dl_limit() {
|
||||
local kbps="$1" # KB/s — 0 = unlimited
|
||||
[[ "${RW_QBIT_ENABLED:-true}" != "true" ]] && return 0
|
||||
[[ -z "$QBIT_URL" ]] && return 0
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would set qBit download limit to ${kbps}KB/s"
|
||||
return 0
|
||||
fi
|
||||
|
||||
local bps=$(( kbps * 1024 ))
|
||||
qbit_login
|
||||
curl -sf --max-time 10 -b "$QBIT_COOKIE" \
|
||||
-X POST "${QBIT_URL}/api/v2/transfer/setDownloadLimit" \
|
||||
-d "limit=${bps}" >/dev/null 2>&1 && log "qBit download limit → ${kbps}KB/s" || warn "qBit API call failed"
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Container Actions ━━━
|
||||
# ==============================================================================================
|
||||
|
||||
# Pause a list of containers — returns newline-separated list of actually-paused containers
|
||||
pause_containers() {
|
||||
local actually_paused=()
|
||||
for container in "$@"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
is_critical "$container" || { log "$container — critical, skipping pause"; continue; }
|
||||
|
||||
local status
|
||||
status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Status}}' "$container" 2>/dev/null)
|
||||
if [[ "$status" != "running" ]]; then
|
||||
log "$container — not running (status: ${status:-unknown}), skipping pause"
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would docker pause $container"
|
||||
actually_paused+=("$container")
|
||||
continue
|
||||
fi
|
||||
|
||||
if timeout "$DOCKER_TIMEOUT" docker pause "$container" >/dev/null 2>&1; then
|
||||
warn "Paused $container (medium pressure)"
|
||||
actually_paused+=("$container")
|
||||
else
|
||||
error "Failed to pause $container"
|
||||
fi
|
||||
done
|
||||
printf '%s,' "${actually_paused[@]}" | sed 's/,$//'
|
||||
}
|
||||
|
||||
# Unpause a comma-separated list of containers
|
||||
unpause_containers() {
|
||||
local IFS=','
|
||||
for container in $1; do
|
||||
[[ -z "$container" ]] && continue
|
||||
|
||||
local status
|
||||
status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Status}}' "$container" 2>/dev/null)
|
||||
if [[ "$status" != "paused" ]]; then
|
||||
log "$container — not paused (status: ${status:-unknown}), skipping unpause"
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would docker unpause $container"
|
||||
continue
|
||||
fi
|
||||
|
||||
if timeout "$DOCKER_TIMEOUT" docker unpause "$container" >/dev/null 2>&1; then
|
||||
warn "Unpaused $container (pressure reduced)"
|
||||
else
|
||||
error "Failed to unpause $container"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
# Stop a list of containers — returns comma-separated list of actually-stopped containers
|
||||
stop_containers() {
|
||||
local actually_stopped=()
|
||||
for container in "$@"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
is_critical "$container" || { log "$container — critical, skipping stop"; continue; }
|
||||
|
||||
local status
|
||||
status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Status}}' "$container" 2>/dev/null)
|
||||
if [[ "$status" != "running" && "$status" != "paused" ]]; then
|
||||
log "$container — not running (status: ${status:-unknown}), skipping stop"
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would docker stop $container"
|
||||
actually_stopped+=("$container")
|
||||
continue
|
||||
fi
|
||||
|
||||
if timeout "$DOCKER_TIMEOUT" docker stop "$container" >/dev/null 2>&1; then
|
||||
warn "Stopped $container (hard pressure)"
|
||||
actually_stopped+=("$container")
|
||||
_RW_TRAP_STOPPED+=("$container")
|
||||
else
|
||||
error "Failed to stop $container"
|
||||
fi
|
||||
done
|
||||
printf '%s,' "${actually_stopped[@]}" | sed 's/,$//'
|
||||
}
|
||||
|
||||
# Start a comma-separated list of containers (only those RM stopped)
|
||||
start_containers() {
|
||||
local IFS=','
|
||||
for container in $1; do
|
||||
[[ -z "$container" ]] && continue
|
||||
|
||||
local status
|
||||
status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Status}}' "$container" 2>/dev/null)
|
||||
if [[ "$status" == "running" ]]; then
|
||||
log "$container — already running"
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would docker start $container"
|
||||
continue
|
||||
fi
|
||||
|
||||
if timeout "$DOCKER_TIMEOUT" docker start "$container" >/dev/null 2>&1; then
|
||||
warn "Started $container (pressure cleared)"
|
||||
else
|
||||
error "Failed to start $container"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Apply Level Actions ━━━
|
||||
# ==============================================================================================
|
||||
|
||||
apply_level_1() {
|
||||
log "Applying level 1 (soft) — throttling downloaders"
|
||||
sabnzbd_set_speed "${RW_SABNZBD_SPEED_SOFT:-50M}"
|
||||
qbit_set_dl_limit "${RW_QBIT_DL_SOFT:-51200}"
|
||||
}
|
||||
|
||||
apply_level_2() {
|
||||
log "Applying level 2 (medium) — throttling + pausing background containers"
|
||||
sabnzbd_set_speed "${RW_SABNZBD_SPEED_MEDIUM:-10M}"
|
||||
qbit_set_dl_limit "${RW_QBIT_DL_MEDIUM:-10240}"
|
||||
|
||||
if [[ ${#RW_PAUSE_CONTAINERS[@]} -gt 0 ]]; then
|
||||
local newly_paused
|
||||
newly_paused=$(pause_containers "${RW_PAUSE_CONTAINERS[@]}")
|
||||
# Merge with existing paused list (avoid duplicates on re-escalation)
|
||||
if [[ -n "$newly_paused" ]]; then
|
||||
if [[ -n "$PAUSED_LIST" ]]; then
|
||||
PAUSED_LIST="${PAUSED_LIST},${newly_paused}"
|
||||
else
|
||||
PAUSED_LIST="$newly_paused"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
apply_level_3() {
|
||||
log "Applying level 3 (hard) — stopping optional containers"
|
||||
sabnzbd_set_speed "${RW_SABNZBD_SPEED_MEDIUM:-10M}" # already at medium from level 2
|
||||
qbit_set_dl_limit "${RW_QBIT_DL_MEDIUM:-10240}"
|
||||
|
||||
if [[ ${#RW_STOP_CONTAINERS[@]} -gt 0 ]]; then
|
||||
local newly_stopped
|
||||
newly_stopped=$(stop_containers "${RW_STOP_CONTAINERS[@]}")
|
||||
if [[ -n "$newly_stopped" ]]; then
|
||||
if [[ -n "$STOPPED_LIST" ]]; then
|
||||
STOPPED_LIST="${STOPPED_LIST},${newly_stopped}"
|
||||
else
|
||||
STOPPED_LIST="$newly_stopped"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# Signal docker_watchdog to defer container restarts
|
||||
rm_state_set_eq "mem_shutdown_active" "true"
|
||||
warn "mem_shutdown_active=true — docker_watchdog will defer restarts"
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Restore Level Actions ━━━
|
||||
# ==============================================================================================
|
||||
|
||||
restore_level_3() {
|
||||
echo "Restoring from level 3 — starting stopped containers"
|
||||
if [[ -n "$STOPPED_LIST" ]]; then
|
||||
start_containers "$STOPPED_LIST"
|
||||
STOPPED_LIST=""
|
||||
fi
|
||||
rm_state_set_eq "mem_shutdown_active" "false"
|
||||
warn "mem_shutdown_active=false — docker_watchdog restoring normal operation"
|
||||
}
|
||||
|
||||
restore_level_2() {
|
||||
echo "Restoring from level 2 — unpausing containers"
|
||||
if [[ -n "$PAUSED_LIST" ]]; then
|
||||
unpause_containers "$PAUSED_LIST"
|
||||
PAUSED_LIST=""
|
||||
fi
|
||||
}
|
||||
|
||||
restore_level_1() {
|
||||
echo "Restoring from level 1 — removing downloader throttle"
|
||||
sabnzbd_set_speed "0"
|
||||
qbit_set_dl_limit 0
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Pressure Decision ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Resource Manager — $MY_ID — $(date '+%H:%M:%S') ━━━"
|
||||
echo " RAM: ${MEM_GB}GB free Load: ${LOAD} Action: ${CURRENT_LEVEL} → ${TARGET_LEVEL} (${LEVEL_NAMES[$TARGET_LEVEL]:-unknown})"
|
||||
|
||||
if [[ "$TARGET_LEVEL" -gt "$CURRENT_LEVEL" ]]; then
|
||||
# ── Escalate ────────────────────────────────────────────────────────────────────────────
|
||||
warn "Pressure escalating to level $TARGET_LEVEL — $TARGET_REASON"
|
||||
notify "Resource Manager: pressure level $TARGET_LEVEL on $(hostname) ($MY_ID) — $TARGET_REASON" \
|
||||
"Resource Manager" "warning"
|
||||
|
||||
for (( lvl = CURRENT_LEVEL + 1; lvl <= TARGET_LEVEL; lvl++ )); do
|
||||
case "$lvl" in
|
||||
1) apply_level_1 ;;
|
||||
2) apply_level_2 ;;
|
||||
3) apply_level_3 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
rm_state_set "rm_action_level" "$TARGET_LEVEL"
|
||||
rm_state_set "rm_recover_cycles" 0
|
||||
|
||||
elif [[ "$TARGET_LEVEL" -lt "$CURRENT_LEVEL" ]]; then
|
||||
# ── Tracking recovery ───────────────────────────────────────────────────────────────────
|
||||
RECOVER_CYCLES=$(( RECOVER_CYCLES + 1 ))
|
||||
rm_state_set "rm_recover_cycles" "$RECOVER_CYCLES"
|
||||
log "Pressure at level $TARGET_LEVEL — recovery cycle $RECOVER_CYCLES/${RW_RECOVER_CYCLES:-3} before restoring level $CURRENT_LEVEL actions"
|
||||
|
||||
if [[ "$RECOVER_CYCLES" -ge "${RW_RECOVER_CYCLES:-3}" ]]; then
|
||||
# Level 3 de-escalation requires RAM above recover threshold
|
||||
if [[ "$CURRENT_LEVEL" -ge 3 && "$MEM_GB" -lt "${RW_RAM_RECOVER_GB:-20}" ]]; then
|
||||
warn "Level 3 restore blocked — RAM ${MEM_GB}GB still below recover threshold ${RW_RAM_RECOVER_GB}GB"
|
||||
else
|
||||
warn "Pressure sustained below level $CURRENT_LEVEL — restoring"
|
||||
case "$CURRENT_LEVEL" in
|
||||
3) restore_level_3 ;;
|
||||
2) restore_level_2 ;;
|
||||
1) restore_level_1 ;;
|
||||
esac
|
||||
|
||||
NEW_LEVEL=$(( CURRENT_LEVEL - 1 ))
|
||||
rm_state_set "rm_action_level" "$NEW_LEVEL"
|
||||
rm_state_set "rm_recover_cycles" 0
|
||||
rm_state_set "rm_paused_containers" "$PAUSED_LIST"
|
||||
rm_state_set "rm_stopped_containers" "$STOPPED_LIST"
|
||||
|
||||
if [[ "$NEW_LEVEL" -gt 0 ]]; then
|
||||
warn "De-escalated to level $NEW_LEVEL (${LEVEL_NAMES[$NEW_LEVEL]}) — ${RW_RECOVER_CYCLES:-3} more cycles to fully clear"
|
||||
else
|
||||
log "All pressure cleared — system at normal operation ✅"
|
||||
notify "Resource Manager: pressure resolved on $(hostname) ($MY_ID) — system back to normal" \
|
||||
"Resource Manager" "normal"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
else
|
||||
# ── Steady state ────────────────────────────────────────────────────────────────────────
|
||||
if [[ "$CURRENT_LEVEL" -gt 0 ]]; then
|
||||
echo "Pressure holding at level $CURRENT_LEVEL — waiting for sustained recovery"
|
||||
else
|
||||
log "System normal — RAM ${MEM_GB}GB free | load ${LOAD} | ${TOTAL_CORES} cores | SABnzbd=${RW_SABNZBD_ENABLED:-true} qBit=${RW_QBIT_ENABLED:-true} ✅"
|
||||
fi
|
||||
rm_state_set "rm_recover_cycles" 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Persist State ━━━
|
||||
# ==============================================================================================
|
||||
rm_state_set "rm_paused_containers" "$PAUSED_LIST"
|
||||
rm_state_set "rm_stopped_containers" "$STOPPED_LIST"
|
||||
trap "_release_all_locks" EXIT # state persisted — disable restart trap, keep lock cleanup
|
||||
# Touch state file each run so docker_watchdog stale guard sees fresh mtime
|
||||
touch "$RW_STATE_FILE" 2>/dev/null
|
||||
+457
@@ -0,0 +1,457 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ============================= Rsync Core Script ==============================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Core rsync engine for the two-server ecosystem. Called per share or per
|
||||
# appdata profile by orchestrators (daily_sync_maintenance, weekly_sync_maintenance,
|
||||
# critical_sync_maintenance) and directly for manual or scheduled dirty syncs.
|
||||
#
|
||||
# Profile is inferred from the directory basename (lowercased). Override with
|
||||
# --profile=name for explicit selection. If no profile matches, global defaults
|
||||
# from master.conf apply and no containers are stopped.
|
||||
#
|
||||
# After each sync, logs transfer data to bandwidth_monitor.sh for the weekly
|
||||
# bandwidth report. Silent on success — only failures produce visible output.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Profiles define per-share behavior:
|
||||
# PROFILE_CRITICAL_CONTAINER_NAMES — containers stopped on both servers before sync
|
||||
# PROFILE_DELAYED_CONTAINERS — containers with a delay before restart after sync
|
||||
# PROFILE_CONTAINER_DELAY — seconds before delayed containers start
|
||||
# PROFILE_RSYNC_OPTS — rsync flags (does not inherit DEFAULT_RSYNC_OPTS)
|
||||
# PROFILE_BW_LIMIT — bandwidth limit in KB/s
|
||||
# PROFILE_RETRY_COUNT — retry attempts on failure
|
||||
# PROFILE_SLEEP — seconds between retry attempts
|
||||
# PROFILE_EXCLUDE_DIRS — paths excluded from transfer
|
||||
# PROFILE_REMOTE_RESTART_CONTAINERS — containers restarted on remote after dirty sync
|
||||
# Was running → restart. Was stopped → leave stopped.
|
||||
#
|
||||
# Two-tier rsync enable/disable:
|
||||
# Tier 1: RSYNC_ENABLED=false → all rsync stops immediately (checked by this script)
|
||||
# Tier 2: per-orchestrator flag (DAILY_RSYNC_ENABLED etc.) → checked by caller
|
||||
#
|
||||
# Bandwidth logging: after each sync, logs profile/duration/status/bytes to
|
||||
# bandwidth_monitor.sh --log-transfer. Bytes captured from rsync --stats via awk
|
||||
# using version-stable field names.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Global Rsync Gate
|
||||
# check_rsync_enabled() — RSYNC_ENABLED=false exits cleanly before any operation.
|
||||
#
|
||||
# Partnership Blocklist
|
||||
# Refuses to sync if REMOTE_SERVER_NAME appears in the partnership blocklist.
|
||||
# Written at offboard — prevents stale access after a partnership ends.
|
||||
#
|
||||
# Version Parity
|
||||
# check_os_version_parity — refuses sync if servers on incompatible unRAID versions.
|
||||
#
|
||||
# Remote Health Pre-flights
|
||||
# check_connectivity() — Tailscale IP reachable before any SSH
|
||||
# check_remote_rootfs() — aborts if remote rootfs exceeds ROOTFS_WARN_PCT
|
||||
# check_remote_share() — aborts if target directory missing or empty on remote
|
||||
# check_remote_disks() — verifies all backing disks online on remote
|
||||
#
|
||||
# Drive Temperature Check
|
||||
# check_local_disk_temps() — runs before any transfer. Exit 1 = skip this profile,
|
||||
# exit 2 = abort all remaining syncs (CRITICAL temperature).
|
||||
#
|
||||
# Remote Docker Daemon Check
|
||||
# check_remote_docker_daemon — verified before any container stop/start operations.
|
||||
# If daemon unresponsive: container operations skipped, rsync proceeds without stopping.
|
||||
#
|
||||
# Per-Profile Concurrency Lock
|
||||
# acquire_rsync_lock() — per-profile lock prevents parallel runs of the same profile.
|
||||
# Global concurrent limit prevents too many simultaneous rsync processes.
|
||||
#
|
||||
# Notification Validated
|
||||
# platform_require_cmd confirms the notify script is present before use.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# master.conf
|
||||
#
|
||||
# RSYNC_ENABLED
|
||||
# Global on/off toggle for all rsync operations. (default: true)
|
||||
#
|
||||
# DEFAULT_RSYNC_OPTS
|
||||
# Base rsync flags for unproiled shares. Note: --delete is intentionally absent —
|
||||
# media shares spread files only, arr cleanup scripts own deletions. Profile-specific
|
||||
# opts set --delete explicitly where needed.
|
||||
#
|
||||
# BW_LIMIT
|
||||
# Default bandwidth cap in KB/s when no PROFILE_BW_LIMIT is set. (default: 0 = unlimited)
|
||||
#
|
||||
# RETRY_COUNT
|
||||
# Default retry attempts on rsync failure. (default: 3)
|
||||
#
|
||||
# SLEEP
|
||||
# Default seconds between retry attempts. (default: 60)
|
||||
#
|
||||
# ROOTFS_WARN_PCT
|
||||
# Abort threshold for remote rootfs percentage full. (default: 75)
|
||||
#
|
||||
# PROFILES["profile_KEY"]
|
||||
# Profile definitions — one entry per PROFILE_* key per profile name.
|
||||
# See OPERATIONAL MODEL above for all supported keys.
|
||||
#
|
||||
# BANDWIDTH_LOG / BANDWIDTH_WARN_GB
|
||||
# Shared with bandwidth_monitor.sh — set once, used by both.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# rsync.sh /path/to/share
|
||||
# Sync the given path to the remote. Profile inferred from directory basename.
|
||||
#
|
||||
# rsync.sh /path/to/share --profile=name
|
||||
# Sync with explicit profile override — bypasses basename inference.
|
||||
#
|
||||
# rsync.sh /path/to/share --dry-run
|
||||
# Run all pre-flight checks and show what rsync would transfer. No transfer,
|
||||
# no container stops.
|
||||
#
|
||||
# rsync.sh /path/to/share --status
|
||||
# Show resolved profile, remote identity, and configuration. Then exit.
|
||||
#
|
||||
# rsync.sh /path/to/share --log
|
||||
# Verbose output throughout — every decision logged.
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
# ── Separate positional directory arg from flags ───────────────────────────────────────────────
|
||||
DIRECTORY=""
|
||||
PROFILE_OVERRIDE=""
|
||||
RAW_ARGS=()
|
||||
|
||||
for ARG in "$@"; do
|
||||
case "$ARG" in
|
||||
--profile=*) PROFILE_OVERRIDE="${ARG#--profile=}" ;;
|
||||
--*|*=*) RAW_ARGS+=("$ARG") ;;
|
||||
*) [[ -z "$DIRECTORY" ]] && DIRECTORY="$ARG" || RAW_ARGS+=("$ARG") ;;
|
||||
esac
|
||||
done
|
||||
|
||||
parse_args "${RAW_ARGS[@]}"
|
||||
|
||||
[[ -z "$DIRECTORY" ]] && {
|
||||
error "No directory specified"
|
||||
error "Usage: rsync.sh <dir> [--dry-run] [--log] [--profile=name]"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
acquire_lock
|
||||
|
||||
if ! command -v docker &>/dev/null; then
|
||||
error "Docker command not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
detect_hosts
|
||||
|
||||
# Tier 1 global gate — Tier 2 (per-orchestrator) checked by caller
|
||||
if ! check_rsync_enabled; then
|
||||
warn "RSYNC_ENABLED=false — exiting cleanly"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Blocklist gate — refuse to sync with a partner blocked after offboard
|
||||
BLOCKLIST_FILE="${PARTNERSHIP_BLOCKLIST_FILE:-${STATE_DIR}/partnership_blocklist.db}"
|
||||
if [[ -f "$BLOCKLIST_FILE" ]] && grep -q "^${REMOTE_SERVER_NAME}|" "$BLOCKLIST_FILE" 2>/dev/null; then
|
||||
error "Rsync blocked — $REMOTE_SERVER_NAME is on the partnership blocklist"
|
||||
error "Re-onboard the partnership to restore access: partnership_manager.sh --onboard"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
resolve_remote_ip
|
||||
|
||||
# ── Profile inference ─────────────────────────────────────────────────────────────────────────
|
||||
if [[ -n "$PROFILE_OVERRIDE" ]]; then
|
||||
PROFILE_NAME="$PROFILE_OVERRIDE"
|
||||
log "Profile override: $PROFILE_NAME"
|
||||
else
|
||||
PROFILE_NAME=$(basename "$DIRECTORY" | tr '[:upper:]' '[:lower:]')
|
||||
log "Profile inferred: $PROFILE_NAME"
|
||||
fi
|
||||
|
||||
# Acquire per-profile lock and check global concurrent limit
|
||||
acquire_rsync_lock "$PROFILE_NAME"
|
||||
|
||||
# Tee all output to a live log file for the Varaverk UI
|
||||
VV_LIVE_LOG="/tmp/unraid_locks/rsync_${PROFILE_NAME}.log"
|
||||
VV_LAST_LOG="/tmp/unraid_locks/rsync_${PROFILE_NAME}.last.log"
|
||||
: > "$VV_LIVE_LOG"
|
||||
exec 1> >(tee -a "$VV_LIVE_LOG") 2>&1
|
||||
|
||||
# ── Load profile settings ─────────────────────────────────────────────────────────────────────
|
||||
BW_LIMIT=${PROFILE_BW_LIMIT[$PROFILE_NAME]:-$BW_LIMIT}
|
||||
RETRY_COUNT=${PROFILE_RETRY_COUNT[$PROFILE_NAME]:-$RETRY_COUNT}
|
||||
SLEEP=${PROFILE_SLEEP[$PROFILE_NAME]:-$SLEEP}
|
||||
CONTAINER_DELAY=${PROFILE_CONTAINER_DELAY[$PROFILE_NAME]:-$CONTAINER_DELAY}
|
||||
|
||||
read -r -a CRITICAL_CONTAINER_NAMES <<< "${PROFILE_CRITICAL_CONTAINER_NAMES[$PROFILE_NAME]:-}"
|
||||
read -r -a DELAYED_CONTAINERS <<< "${PROFILE_DELAYED_CONTAINERS[$PROFILE_NAME]:-}"
|
||||
read -r -a EXCLUDE_DIRS <<< "${PROFILE_EXCLUDE_DIRS[$PROFILE_NAME]:-}"
|
||||
read -r -a REMOTE_RESTART_CONTAINERS <<< "${PROFILE_REMOTE_RESTART_CONTAINERS[$PROFILE_NAME]:-}"
|
||||
|
||||
# Local containers use same names as remote (mirrored naming scheme)
|
||||
LOCAL_CRITICAL_CONTAINER_NAMES=("${CRITICAL_CONTAINER_NAMES[@]}")
|
||||
|
||||
log "$ICON_GEAR Config: profile=${PROFILE_NAME} bw-limit=${BW_LIMIT}KB/s retry=${RETRY_COUNT} sleep=${SLEEP}s container-delay=${CONTAINER_DELAY}s"
|
||||
log "$ICON_GEAR Containers: critical=${CRITICAL_CONTAINER_NAMES[*]:-none} delayed=${DELAYED_CONTAINERS[*]:-none} remote-restart=${REMOTE_RESTART_CONTAINERS[*]:-none}"
|
||||
|
||||
[[ "$SHOW_STATUS" == true ]] && show_status && exit 0
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Pre-flight Checks ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_SHIELD Pre-flight ━━━"
|
||||
|
||||
# Disk temp — before touching remote or moving data
|
||||
# Exit 1 = skip this profile | Exit 2 = abort all remaining profiles
|
||||
check_local_disk_temps
|
||||
TEMP_RESULT=$?
|
||||
if [[ "$TEMP_RESULT" -eq 2 ]]; then
|
||||
error "Drive temps CRITICAL — aborting all remaining syncs"
|
||||
exit 2
|
||||
elif [[ "$TEMP_RESULT" -eq 1 ]]; then
|
||||
warn "Drive temps high — skipping profile [$PROFILE_NAME]"
|
||||
exit 1
|
||||
else
|
||||
log "Drive temps OK — $TEMP_CHECK_RESULT"
|
||||
fi
|
||||
|
||||
# Version parity — refuse if servers on incompatible unRAID versions
|
||||
check_os_version_parity || exit 1
|
||||
|
||||
check_connectivity
|
||||
check_remote_rootfs
|
||||
check_remote_share "$DIRECTORY"
|
||||
check_remote_disks "$DIRECTORY"
|
||||
|
||||
# Remote Docker daemon — check before attempting container operations
|
||||
if [[ ${#CRITICAL_CONTAINER_NAMES[@]} -gt 0 ]] || [[ ${#REMOTE_RESTART_CONTAINERS[@]} -gt 0 ]]; then
|
||||
check_remote_docker_daemon || {
|
||||
warn "Remote Docker daemon unresponsive — skipping container operations"
|
||||
warn "Proceeding with rsync only — containers will not be stopped or restarted"
|
||||
CRITICAL_CONTAINER_NAMES=()
|
||||
LOCAL_CRITICAL_CONTAINER_NAMES=()
|
||||
REMOTE_RESTART_CONTAINERS=()
|
||||
}
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Stop Containers ━━━
|
||||
# ==============================================================================================
|
||||
if [[ ${#CRITICAL_CONTAINER_NAMES[@]} -gt 0 || ${#LOCAL_CRITICAL_CONTAINER_NAMES[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_STOP $ICON_CONTAINERS Stop Containers ━━━"
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
for c in "${LOCAL_CRITICAL_CONTAINER_NAMES[@]:-}"; do
|
||||
[[ -n "$c" ]] && warn "DRY RUN — would stop local: $c"
|
||||
done
|
||||
for c in "${CRITICAL_CONTAINER_NAMES[@]:-}"; do
|
||||
[[ -n "$c" ]] && warn "DRY RUN — would stop remote: $c"
|
||||
done
|
||||
else
|
||||
# Local first — flush local databases before pushing
|
||||
stop_local_containers
|
||||
# Remote next — prevent writes while receiving
|
||||
stop_containers
|
||||
fi
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Transfer ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Transfer ━━━"
|
||||
echo "$ICON_RUN Source: $DIRECTORY"
|
||||
echo "$ICON_NET Remote: $REMOTE_SERVER:$DIRECTORY"
|
||||
echo "$ICON_GEAR Profile: $PROFILE_NAME"
|
||||
echo "$ICON_HOST Identity: $MY_ID → $REMOTE_ID"
|
||||
echo ""
|
||||
|
||||
get_rsync_opts
|
||||
|
||||
# Append profile excludes
|
||||
for ex in "${EXCLUDE_DIRS[@]:-}"; do
|
||||
[[ -n "$ex" ]] && RSYNC_OPTS+=(--exclude="$ex")
|
||||
done
|
||||
|
||||
# Add --stats to capture bytes transferred for bandwidth logging
|
||||
RSYNC_OPTS+=(--stats)
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && RSYNC_OPTS+=("--dry-run")
|
||||
|
||||
START=$(date +%s)
|
||||
RSYNC_SUCCESS=false
|
||||
BYTES_TRANSFERRED=0
|
||||
ATTEMPT=0
|
||||
|
||||
for (( ATTEMPT=1; ATTEMPT<=RETRY_COUNT; ATTEMPT++ )); do
|
||||
log "$ICON_RETRY Attempt $ATTEMPT of $RETRY_COUNT..."
|
||||
echo "$ICON_SYNC Rsync running — this may take a while..."
|
||||
|
||||
RSYNC_OUTPUT=$(rsync "${RSYNC_OPTS[@]}" \
|
||||
-e "ssh -i $SSH_KEY -T -o Compression=no -o IPQoS=throughput" \
|
||||
"$DIRECTORY" "root@${REMOTE_SERVER}:$(dirname "$DIRECTORY")/" 2>&1)
|
||||
|
||||
RSYNC_EXIT=$?
|
||||
|
||||
if [[ "$RSYNC_EXIT" -eq 0 ]]; then
|
||||
# Parse bytes transferred from --stats output
|
||||
BYTES_TRANSFERRED=$(echo "$RSYNC_OUTPUT" | \
|
||||
awk '/Total transferred file size:/{gsub(/,/,"",$NF); gsub(/[^0-9]/,"",$NF); print $NF+0}')
|
||||
BYTES_TRANSFERRED="${BYTES_TRANSFERRED:-0}"
|
||||
|
||||
log "$ICON_DONE Rsync complete — $BYTES_TRANSFERRED bytes transferred"
|
||||
RSYNC_SUCCESS=true
|
||||
break
|
||||
else
|
||||
warn "$ICON_RETRY Rsync failed (attempt $ATTEMPT/$RETRY_COUNT)"
|
||||
log "Exit code: $RSYNC_EXIT"
|
||||
if [[ "$ATTEMPT" -lt "$RETRY_COUNT" ]]; then
|
||||
log "Retrying in ${SLEEP}s..."
|
||||
sleep "$SLEEP"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Start Containers ━━━
|
||||
# ==============================================================================================
|
||||
if [[ ${#CRITICAL_CONTAINER_NAMES[@]} -gt 0 || ${#LOCAL_CRITICAL_CONTAINER_NAMES[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_START $ICON_CONTAINERS Start Containers ━━━"
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
for c in "${CRITICAL_CONTAINER_NAMES[@]:-}"; do
|
||||
[[ -n "$c" ]] && warn "DRY RUN — would start remote: $c"
|
||||
done
|
||||
for c in "${LOCAL_CRITICAL_CONTAINER_NAMES[@]:-}"; do
|
||||
[[ -n "$c" ]] && warn "DRY RUN — would start local: $c"
|
||||
done
|
||||
else
|
||||
# Remote first — can be coming up while local restarts
|
||||
start_containers
|
||||
# Local next
|
||||
start_local_containers
|
||||
fi
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Remote Restart (dirty sync profiles) ━━━
|
||||
# ==============================================================================================
|
||||
# For dirty sync profiles (critical-fallback, emby-fallback) — restart containers on remote
|
||||
# that were running before sync so they pick up config changes from the dirty sync window.
|
||||
# Was running → restart. Was stopped → leave stopped.
|
||||
if [[ ${#REMOTE_RESTART_CONTAINERS[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_START $ICON_CONTAINERS Remote Restart (post dirty sync) ━━━"
|
||||
log "Restarting configured containers on $REMOTE_SERVER_NAME..."
|
||||
|
||||
for container in "${REMOTE_RESTART_CONTAINERS[@]}"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
|
||||
# Check if container was running before sync (still tracked via RUNNING_CONTAINERS)
|
||||
WAS_RUNNING=false
|
||||
for prev in "${RUNNING_CONTAINERS[@]:-}"; do
|
||||
[[ "$prev" == "$container" ]] && WAS_RUNNING=true && break
|
||||
done
|
||||
|
||||
if [[ "$WAS_RUNNING" == false ]]; then
|
||||
# Not in stop list — check current remote state
|
||||
REMOTE_STATUS=$(timeout 15 ssh -i "$SSH_KEY" \
|
||||
-o ConnectTimeout=10 root@"$REMOTE_SERVER" \
|
||||
"docker inspect -f '{{.State.Running}}' $container 2>/dev/null" 2>/dev/null)
|
||||
[[ "$REMOTE_STATUS" != "true" ]] && \
|
||||
log "$container not running on $REMOTE_SERVER_NAME — skipping remote restart" && \
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would restart $container on $REMOTE_SERVER_NAME"
|
||||
continue
|
||||
fi
|
||||
|
||||
timeout 15 ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \
|
||||
"docker restart $container" >/dev/null 2>&1 && \
|
||||
log "$ICON_STARTED $container restarted on $REMOTE_SERVER_NAME ✅" || \
|
||||
warn "Failed to restart $container on $REMOTE_SERVER_NAME"
|
||||
done
|
||||
fi
|
||||
|
||||
END=$(date +%s)
|
||||
DURATION=$(( END - START ))
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Bandwidth Logging ━━━
|
||||
# ==============================================================================================
|
||||
# Logs to bandwidth_monitor.sh — new format includes bytes transferred and warn flag.
|
||||
# Only logs on actual runs (not dry-run) and only when bandwidth_monitor.sh exists.
|
||||
BANDWIDTH_MONITOR="$SCRIPT_DIR/../Monitors/bandwidth_monitor.sh"
|
||||
|
||||
if [[ "$DRY_RUN" == false ]] && [[ -f "$BANDWIDTH_MONITOR" ]]; then
|
||||
STATUS="success"
|
||||
[[ "$RSYNC_SUCCESS" == false ]] && STATUS="failed"
|
||||
bash "$BANDWIDTH_MONITOR" --log-transfer \
|
||||
"$PROFILE_NAME" "$DURATION" "$STATUS" "$BYTES_TRANSFERRED"
|
||||
log "$ICON_BANDWIDTH Transfer logged to bandwidth monitor ($BYTES_TRANSFERRED bytes)"
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY RSYNC SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_RUN Directory: $DIRECTORY"
|
||||
echo "$ICON_GEAR Profile: $PROFILE_NAME"
|
||||
echo "$ICON_TIME Duration: $(format_duration $DURATION)"
|
||||
[[ "$BYTES_TRANSFERRED" -gt 0 ]] && \
|
||||
echo "$ICON_BANDWIDTH Transferred: $(awk "BEGIN {printf \"%.2fGB\", $BYTES_TRANSFERRED / 1073741824}")"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no changes made"
|
||||
elif [[ "$RSYNC_SUCCESS" == true ]]; then
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS DONE"
|
||||
else
|
||||
echo "$ICON_ERROR Status: FAILED after $RETRY_COUNT attempts"
|
||||
notify "Rsync FAILED — $DIRECTORY ($PROFILE_NAME) after $RETRY_COUNT attempts on $(hostname)" \
|
||||
"Rsync" "warning"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
# Flush tee and preserve log for UI — close both ends of the pipe so tee gets EOF
|
||||
exec 1>&- 2>&-; wait
|
||||
cp "$VV_LIVE_LOG" "$VV_LAST_LOG" 2>/dev/null
|
||||
rm -f "$VV_LIVE_LOG"
|
||||
|
||||
[[ "$RSYNC_SUCCESS" == false ]] && [[ "$DRY_RUN" == false ]] && exit 1
|
||||
exit 0
|
||||
+195
@@ -0,0 +1,195 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ================================= Mover Stop =================================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Safely stops the unRAID mover with a wall warning, configurable timeout, and
|
||||
# SIGTERM → SIGKILL sequence. Use before planned reboots, disk operations, or
|
||||
# any operation where mover and rsync running simultaneously could corrupt files.
|
||||
# Exits cleanly if mover is not running.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Stop Sequence
|
||||
# 1. Check if mover is running — exit cleanly if not
|
||||
# 2. Wall message to all logged-in terminal users
|
||||
# 3. Wait MOVER_STOP_TIMEOUT seconds
|
||||
# 4. SIGTERM — allows mover to finish its current file before stopping
|
||||
# (no partial files — the mover completes what it is working on)
|
||||
# 5. Wait 5 seconds → verify stopped
|
||||
# 6. SIGKILL if still running — forced stop, partial files possible
|
||||
# 7. Final verify — error if still running after SIGKILL
|
||||
#
|
||||
# SIGTERM first because the mover has an opportunity to finish the file it is
|
||||
# currently moving, leaving no partial copies on cache or array. SIGKILL is only
|
||||
# used as a last resort and may leave a file split across cache and array.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Single Instance Lock
|
||||
# acquire_lock prevents concurrent stop attempts racing each other.
|
||||
#
|
||||
# Root Required
|
||||
# pkill on emhttp processes requires root.
|
||||
#
|
||||
# Final Verify
|
||||
# Confirms mover is actually stopped after the kill sequence — errors if it
|
||||
# is still running after SIGKILL.
|
||||
#
|
||||
# Silent When Clean
|
||||
# Mover not running = log() only, no visible output.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# master.conf
|
||||
#
|
||||
# MOVER_STOP_TIMEOUT
|
||||
# Seconds between wall warning and SIGTERM. (default: 30)
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# mover_stop.sh
|
||||
# Check if mover is running. If so, warn users and stop it.
|
||||
#
|
||||
# mover_stop.sh --dry-run
|
||||
# Show mover state and what would happen. No signals sent.
|
||||
#
|
||||
# mover_stop.sh --status
|
||||
# Show mover state (running, PID, start time). Then exit.
|
||||
#
|
||||
# mover_stop.sh --log
|
||||
# Verbose output showing each step of the stop sequence.
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../../../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root — pkill on emhttp processes requires root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
platform_require_cmd \
|
||||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||||
"" "" \
|
||||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||
|
||||
validate_int MOVER_STOP_TIMEOUT "$MOVER_STOP_TIMEOUT"
|
||||
|
||||
acquire_lock
|
||||
|
||||
detect_hosts
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_MOVER Timeout: ${MOVER_STOP_TIMEOUT}s"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo ""
|
||||
if platform_is_mover_running; then
|
||||
MOVER_PID=$(platform_get_mover_pid)
|
||||
MOVER_START=$(ps -o lstart= -p "$MOVER_PID" 2>/dev/null | xargs)
|
||||
echo " $ICON_MOVER Mover: RUNNING (PID $MOVER_PID)"
|
||||
[[ -n "$MOVER_START" ]] && echo " $ICON_TIME Started: $MOVER_START"
|
||||
else
|
||||
echo " $ICON_MOVER Mover: not running"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Mover Stop ━━━
|
||||
# ==============================================================================================
|
||||
START=$(date +%s)
|
||||
|
||||
if ! platform_is_mover_running; then
|
||||
echo "Mover is not running — nothing to do"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
MOVER_PID=$(platform_get_mover_pid)
|
||||
MOVER_START=$(ps -o lstart= -p "$MOVER_PID" 2>/dev/null | xargs)
|
||||
MOVER_ELAPSED=$(ps -o etimes= -p "$MOVER_PID" 2>/dev/null | tr -d ' ')
|
||||
warn "Mover is running (PID $MOVER_PID) — stopping in ${MOVER_STOP_TIMEOUT}s"
|
||||
log "$ICON_TIME Mover started: ${MOVER_START:-unknown} — running for $(format_duration "${MOVER_ELAPSED:-0}")"
|
||||
|
||||
# ── Warn users via wall ───────────────────────────────────────────────────────────────────────
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
wall "$ICON_WARN $MY_ID ($LOCAL_SERVER_NAME) — unRAID Mover stopping in ${MOVER_STOP_TIMEOUT}s"
|
||||
echo "Wall message sent — waiting ${MOVER_STOP_TIMEOUT}s..."
|
||||
sleep "$MOVER_STOP_TIMEOUT"
|
||||
else
|
||||
warn "DRY RUN — would send wall warning and wait ${MOVER_STOP_TIMEOUT}s"
|
||||
fi
|
||||
|
||||
# ── SIGTERM — graceful stop ───────────────────────────────────────────────────────────────────
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would send SIGTERM to mover (PID $MOVER_PID)"
|
||||
else
|
||||
log "Sending SIGTERM to mover (PID $MOVER_PID)..."
|
||||
kill -TERM "$MOVER_PID" 2>/dev/null || true
|
||||
sleep 5
|
||||
|
||||
# Verify stopped after SIGTERM
|
||||
if ! platform_is_mover_running; then
|
||||
warn "Mover stopped cleanly (SIGTERM) ✅"
|
||||
else
|
||||
# ── SIGKILL — forced stop ─────────────────────────────────────────────────────────────
|
||||
warn "Mover still running after SIGTERM — sending SIGKILL (may leave partial files)"
|
||||
kill -KILL "$MOVER_PID" 2>/dev/null || true
|
||||
sleep 2
|
||||
|
||||
# Final verify
|
||||
if platform_is_mover_running; then
|
||||
error "Mover still running after SIGKILL — manual intervention needed"
|
||||
notify "Mover stop failed on $(hostname) ($MY_ID) — process unkillable" \
|
||||
"Mover Stop" "warning"
|
||||
exit 1
|
||||
else
|
||||
warn "Mover force-stopped (SIGKILL) — check for partial files on cache"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY MOVER STOP SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_MOVER Timeout: ${MOVER_STOP_TIMEOUT}s"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no changes made"
|
||||
else
|
||||
echo "$ICON_DONE Status: done — mover stopped ✅"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
+1457
File diff suppressed because it is too large
Load Diff
+1452
File diff suppressed because it is too large
Load Diff
+1451
File diff suppressed because it is too large
Load Diff
+1454
File diff suppressed because it is too large
Load Diff
+399
@@ -0,0 +1,399 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ================================= Docker Update ==============================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Pulls the latest images for configured containers. Two modes: normal (daily)
|
||||
# and remainder (weekly).
|
||||
#
|
||||
# Normal mode is called by daily_sync_maintenance.sh before docker_daily_restart.sh.
|
||||
# Containers stay running during the pull — no extra downtime beyond what the
|
||||
# nightly restart already causes.
|
||||
#
|
||||
# Remainder mode is called by weekly_sync_maintenance.sh as the final update step.
|
||||
# It catches everything that normal mode and the weekly sync window did not already
|
||||
# update — derived automatically from docker ps, nothing to configure.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Normal mode (daily):
|
||||
# Targets DAILY_RESTART_CONTAINERS — same list used by docker_daily_restart.sh.
|
||||
# Pull → compare old vs new image ID → mark updated or already current.
|
||||
# docker_daily_restart.sh runs after — containers restart onto the fresh image.
|
||||
#
|
||||
# Remainder mode (weekly):
|
||||
# Targets all currently running containers NOT in:
|
||||
# DAILY_RESTART_CONTAINERS — already updated daily
|
||||
# emby + critical-data profiles — updated inline by the weekly sync window
|
||||
# FALLBACK_*_TIER* — owned by the remote server's update cycle
|
||||
# Pull → compare → prune dangling images.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# DESIGN PRINCIPLES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Single List
|
||||
# Normal mode reuses DAILY_RESTART_CONTAINERS rather than maintaining a
|
||||
# separate update list. Adding or removing a container from the restart list
|
||||
# automatically updates the image pull list — one change, both places.
|
||||
#
|
||||
# Version Ownership
|
||||
# Fallback containers are excluded from remainder mode. This server only runs
|
||||
# them during a fallback. The remote server owns their version — if remainder
|
||||
# updates them independently and a handback occurs, the remote's older image
|
||||
# may not handle data written by the newer version.
|
||||
#
|
||||
# State Respect
|
||||
# Stopped containers are never targeted. Pulling while stopped adds no value
|
||||
# and a stopped container was likely halted intentionally.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Lock Acquisition
|
||||
# Prevents concurrent execution via acquire_lock(). Safe to call from
|
||||
# maintenance scripts without risk of overlap.
|
||||
#
|
||||
# Host Detection
|
||||
# detect_hosts() identifies which server is running the script and aliases
|
||||
# HOST*_DAILY_RESTART_CONTAINERS to the correct host's values.
|
||||
#
|
||||
# Root Enforcement
|
||||
# Docker operations require root privileges.
|
||||
#
|
||||
# DAILY_CONTAINER_UPDATES Toggle
|
||||
# Normal mode exits cleanly when disabled. docker_daily_restart.sh still runs
|
||||
# regardless — update and restart are independent operations.
|
||||
#
|
||||
# Fallback Exclusion
|
||||
# Remainder mode excludes containers owned by the remote server's update cycle
|
||||
# to prevent version divergence across the fallback boundary.
|
||||
#
|
||||
# Running-Only Filter
|
||||
# Stopped containers excluded from remainder mode — intentionally down.
|
||||
#
|
||||
# Image ID Comparison
|
||||
# Containers not restarted unless their image actually changed. Pulls that
|
||||
# result in "already up to date" produce no restart.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# master.conf
|
||||
#
|
||||
# DAILY_CONTAINER_UPDATES
|
||||
# Enable or disable normal mode. docker_daily_restart.sh runs regardless.
|
||||
# (default: true)
|
||||
#
|
||||
# PROFILE_CRITICAL_CONTAINER_NAMES[emby|critical-data]
|
||||
# Container names for emby and critical-data profiles — excluded from
|
||||
# remainder mode (already updated by the weekly sync window)
|
||||
#
|
||||
# host*.conf
|
||||
#
|
||||
# HOST*_DAILY_RESTART_CONTAINERS
|
||||
# Containers updated in normal mode. Aliased by detect_hosts() →
|
||||
# DAILY_RESTART_CONTAINERS
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# docker_update.sh
|
||||
# Normal mode — pull latest images for DAILY_RESTART_CONTAINERS
|
||||
#
|
||||
# docker_update.sh --remainder
|
||||
# Remainder mode — pull all running containers not in managed lists,
|
||||
# restart those that received updates, prune dangling images
|
||||
#
|
||||
# docker_update.sh --dry-run
|
||||
# Preview which containers would be pulled without making changes
|
||||
#
|
||||
# docker_update.sh --status
|
||||
# Show configuration and container list for current mode
|
||||
#
|
||||
# docker_update.sh --log
|
||||
# Verbose per-container pull and comparison output
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
# Pre-parse --remainder before parse_args (unknown args pass through to PARSED_ARGS)
|
||||
REMAINDER_MODE=false
|
||||
_filtered_args=()
|
||||
for _arg in "$@"; do
|
||||
if [[ "$_arg" == "--remainder" ]]; then
|
||||
REMAINDER_MODE=true
|
||||
else
|
||||
_filtered_args+=("$_arg")
|
||||
fi
|
||||
done
|
||||
unset _arg
|
||||
|
||||
parse_args "${_filtered_args[@]}"
|
||||
unset _filtered_args
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
acquire_lock
|
||||
|
||||
if ! command -v docker &>/dev/null; then
|
||||
error "Docker command not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
detect_hosts
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Container Discovery ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$REMAINDER_MODE" == true ]]; then
|
||||
declare -A _exclude=()
|
||||
|
||||
# Daily containers — updated by docker_update.sh normal mode
|
||||
for _c in "${DAILY_RESTART_CONTAINERS[@]}"; do
|
||||
[[ -n "$_c" ]] && _exclude["$_c"]=1
|
||||
done
|
||||
|
||||
# Weekly sync-window containers (emby + critical-data) — updated inline by weekly_sync_maintenance.sh
|
||||
_weekly_str="${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-}"
|
||||
read -r -a _weekly_arr <<< "$_weekly_str"
|
||||
for _c in "${_weekly_arr[@]}"; do
|
||||
[[ -n "$_c" ]] && _exclude["$_c"]=1
|
||||
done
|
||||
unset _weekly_str _weekly_arr
|
||||
|
||||
# Fallback coverage containers — owned by the remote server's update cycle.
|
||||
# This server runs them during fallback but should never update them independently.
|
||||
# Updating them here risks version divergence: if remote's writeback after handback
|
||||
# encounters data written by a newer version, it may not handle it correctly.
|
||||
for _tier in 1 2 3 4; do
|
||||
_tier_var="FALLBACK_${MY_ID}_COVERS_${REMOTE_ID}_TIER${_tier}"
|
||||
eval "_tier_arr=(\"\${${_tier_var}[@]:-}\")" 2>/dev/null
|
||||
for _c in "${_tier_arr[@]}"; do
|
||||
[[ -n "$_c" ]] && _exclude["$_c"]=1
|
||||
done
|
||||
done
|
||||
unset _tier _tier_var _tier_arr _c
|
||||
|
||||
mapfile -t _all_running < <(docker ps --format '{{.Names}}' | sort)
|
||||
TARGET_CONTAINERS=()
|
||||
for _c in "${_all_running[@]}"; do
|
||||
[[ -z "${_exclude[$_c]+x}" ]] && TARGET_CONTAINERS+=("$_c")
|
||||
done
|
||||
unset _all_running _exclude _c
|
||||
else
|
||||
if [[ "${DAILY_CONTAINER_UPDATES:-true}" != "true" ]]; then
|
||||
echo "DAILY_CONTAINER_UPDATES=false — skipping container updates"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ ${#DAILY_RESTART_CONTAINERS[@]} -eq 0 ]]; then
|
||||
warn "DAILY_RESTART_CONTAINERS is empty for $MY_ID — nothing to update"
|
||||
warn "Check HOST${MY_ID#HOST}_DAILY_RESTART_CONTAINERS in host*.conf"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
TARGET_CONTAINERS=("${DAILY_RESTART_CONTAINERS[@]}")
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_GEAR Mode: $([[ "$REMAINDER_MODE" == true ]] && echo "remainder" || echo "normal (daily)")"
|
||||
if [[ "$REMAINDER_MODE" == true ]]; then
|
||||
echo "$ICON_CONTAINERS Containers: ${#TARGET_CONTAINERS[@]} running (excluding daily, weekly sync, and fallback)"
|
||||
for _c in "${TARGET_CONTAINERS[@]}"; do echo " $_c"; done
|
||||
else
|
||||
echo "$ICON_CONTAINERS Containers: ${TARGET_CONTAINERS[*]}"
|
||||
echo "$ICON_GEAR Enabled: ${DAILY_CONTAINER_UPDATES:-true}"
|
||||
fi
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no images will be pulled"
|
||||
|
||||
if [[ ${#TARGET_CONTAINERS[@]} -eq 0 ]]; then
|
||||
echo "No containers to update"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Pull Updates ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
if [[ "$REMAINDER_MODE" == true ]]; then
|
||||
echo "━━━ $ICON_CONTAINERS Docker Update (remainder) — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo "$ICON_CONTAINERS Updating ${#TARGET_CONTAINERS[@]} container(s) (not in daily or weekly sync)"
|
||||
log "$ICON_CONTAINERS Remainder targets: ${TARGET_CONTAINERS[*]}"
|
||||
else
|
||||
echo "━━━ $ICON_CONTAINERS Docker Update — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
log "$ICON_CONTAINERS Containers: ${TARGET_CONTAINERS[*]}"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
START=$(date +%s)
|
||||
UPDATED=()
|
||||
UP_TO_DATE=()
|
||||
FAILED=()
|
||||
OLD_IMAGE_IDS=() # old image IDs to explicitly remove after rebuilds
|
||||
SKIPPED=()
|
||||
|
||||
for container in "${TARGET_CONTAINERS[@]}"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
log "━━━ $ICON_CONTAINERS $container ━━━"
|
||||
|
||||
if ! docker inspect "$container" &>/dev/null; then
|
||||
warn "$container — not found, skipping"
|
||||
SKIPPED+=("$container")
|
||||
continue
|
||||
fi
|
||||
|
||||
IMAGE=$(docker inspect --format='{{.Config.Image}}' "$container" 2>/dev/null)
|
||||
if [[ -z "$IMAGE" ]]; then
|
||||
warn "$container — could not determine image, skipping"
|
||||
SKIPPED+=("$container")
|
||||
continue
|
||||
fi
|
||||
|
||||
log "$container — image: $IMAGE"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would pull: $IMAGE"
|
||||
UPDATED+=("$container")
|
||||
continue
|
||||
fi
|
||||
|
||||
# Capture the image ID the container is currently running on, and the
|
||||
# image ID :latest points to before the pull. After pulling, we rebuild if
|
||||
# either a new digest landed OR the container is behind what :latest is now.
|
||||
CONTAINER_IMAGE_ID=$(docker inspect "$container" --format='{{.Image}}' 2>/dev/null || echo "")
|
||||
OLD_ID=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "")
|
||||
|
||||
log "$ICON_SYNC Pulling $IMAGE..."
|
||||
if [[ "$ENABLE_LOGGING" == "true" ]]; then
|
||||
docker pull "$IMAGE" 2>&1 | grep -E "^(Status:|Digest:|Error|error)" | sed 's/^/ /'
|
||||
_pull_rc=${PIPESTATUS[0]}
|
||||
else
|
||||
docker pull "$IMAGE" >/dev/null 2>&1
|
||||
_pull_rc=$?
|
||||
fi
|
||||
NEW_ID=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "")
|
||||
|
||||
if [[ $_pull_rc -eq 0 ]]; then
|
||||
_pull_new=$( [[ -n "$OLD_ID" && "$OLD_ID" != "$NEW_ID" ]] && echo true || echo false)
|
||||
_container_behind=$([[ -n "$CONTAINER_IMAGE_ID" && -n "$NEW_ID" && "$CONTAINER_IMAGE_ID" != "$NEW_ID" ]] && echo true || echo false)
|
||||
|
||||
if [[ "$_pull_new" == true || "$_container_behind" == true ]]; then
|
||||
[[ "$_pull_new" == true ]] && log "$ICON_DONE $container — new image (${OLD_ID:7:12} → ${NEW_ID:7:12})"
|
||||
[[ "$_container_behind" == true && "$_pull_new" == false ]] && log "$ICON_DONE $container — image already pulled, container behind (${CONTAINER_IMAGE_ID:7:12} → ${NEW_ID:7:12})"
|
||||
UPDATED+=("$container")
|
||||
OLD_IMAGE_IDS+=("$CONTAINER_IMAGE_ID")
|
||||
else
|
||||
log "$container — up to date (${NEW_ID:7:12})"
|
||||
UP_TO_DATE+=("$container")
|
||||
fi
|
||||
else
|
||||
warn "$container — pull failed ($IMAGE)"
|
||||
FAILED+=("$container")
|
||||
fi
|
||||
|
||||
done
|
||||
|
||||
# ── Recreate containers that received a new image ────────────────────────────
|
||||
# docker restart uses the image ID baked in at creation time — it never picks
|
||||
# up the new digest. rebuild_container reads the stored XML template, stops the
|
||||
# old container, recreates it (new image, same config), then prunes the old image.
|
||||
REBUILT=()
|
||||
REBUILD_FAILED=()
|
||||
if [[ ${#UPDATED[@]} -gt 0 ]]; then
|
||||
for container in "${UPDATED[@]}"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would rebuild $container from template"
|
||||
REBUILT+=("$container")
|
||||
continue
|
||||
fi
|
||||
log "$ICON_SYNC Rebuilding $container from template on new image..."
|
||||
if platform_rebuild_container "$container"; then
|
||||
log "$ICON_DONE $container rebuilt ✅"
|
||||
REBUILT+=("$container")
|
||||
else
|
||||
error "Failed to rebuild $container — will be picked up by docker_daily_restart.sh"
|
||||
notify "$container failed to rebuild after image update on $(hostname)" "Docker Update" "warning"
|
||||
REBUILD_FAILED+=("$container")
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
# ── Remove old images ────────────────────────────────────────────────────────
|
||||
# Explicitly rmi by the IDs captured before each pull. Tagged images are never
|
||||
# caught by dangling-only prune, so this is the only reliable cleanup path.
|
||||
# Fall through to dangling prune to catch any leftovers from other update paths.
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would remove ${#OLD_IMAGE_IDS[@]} old image(s) and prune dangling"
|
||||
PRUNED_SUMMARY="(dry run)"
|
||||
else
|
||||
for _old_id in "${OLD_IMAGE_IDS[@]}"; do
|
||||
docker rmi "$_old_id" >/dev/null 2>&1 || true
|
||||
done
|
||||
PRUNED_OUTPUT=$(docker image prune -f 2>&1)
|
||||
[[ "$ENABLE_LOGGING" == "true" ]] && echo "$PRUNED_OUTPUT" | sed 's/^/ /'
|
||||
PRUNED_SUMMARY=$(echo "$PRUNED_OUTPUT" | grep -E "^Total reclaimed" || echo "nothing reclaimed")
|
||||
fi
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$REMAINDER_MODE" == true ]]; then
|
||||
echo "━━━━━ $ICON_SUMMARY DOCKER UPDATE (REMAINDER) SUMMARY ━━━━━"
|
||||
else
|
||||
echo "━━━━━ $ICON_SUMMARY DOCKER UPDATE SUMMARY ━━━━━"
|
||||
fi
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
if [[ ${#UPDATED[@]} -gt 0 ]]; then
|
||||
echo "$ICON_DONE Updated: ${#UPDATED[@]}"
|
||||
log " ${UPDATED[*]}"
|
||||
fi
|
||||
[[ ${#REBUILT[@]} -gt 0 ]] && echo "$ICON_SYNC Rebuilt: ${#REBUILT[@]}"
|
||||
[[ ${#REBUILD_FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Rebuild fail:${REBUILD_FAILED[*]}"
|
||||
[[ ${#UP_TO_DATE[@]} -gt 0 ]] && log "$ICON_RUNNING Up to date: ${UP_TO_DATE[*]}"
|
||||
[[ ${#SKIPPED[@]} -gt 0 ]] && log "$ICON_WARN Skipped: ${SKIPPED[*]}"
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}"
|
||||
echo "$ICON_SYNC Pruned: ${PRUNED_SUMMARY:-none}"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no images pulled"
|
||||
elif [[ ${#FAILED[@]} -eq 0 ]]; then
|
||||
echo "$ICON_DONE Status: done ✅ — ${#UPDATED[@]} updated, ${#UP_TO_DATE[@]} current"
|
||||
else
|
||||
warn "Status: ${#FAILED[@]} pull(s) failed — restart will proceed with existing images"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
# Pull failures are non-fatal — restart proceeds regardless
|
||||
exit 0
|
||||
+405
@@ -0,0 +1,405 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ================================= Docker Update ==============================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Pulls the latest images for configured containers. Two modes: normal (daily)
|
||||
# and remainder (weekly).
|
||||
#
|
||||
# Normal mode is called by daily_sync_maintenance.sh before docker_daily_restart.sh.
|
||||
# Containers stay running during the pull — no extra downtime beyond what the
|
||||
# nightly restart already causes.
|
||||
#
|
||||
# Remainder mode is called by weekly_sync_maintenance.sh as the final update step.
|
||||
# It catches everything that normal mode and the weekly sync window did not already
|
||||
# update — derived automatically from docker ps, nothing to configure.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Normal mode (daily):
|
||||
# Targets DAILY_RESTART_CONTAINERS — same list used by docker_daily_restart.sh.
|
||||
# Pull → compare old vs new image ID → mark updated or already current.
|
||||
# docker_daily_restart.sh runs after — containers restart onto the fresh image.
|
||||
#
|
||||
# Remainder mode (weekly):
|
||||
# Targets all currently running containers NOT in:
|
||||
# DAILY_RESTART_CONTAINERS — already updated daily
|
||||
# emby + critical-data profiles — updated inline by the weekly sync window
|
||||
# WEEKLY_RESTART_CONTAINERS — restarted by docker_weekly_restart.sh after sync
|
||||
# FALLBACK_*_TIER* — owned by the remote server's update cycle
|
||||
# Pull → compare → prune dangling images.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# DESIGN PRINCIPLES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Single List
|
||||
# Normal mode reuses DAILY_RESTART_CONTAINERS rather than maintaining a
|
||||
# separate update list. Adding or removing a container from the restart list
|
||||
# automatically updates the image pull list — one change, both places.
|
||||
#
|
||||
# Version Ownership
|
||||
# Fallback containers are excluded from remainder mode. This server only runs
|
||||
# them during a fallback. The remote server owns their version — if remainder
|
||||
# updates them independently and a handback occurs, the remote's older image
|
||||
# may not handle data written by the newer version.
|
||||
#
|
||||
# State Respect
|
||||
# Stopped containers are never targeted. Pulling while stopped adds no value
|
||||
# and a stopped container was likely halted intentionally.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Lock Acquisition
|
||||
# Prevents concurrent execution via acquire_lock(). Safe to call from
|
||||
# maintenance scripts without risk of overlap.
|
||||
#
|
||||
# Host Detection
|
||||
# detect_hosts() identifies which server is running the script and aliases
|
||||
# HOST*_DAILY_RESTART_CONTAINERS to the correct host's values.
|
||||
#
|
||||
# Root Enforcement
|
||||
# Docker operations require root privileges.
|
||||
#
|
||||
# DAILY_CONTAINER_UPDATES Toggle
|
||||
# Normal mode exits cleanly when disabled. docker_daily_restart.sh still runs
|
||||
# regardless — update and restart are independent operations.
|
||||
#
|
||||
# Fallback Exclusion
|
||||
# Remainder mode excludes containers owned by the remote server's update cycle
|
||||
# to prevent version divergence across the fallback boundary.
|
||||
#
|
||||
# Running-Only Filter
|
||||
# Stopped containers excluded from remainder mode — intentionally down.
|
||||
#
|
||||
# Image ID Comparison
|
||||
# Containers not restarted unless their image actually changed. Pulls that
|
||||
# result in "already up to date" produce no restart.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# master.conf
|
||||
#
|
||||
# DAILY_CONTAINER_UPDATES
|
||||
# Enable or disable normal mode. docker_daily_restart.sh runs regardless.
|
||||
# (default: true)
|
||||
#
|
||||
# PROFILE_CRITICAL_CONTAINER_NAMES[emby|critical-data]
|
||||
# Container names for emby and critical-data profiles — excluded from
|
||||
# remainder mode (already updated by the weekly sync window)
|
||||
#
|
||||
# host*.conf
|
||||
#
|
||||
# HOST*_DAILY_RESTART_CONTAINERS
|
||||
# Containers updated in normal mode. Aliased by detect_hosts() →
|
||||
# DAILY_RESTART_CONTAINERS
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# docker_update.sh
|
||||
# Normal mode — pull latest images for DAILY_RESTART_CONTAINERS
|
||||
#
|
||||
# docker_update.sh --remainder
|
||||
# Remainder mode — pull all running containers not in managed lists,
|
||||
# restart those that received updates, prune dangling images
|
||||
#
|
||||
# docker_update.sh --dry-run
|
||||
# Preview which containers would be pulled without making changes
|
||||
#
|
||||
# docker_update.sh --status
|
||||
# Show configuration and container list for current mode
|
||||
#
|
||||
# docker_update.sh --log
|
||||
# Verbose per-container pull and comparison output
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
# Pre-parse --remainder before parse_args (unknown args pass through to PARSED_ARGS)
|
||||
REMAINDER_MODE=false
|
||||
_filtered_args=()
|
||||
for _arg in "$@"; do
|
||||
if [[ "$_arg" == "--remainder" ]]; then
|
||||
REMAINDER_MODE=true
|
||||
else
|
||||
_filtered_args+=("$_arg")
|
||||
fi
|
||||
done
|
||||
unset _arg
|
||||
|
||||
parse_args "${_filtered_args[@]}"
|
||||
unset _filtered_args
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
acquire_lock
|
||||
|
||||
if ! command -v docker &>/dev/null; then
|
||||
error "Docker command not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
detect_hosts
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Container Discovery ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$REMAINDER_MODE" == true ]]; then
|
||||
declare -A _exclude=()
|
||||
|
||||
# Daily containers — updated by docker_update.sh normal mode
|
||||
for _c in "${DAILY_RESTART_CONTAINERS[@]}"; do
|
||||
[[ -n "$_c" ]] && _exclude["$_c"]=1
|
||||
done
|
||||
|
||||
# Weekly sync-window containers (emby + critical-data) — updated inline by weekly_sync_maintenance.sh
|
||||
_weekly_str="${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-}"
|
||||
read -r -a _weekly_arr <<< "$_weekly_str"
|
||||
for _c in "${_weekly_arr[@]}"; do
|
||||
[[ -n "$_c" ]] && _exclude["$_c"]=1
|
||||
done
|
||||
unset _weekly_str _weekly_arr
|
||||
|
||||
# Weekly restart containers — restarted by docker_weekly_restart.sh after sync
|
||||
for _c in "${WEEKLY_RESTART_CONTAINERS[@]}"; do
|
||||
[[ -n "$_c" ]] && _exclude["$_c"]=1
|
||||
done
|
||||
|
||||
# Fallback coverage containers — owned by the remote server's update cycle.
|
||||
# This server runs them during fallback but should never update them independently.
|
||||
# Updating them here risks version divergence: if remote's writeback after handback
|
||||
# encounters data written by a newer version, it may not handle it correctly.
|
||||
for _tier in 1 2 3 4; do
|
||||
_tier_var="FALLBACK_${MY_ID}_COVERS_${REMOTE_ID}_TIER${_tier}"
|
||||
eval "_tier_arr=(\"\${${_tier_var}[@]:-}\")" 2>/dev/null
|
||||
for _c in "${_tier_arr[@]}"; do
|
||||
[[ -n "$_c" ]] && _exclude["$_c"]=1
|
||||
done
|
||||
done
|
||||
unset _tier _tier_var _tier_arr _c
|
||||
|
||||
mapfile -t _all_running < <(docker ps --format '{{.Names}}' | sort)
|
||||
TARGET_CONTAINERS=()
|
||||
for _c in "${_all_running[@]}"; do
|
||||
[[ -z "${_exclude[$_c]+x}" ]] && TARGET_CONTAINERS+=("$_c")
|
||||
done
|
||||
unset _all_running _exclude _c
|
||||
else
|
||||
if [[ "${DAILY_CONTAINER_UPDATES:-true}" != "true" ]]; then
|
||||
echo "DAILY_CONTAINER_UPDATES=false — skipping container updates"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ ${#DAILY_RESTART_CONTAINERS[@]} -eq 0 ]]; then
|
||||
warn "DAILY_RESTART_CONTAINERS is empty for $MY_ID — nothing to update"
|
||||
warn "Check HOST${MY_ID#HOST}_DAILY_RESTART_CONTAINERS in host*.conf"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
TARGET_CONTAINERS=("${DAILY_RESTART_CONTAINERS[@]}")
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_GEAR Mode: $([[ "$REMAINDER_MODE" == true ]] && echo "remainder" || echo "normal (daily)")"
|
||||
if [[ "$REMAINDER_MODE" == true ]]; then
|
||||
echo "$ICON_CONTAINERS Containers: ${#TARGET_CONTAINERS[@]} running (excluding daily, weekly sync, weekly restart, and fallback)"
|
||||
for _c in "${TARGET_CONTAINERS[@]}"; do echo " $_c"; done
|
||||
else
|
||||
echo "$ICON_CONTAINERS Containers: ${TARGET_CONTAINERS[*]}"
|
||||
echo "$ICON_GEAR Enabled: ${DAILY_CONTAINER_UPDATES:-true}"
|
||||
fi
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no images will be pulled"
|
||||
|
||||
if [[ ${#TARGET_CONTAINERS[@]} -eq 0 ]]; then
|
||||
echo "No containers to update"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Pull Updates ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
if [[ "$REMAINDER_MODE" == true ]]; then
|
||||
echo "━━━ $ICON_CONTAINERS Docker Update (remainder) — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo "$ICON_CONTAINERS Updating ${#TARGET_CONTAINERS[@]} container(s) (not in daily or weekly sync)"
|
||||
log "$ICON_CONTAINERS Remainder targets: ${TARGET_CONTAINERS[*]}"
|
||||
else
|
||||
echo "━━━ $ICON_CONTAINERS Docker Update — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
log "$ICON_CONTAINERS Containers: ${TARGET_CONTAINERS[*]}"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
START=$(date +%s)
|
||||
UPDATED=()
|
||||
UP_TO_DATE=()
|
||||
FAILED=()
|
||||
OLD_IMAGE_IDS=() # old image IDs to explicitly remove after rebuilds
|
||||
SKIPPED=()
|
||||
|
||||
for container in "${TARGET_CONTAINERS[@]}"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
log "━━━ $ICON_CONTAINERS $container ━━━"
|
||||
|
||||
if ! docker inspect "$container" &>/dev/null; then
|
||||
warn "$container — not found, skipping"
|
||||
SKIPPED+=("$container")
|
||||
continue
|
||||
fi
|
||||
|
||||
IMAGE=$(docker inspect --format='{{.Config.Image}}' "$container" 2>/dev/null)
|
||||
if [[ -z "$IMAGE" ]]; then
|
||||
warn "$container — could not determine image, skipping"
|
||||
SKIPPED+=("$container")
|
||||
continue
|
||||
fi
|
||||
|
||||
log "$container — image: $IMAGE"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would pull: $IMAGE"
|
||||
UPDATED+=("$container")
|
||||
continue
|
||||
fi
|
||||
|
||||
# Capture the image ID the container is currently running on, and the
|
||||
# image ID :latest points to before the pull. After pulling, we rebuild if
|
||||
# either a new digest landed OR the container is behind what :latest is now.
|
||||
CONTAINER_IMAGE_ID=$(docker inspect "$container" --format='{{.Image}}' 2>/dev/null || echo "")
|
||||
OLD_ID=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "")
|
||||
|
||||
log "$ICON_SYNC Pulling $IMAGE..."
|
||||
if [[ "$ENABLE_LOGGING" == "true" ]]; then
|
||||
docker pull "$IMAGE" 2>&1 | grep -E "^(Status:|Digest:|Error|error)" | sed 's/^/ /'
|
||||
_pull_rc=${PIPESTATUS[0]}
|
||||
else
|
||||
docker pull "$IMAGE" >/dev/null 2>&1
|
||||
_pull_rc=$?
|
||||
fi
|
||||
NEW_ID=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "")
|
||||
|
||||
if [[ $_pull_rc -eq 0 ]]; then
|
||||
_pull_new=$( [[ -n "$OLD_ID" && "$OLD_ID" != "$NEW_ID" ]] && echo true || echo false)
|
||||
_container_behind=$([[ -n "$CONTAINER_IMAGE_ID" && -n "$NEW_ID" && "$CONTAINER_IMAGE_ID" != "$NEW_ID" ]] && echo true || echo false)
|
||||
|
||||
if [[ "$_pull_new" == true || "$_container_behind" == true ]]; then
|
||||
[[ "$_pull_new" == true ]] && log "$ICON_DONE $container — new image (${OLD_ID:7:12} → ${NEW_ID:7:12})"
|
||||
[[ "$_container_behind" == true && "$_pull_new" == false ]] && log "$ICON_DONE $container — image already pulled, container behind (${CONTAINER_IMAGE_ID:7:12} → ${NEW_ID:7:12})"
|
||||
UPDATED+=("$container")
|
||||
OLD_IMAGE_IDS+=("$CONTAINER_IMAGE_ID")
|
||||
else
|
||||
log "$container — up to date (${NEW_ID:7:12})"
|
||||
UP_TO_DATE+=("$container")
|
||||
fi
|
||||
else
|
||||
warn "$container — pull failed ($IMAGE)"
|
||||
FAILED+=("$container")
|
||||
fi
|
||||
|
||||
done
|
||||
|
||||
# ── Recreate containers that received a new image ────────────────────────────
|
||||
# docker restart uses the image ID baked in at creation time — it never picks
|
||||
# up the new digest. rebuild_container reads the stored XML template, stops the
|
||||
# old container, recreates it (new image, same config), then prunes the old image.
|
||||
REBUILT=()
|
||||
REBUILD_FAILED=()
|
||||
if [[ ${#UPDATED[@]} -gt 0 ]]; then
|
||||
for container in "${UPDATED[@]}"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would rebuild $container from template"
|
||||
REBUILT+=("$container")
|
||||
continue
|
||||
fi
|
||||
log "$ICON_SYNC Rebuilding $container from template on new image..."
|
||||
if platform_rebuild_container "$container"; then
|
||||
log "$ICON_DONE $container rebuilt ✅"
|
||||
REBUILT+=("$container")
|
||||
else
|
||||
error "Failed to rebuild $container — will be picked up by docker_daily_restart.sh"
|
||||
notify "$container failed to rebuild after image update on $(hostname)" "Docker Update" "warning"
|
||||
REBUILD_FAILED+=("$container")
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
# ── Remove old images ────────────────────────────────────────────────────────
|
||||
# Explicitly rmi by the IDs captured before each pull. Tagged images are never
|
||||
# caught by dangling-only prune, so this is the only reliable cleanup path.
|
||||
# Fall through to dangling prune to catch any leftovers from other update paths.
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would remove ${#OLD_IMAGE_IDS[@]} old image(s) and prune dangling"
|
||||
PRUNED_SUMMARY="(dry run)"
|
||||
else
|
||||
for _old_id in "${OLD_IMAGE_IDS[@]}"; do
|
||||
docker rmi "$_old_id" >/dev/null 2>&1 || true
|
||||
done
|
||||
PRUNED_OUTPUT=$(docker image prune -f 2>&1)
|
||||
[[ "$ENABLE_LOGGING" == "true" ]] && echo "$PRUNED_OUTPUT" | sed 's/^/ /'
|
||||
PRUNED_SUMMARY=$(echo "$PRUNED_OUTPUT" | grep -E "^Total reclaimed" || echo "nothing reclaimed")
|
||||
fi
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$REMAINDER_MODE" == true ]]; then
|
||||
echo "━━━━━ $ICON_SUMMARY DOCKER UPDATE (REMAINDER) SUMMARY ━━━━━"
|
||||
else
|
||||
echo "━━━━━ $ICON_SUMMARY DOCKER UPDATE SUMMARY ━━━━━"
|
||||
fi
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
if [[ ${#UPDATED[@]} -gt 0 ]]; then
|
||||
echo "$ICON_DONE Updated: ${#UPDATED[@]}"
|
||||
log " ${UPDATED[*]}"
|
||||
fi
|
||||
[[ ${#REBUILT[@]} -gt 0 ]] && echo "$ICON_SYNC Rebuilt: ${#REBUILT[@]}"
|
||||
[[ ${#REBUILD_FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Rebuild fail:${REBUILD_FAILED[*]}"
|
||||
[[ ${#UP_TO_DATE[@]} -gt 0 ]] && log "$ICON_RUNNING Up to date: ${UP_TO_DATE[*]}"
|
||||
[[ ${#SKIPPED[@]} -gt 0 ]] && log "$ICON_WARN Skipped: ${SKIPPED[*]}"
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}"
|
||||
echo "$ICON_SYNC Pruned: ${PRUNED_SUMMARY:-none}"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no images pulled"
|
||||
elif [[ ${#FAILED[@]} -eq 0 ]]; then
|
||||
echo "$ICON_DONE Status: done ✅ — ${#UPDATED[@]} updated, ${#UP_TO_DATE[@]} current"
|
||||
else
|
||||
warn "Status: ${#FAILED[@]} pull(s) failed — restart will proceed with existing images"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
# Pull failures are non-fatal — restart proceeds regardless
|
||||
exit 0
|
||||
+413
@@ -0,0 +1,413 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ================================= Docker Update ==============================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Pulls the latest images for configured containers. Two modes: normal (daily)
|
||||
# and remainder (weekly).
|
||||
#
|
||||
# Normal mode is called by daily_sync_maintenance.sh before docker_daily_restart.sh.
|
||||
# Containers stay running during the pull — no extra downtime beyond what the
|
||||
# nightly restart already causes.
|
||||
#
|
||||
# Remainder mode is called by weekly_sync_maintenance.sh as the final update step.
|
||||
# It catches everything that normal mode and the weekly sync window did not already
|
||||
# update — derived automatically from docker ps, nothing to configure.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Normal mode (daily):
|
||||
# Targets DAILY_RESTART_CONTAINERS — same list used by docker_daily_restart.sh.
|
||||
# Pull → compare old vs new image ID → mark updated or already current.
|
||||
# docker_daily_restart.sh runs after — containers restart onto the fresh image.
|
||||
#
|
||||
# Remainder mode (weekly):
|
||||
# Targets all currently running containers NOT in:
|
||||
# DAILY_RESTART_CONTAINERS — already updated daily
|
||||
# emby + critical-data profiles — updated inline by the weekly sync window
|
||||
# WEEKLY_RESTART_CONTAINERS — restarted by docker_weekly_restart.sh after sync
|
||||
# FALLBACK_*_TIER* — owned by the remote server's update cycle
|
||||
# Pull → compare → prune dangling images.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# DESIGN PRINCIPLES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Single List
|
||||
# Normal mode reuses DAILY_RESTART_CONTAINERS rather than maintaining a
|
||||
# separate update list. Adding or removing a container from the restart list
|
||||
# automatically updates the image pull list — one change, both places.
|
||||
#
|
||||
# Version Ownership
|
||||
# Fallback containers are excluded from remainder mode. This server only runs
|
||||
# them during a fallback. The remote server owns their version — if remainder
|
||||
# updates them independently and a handback occurs, the remote's older image
|
||||
# may not handle data written by the newer version.
|
||||
#
|
||||
# State Respect
|
||||
# Stopped containers are never targeted. Pulling while stopped adds no value
|
||||
# and a stopped container was likely halted intentionally.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Lock Acquisition
|
||||
# Prevents concurrent execution via acquire_lock(). Safe to call from
|
||||
# maintenance scripts without risk of overlap.
|
||||
#
|
||||
# Host Detection
|
||||
# detect_hosts() identifies which server is running the script and aliases
|
||||
# HOST*_DAILY_RESTART_CONTAINERS to the correct host's values.
|
||||
#
|
||||
# Root Enforcement
|
||||
# Docker operations require root privileges.
|
||||
#
|
||||
# DAILY_CONTAINER_UPDATES Toggle
|
||||
# Normal mode exits cleanly when disabled. docker_daily_restart.sh still runs
|
||||
# regardless — update and restart are independent operations.
|
||||
#
|
||||
# Fallback Exclusion
|
||||
# Remainder mode excludes containers owned by the remote server's update cycle
|
||||
# to prevent version divergence across the fallback boundary.
|
||||
#
|
||||
# Running-Only Filter
|
||||
# Stopped containers excluded from remainder mode — intentionally down.
|
||||
#
|
||||
# Image ID Comparison
|
||||
# Containers not restarted unless their image actually changed. Pulls that
|
||||
# result in "already up to date" produce no restart.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# master.conf
|
||||
#
|
||||
# DAILY_CONTAINER_UPDATES
|
||||
# Enable or disable normal mode. docker_daily_restart.sh runs regardless.
|
||||
# (default: true)
|
||||
#
|
||||
# WEEKLY_REMAINING_UPDATES
|
||||
# Enable or disable remainder mode. (default: true)
|
||||
#
|
||||
# PROFILE_CRITICAL_CONTAINER_NAMES[emby|critical-data]
|
||||
# Container names for emby and critical-data profiles — excluded from
|
||||
# remainder mode (already updated by the weekly sync window)
|
||||
#
|
||||
# host*.conf
|
||||
#
|
||||
# HOST*_DAILY_RESTART_CONTAINERS
|
||||
# Containers updated in normal mode. Aliased by detect_hosts() →
|
||||
# DAILY_RESTART_CONTAINERS
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# docker_update.sh
|
||||
# Normal mode — pull latest images for DAILY_RESTART_CONTAINERS
|
||||
#
|
||||
# docker_update.sh --remainder
|
||||
# Remainder mode — pull all running containers not in managed lists,
|
||||
# restart those that received updates, prune dangling images
|
||||
#
|
||||
# docker_update.sh --dry-run
|
||||
# Preview which containers would be pulled without making changes
|
||||
#
|
||||
# docker_update.sh --status
|
||||
# Show configuration and container list for current mode
|
||||
#
|
||||
# docker_update.sh --log
|
||||
# Verbose per-container pull and comparison output
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
# Pre-parse --remainder before parse_args (unknown args pass through to PARSED_ARGS)
|
||||
REMAINDER_MODE=false
|
||||
_filtered_args=()
|
||||
for _arg in "$@"; do
|
||||
if [[ "$_arg" == "--remainder" ]]; then
|
||||
REMAINDER_MODE=true
|
||||
else
|
||||
_filtered_args+=("$_arg")
|
||||
fi
|
||||
done
|
||||
unset _arg
|
||||
|
||||
parse_args "${_filtered_args[@]}"
|
||||
unset _filtered_args
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
acquire_lock
|
||||
|
||||
if ! command -v docker &>/dev/null; then
|
||||
error "Docker command not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
detect_hosts
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Container Discovery ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$REMAINDER_MODE" == true ]]; then
|
||||
if [[ "${WEEKLY_REMAINING_UPDATES:-true}" != "true" ]]; then
|
||||
echo "WEEKLY_REMAINING_UPDATES=false — skipping remainder container updates"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
declare -A _exclude=()
|
||||
|
||||
# Daily containers — updated by docker_update.sh normal mode
|
||||
for _c in "${DAILY_RESTART_CONTAINERS[@]}"; do
|
||||
[[ -n "$_c" ]] && _exclude["$_c"]=1
|
||||
done
|
||||
|
||||
# Weekly sync-window containers (emby + critical-data) — updated inline by weekly_sync_maintenance.sh
|
||||
_weekly_str="${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-}"
|
||||
read -r -a _weekly_arr <<< "$_weekly_str"
|
||||
for _c in "${_weekly_arr[@]}"; do
|
||||
[[ -n "$_c" ]] && _exclude["$_c"]=1
|
||||
done
|
||||
unset _weekly_str _weekly_arr
|
||||
|
||||
# Weekly restart containers — restarted by docker_weekly_restart.sh after sync
|
||||
for _c in "${WEEKLY_RESTART_CONTAINERS[@]}"; do
|
||||
[[ -n "$_c" ]] && _exclude["$_c"]=1
|
||||
done
|
||||
|
||||
# Fallback coverage containers — owned by the remote server's update cycle.
|
||||
# This server runs them during fallback but should never update them independently.
|
||||
# Updating them here risks version divergence: if remote's writeback after handback
|
||||
# encounters data written by a newer version, it may not handle it correctly.
|
||||
for _tier in 1 2 3 4; do
|
||||
_tier_var="FALLBACK_${MY_ID}_COVERS_${REMOTE_ID}_TIER${_tier}"
|
||||
eval "_tier_arr=(\"\${${_tier_var}[@]:-}\")" 2>/dev/null
|
||||
for _c in "${_tier_arr[@]}"; do
|
||||
[[ -n "$_c" ]] && _exclude["$_c"]=1
|
||||
done
|
||||
done
|
||||
unset _tier _tier_var _tier_arr _c
|
||||
|
||||
mapfile -t _all_running < <(docker ps --format '{{.Names}}' | sort)
|
||||
TARGET_CONTAINERS=()
|
||||
for _c in "${_all_running[@]}"; do
|
||||
[[ -z "${_exclude[$_c]+x}" ]] && TARGET_CONTAINERS+=("$_c")
|
||||
done
|
||||
unset _all_running _exclude _c
|
||||
else
|
||||
if [[ "${DAILY_CONTAINER_UPDATES:-true}" != "true" ]]; then
|
||||
echo "DAILY_CONTAINER_UPDATES=false — skipping container updates"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ ${#DAILY_RESTART_CONTAINERS[@]} -eq 0 ]]; then
|
||||
warn "DAILY_RESTART_CONTAINERS is empty for $MY_ID — nothing to update"
|
||||
warn "Check HOST${MY_ID#HOST}_DAILY_RESTART_CONTAINERS in host*.conf"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
TARGET_CONTAINERS=("${DAILY_RESTART_CONTAINERS[@]}")
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_GEAR Mode: $([[ "$REMAINDER_MODE" == true ]] && echo "remainder" || echo "normal (daily)")"
|
||||
if [[ "$REMAINDER_MODE" == true ]]; then
|
||||
echo "$ICON_CONTAINERS Containers: ${#TARGET_CONTAINERS[@]} running (excluding daily, weekly sync, weekly restart, and fallback)"
|
||||
for _c in "${TARGET_CONTAINERS[@]}"; do echo " $_c"; done
|
||||
else
|
||||
echo "$ICON_CONTAINERS Containers: ${TARGET_CONTAINERS[*]}"
|
||||
echo "$ICON_GEAR Enabled: ${DAILY_CONTAINER_UPDATES:-true}"
|
||||
fi
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no images will be pulled"
|
||||
|
||||
if [[ ${#TARGET_CONTAINERS[@]} -eq 0 ]]; then
|
||||
echo "No containers to update"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Pull Updates ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
if [[ "$REMAINDER_MODE" == true ]]; then
|
||||
echo "━━━ $ICON_CONTAINERS Docker Update (remainder) — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo "$ICON_CONTAINERS Updating ${#TARGET_CONTAINERS[@]} container(s) (not in daily or weekly sync)"
|
||||
log "$ICON_CONTAINERS Remainder targets: ${TARGET_CONTAINERS[*]}"
|
||||
else
|
||||
echo "━━━ $ICON_CONTAINERS Docker Update — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
log "$ICON_CONTAINERS Containers: ${TARGET_CONTAINERS[*]}"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
START=$(date +%s)
|
||||
UPDATED=()
|
||||
UP_TO_DATE=()
|
||||
FAILED=()
|
||||
OLD_IMAGE_IDS=() # old image IDs to explicitly remove after rebuilds
|
||||
SKIPPED=()
|
||||
|
||||
for container in "${TARGET_CONTAINERS[@]}"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
log "━━━ $ICON_CONTAINERS $container ━━━"
|
||||
|
||||
if ! docker inspect "$container" &>/dev/null; then
|
||||
warn "$container — not found, skipping"
|
||||
SKIPPED+=("$container")
|
||||
continue
|
||||
fi
|
||||
|
||||
IMAGE=$(docker inspect --format='{{.Config.Image}}' "$container" 2>/dev/null)
|
||||
if [[ -z "$IMAGE" ]]; then
|
||||
warn "$container — could not determine image, skipping"
|
||||
SKIPPED+=("$container")
|
||||
continue
|
||||
fi
|
||||
|
||||
log "$container — image: $IMAGE"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would pull: $IMAGE"
|
||||
UPDATED+=("$container")
|
||||
continue
|
||||
fi
|
||||
|
||||
# Capture the image ID the container is currently running on, and the
|
||||
# image ID :latest points to before the pull. After pulling, we rebuild if
|
||||
# either a new digest landed OR the container is behind what :latest is now.
|
||||
CONTAINER_IMAGE_ID=$(docker inspect "$container" --format='{{.Image}}' 2>/dev/null || echo "")
|
||||
OLD_ID=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "")
|
||||
|
||||
log "$ICON_SYNC Pulling $IMAGE..."
|
||||
if [[ "$ENABLE_LOGGING" == "true" ]]; then
|
||||
docker pull "$IMAGE" 2>&1 | grep -E "^(Status:|Digest:|Error|error)" | sed 's/^/ /'
|
||||
_pull_rc=${PIPESTATUS[0]}
|
||||
else
|
||||
docker pull "$IMAGE" >/dev/null 2>&1
|
||||
_pull_rc=$?
|
||||
fi
|
||||
NEW_ID=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "")
|
||||
|
||||
if [[ $_pull_rc -eq 0 ]]; then
|
||||
_pull_new=$( [[ -n "$OLD_ID" && "$OLD_ID" != "$NEW_ID" ]] && echo true || echo false)
|
||||
_container_behind=$([[ -n "$CONTAINER_IMAGE_ID" && -n "$NEW_ID" && "$CONTAINER_IMAGE_ID" != "$NEW_ID" ]] && echo true || echo false)
|
||||
|
||||
if [[ "$_pull_new" == true || "$_container_behind" == true ]]; then
|
||||
[[ "$_pull_new" == true ]] && log "$ICON_DONE $container — new image (${OLD_ID:7:12} → ${NEW_ID:7:12})"
|
||||
[[ "$_container_behind" == true && "$_pull_new" == false ]] && log "$ICON_DONE $container — image already pulled, container behind (${CONTAINER_IMAGE_ID:7:12} → ${NEW_ID:7:12})"
|
||||
UPDATED+=("$container")
|
||||
OLD_IMAGE_IDS+=("$CONTAINER_IMAGE_ID")
|
||||
else
|
||||
log "$container — up to date (${NEW_ID:7:12})"
|
||||
UP_TO_DATE+=("$container")
|
||||
fi
|
||||
else
|
||||
warn "$container — pull failed ($IMAGE)"
|
||||
FAILED+=("$container")
|
||||
fi
|
||||
|
||||
done
|
||||
|
||||
# ── Recreate containers that received a new image ────────────────────────────
|
||||
# docker restart uses the image ID baked in at creation time — it never picks
|
||||
# up the new digest. rebuild_container reads the stored XML template, stops the
|
||||
# old container, recreates it (new image, same config), then prunes the old image.
|
||||
REBUILT=()
|
||||
REBUILD_FAILED=()
|
||||
if [[ ${#UPDATED[@]} -gt 0 ]]; then
|
||||
for container in "${UPDATED[@]}"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would rebuild $container from template"
|
||||
REBUILT+=("$container")
|
||||
continue
|
||||
fi
|
||||
log "$ICON_SYNC Rebuilding $container from template on new image..."
|
||||
if platform_rebuild_container "$container"; then
|
||||
log "$ICON_DONE $container rebuilt ✅"
|
||||
REBUILT+=("$container")
|
||||
else
|
||||
error "Failed to rebuild $container — will be picked up by docker_daily_restart.sh"
|
||||
notify "$container failed to rebuild after image update on $(hostname)" "Docker Update" "warning"
|
||||
REBUILD_FAILED+=("$container")
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
# ── Remove old images ────────────────────────────────────────────────────────
|
||||
# Explicitly rmi by the IDs captured before each pull. Tagged images are never
|
||||
# caught by dangling-only prune, so this is the only reliable cleanup path.
|
||||
# Fall through to dangling prune to catch any leftovers from other update paths.
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would remove ${#OLD_IMAGE_IDS[@]} old image(s) and prune dangling"
|
||||
PRUNED_SUMMARY="(dry run)"
|
||||
else
|
||||
for _old_id in "${OLD_IMAGE_IDS[@]}"; do
|
||||
docker rmi "$_old_id" >/dev/null 2>&1 || true
|
||||
done
|
||||
PRUNED_OUTPUT=$(docker image prune -f 2>&1)
|
||||
[[ "$ENABLE_LOGGING" == "true" ]] && echo "$PRUNED_OUTPUT" | sed 's/^/ /'
|
||||
PRUNED_SUMMARY=$(echo "$PRUNED_OUTPUT" | grep -E "^Total reclaimed" || echo "nothing reclaimed")
|
||||
fi
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$REMAINDER_MODE" == true ]]; then
|
||||
echo "━━━━━ $ICON_SUMMARY DOCKER UPDATE (REMAINDER) SUMMARY ━━━━━"
|
||||
else
|
||||
echo "━━━━━ $ICON_SUMMARY DOCKER UPDATE SUMMARY ━━━━━"
|
||||
fi
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
if [[ ${#UPDATED[@]} -gt 0 ]]; then
|
||||
echo "$ICON_DONE Updated: ${#UPDATED[@]}"
|
||||
log " ${UPDATED[*]}"
|
||||
fi
|
||||
[[ ${#REBUILT[@]} -gt 0 ]] && echo "$ICON_SYNC Rebuilt: ${#REBUILT[@]}"
|
||||
[[ ${#REBUILD_FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Rebuild fail:${REBUILD_FAILED[*]}"
|
||||
[[ ${#UP_TO_DATE[@]} -gt 0 ]] && log "$ICON_RUNNING Up to date: ${UP_TO_DATE[*]}"
|
||||
[[ ${#SKIPPED[@]} -gt 0 ]] && log "$ICON_WARN Skipped: ${SKIPPED[*]}"
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}"
|
||||
echo "$ICON_SYNC Pruned: ${PRUNED_SUMMARY:-none}"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no images pulled"
|
||||
elif [[ ${#FAILED[@]} -eq 0 ]]; then
|
||||
echo "$ICON_DONE Status: done ✅ — ${#UPDATED[@]} updated, ${#UP_TO_DATE[@]} current"
|
||||
else
|
||||
warn "Status: ${#FAILED[@]} pull(s) failed — restart will proceed with existing images"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
# Pull failures are non-fatal — restart proceeds regardless
|
||||
exit 0
|
||||
+415
@@ -0,0 +1,415 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ================================= Docker Update ==============================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Pulls the latest images for configured containers. Two modes: normal (daily)
|
||||
# and remainder (weekly).
|
||||
#
|
||||
# Normal mode is called by daily_sync_maintenance.sh before docker_daily_restart.sh.
|
||||
# Containers stay running during the pull — no extra downtime beyond what the
|
||||
# nightly restart already causes.
|
||||
#
|
||||
# Remainder mode is called by weekly_sync_maintenance.sh as the final update step.
|
||||
# It catches everything that normal mode and the weekly sync window did not already
|
||||
# update — derived automatically from docker ps, nothing to configure.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Normal mode (daily):
|
||||
# Targets DAILY_RESTART_CONTAINERS — same list used by docker_daily_restart.sh.
|
||||
# Pull → compare old vs new image ID → mark updated or already current.
|
||||
# docker_daily_restart.sh runs after — containers restart onto the fresh image.
|
||||
#
|
||||
# Remainder mode (weekly):
|
||||
# Targets all currently running containers NOT in:
|
||||
# DAILY_RESTART_CONTAINERS — already updated daily
|
||||
# emby + critical-data profiles — updated inline by the weekly sync window
|
||||
# WEEKLY_RESTART_CONTAINERS — restarted by docker_weekly_restart.sh after sync
|
||||
# FALLBACK_*_TIER* — owned by the remote server's update cycle
|
||||
# Pull → compare → prune dangling images.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# DESIGN PRINCIPLES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Single List
|
||||
# Normal mode reuses DAILY_RESTART_CONTAINERS rather than maintaining a
|
||||
# separate update list. Adding or removing a container from the restart list
|
||||
# automatically updates the image pull list — one change, both places.
|
||||
#
|
||||
# Version Ownership
|
||||
# Fallback containers are excluded from remainder mode. This server only runs
|
||||
# them during a fallback. The remote server owns their version — if remainder
|
||||
# updates them independently and a handback occurs, the remote's older image
|
||||
# may not handle data written by the newer version.
|
||||
#
|
||||
# State Respect
|
||||
# Stopped containers are never targeted. Pulling while stopped adds no value
|
||||
# and a stopped container was likely halted intentionally.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Lock Acquisition
|
||||
# Prevents concurrent execution via acquire_lock(). Safe to call from
|
||||
# maintenance scripts without risk of overlap.
|
||||
#
|
||||
# Host Detection
|
||||
# detect_hosts() identifies which server is running the script and aliases
|
||||
# HOST*_DAILY_RESTART_CONTAINERS to the correct host's values.
|
||||
#
|
||||
# Root Enforcement
|
||||
# Docker operations require root privileges.
|
||||
#
|
||||
# DAILY_CONTAINER_UPDATES Toggle
|
||||
# Normal mode exits cleanly when disabled. docker_daily_restart.sh still runs
|
||||
# regardless — update and restart are independent operations.
|
||||
#
|
||||
# Fallback Exclusion
|
||||
# Remainder mode excludes containers owned by the remote server's update cycle
|
||||
# to prevent version divergence across the fallback boundary.
|
||||
#
|
||||
# Running-Only Filter
|
||||
# Stopped containers excluded from remainder mode — intentionally down.
|
||||
#
|
||||
# Image ID Comparison
|
||||
# Containers not restarted unless their image actually changed. Pulls that
|
||||
# result in "already up to date" produce no restart.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# master.conf
|
||||
#
|
||||
# DAILY_CONTAINER_UPDATES
|
||||
# Enable or disable normal mode. docker_daily_restart.sh runs regardless.
|
||||
# (default: true)
|
||||
#
|
||||
# WEEKLY_REMAINING_UPDATES
|
||||
# Enable or disable remainder mode. (default: true)
|
||||
#
|
||||
# PROFILE_CRITICAL_CONTAINER_NAMES[emby|critical-data]
|
||||
# Container names for emby and critical-data profiles — excluded from
|
||||
# remainder mode (already updated by the weekly sync window)
|
||||
#
|
||||
# host*.conf
|
||||
#
|
||||
# HOST*_DAILY_RESTART_CONTAINERS
|
||||
# Containers updated in normal mode. Aliased by detect_hosts() →
|
||||
# DAILY_RESTART_CONTAINERS
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# docker_update.sh
|
||||
# Normal mode — pull latest images for DAILY_RESTART_CONTAINERS
|
||||
# Called by daily_sync_maintenance.sh before docker_daily_restart.sh
|
||||
#
|
||||
# docker_update.sh --remainder
|
||||
# Remainder mode — pull all running containers not in managed lists,
|
||||
# restart those that received updates, prune dangling images
|
||||
# Called by weekly_sync_maintenance.sh at the end of the Sunday window
|
||||
#
|
||||
# docker_update.sh --dry-run
|
||||
# Preview which containers would be pulled without making changes
|
||||
#
|
||||
# docker_update.sh --status
|
||||
# Show configuration and container list for current mode
|
||||
#
|
||||
# docker_update.sh --log
|
||||
# Verbose per-container pull and comparison output
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
# Pre-parse --remainder before parse_args (unknown args pass through to PARSED_ARGS)
|
||||
REMAINDER_MODE=false
|
||||
_filtered_args=()
|
||||
for _arg in "$@"; do
|
||||
if [[ "$_arg" == "--remainder" ]]; then
|
||||
REMAINDER_MODE=true
|
||||
else
|
||||
_filtered_args+=("$_arg")
|
||||
fi
|
||||
done
|
||||
unset _arg
|
||||
|
||||
parse_args "${_filtered_args[@]}"
|
||||
unset _filtered_args
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
acquire_lock
|
||||
|
||||
if ! command -v docker &>/dev/null; then
|
||||
error "Docker command not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
detect_hosts
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Container Discovery ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$REMAINDER_MODE" == true ]]; then
|
||||
if [[ "${WEEKLY_REMAINING_UPDATES:-true}" != "true" ]]; then
|
||||
echo "WEEKLY_REMAINING_UPDATES=false — skipping remainder container updates"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
declare -A _exclude=()
|
||||
|
||||
# Daily containers — updated by docker_update.sh normal mode
|
||||
for _c in "${DAILY_RESTART_CONTAINERS[@]}"; do
|
||||
[[ -n "$_c" ]] && _exclude["$_c"]=1
|
||||
done
|
||||
|
||||
# Weekly sync-window containers (emby + critical-data) — updated inline by weekly_sync_maintenance.sh
|
||||
_weekly_str="${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-}"
|
||||
read -r -a _weekly_arr <<< "$_weekly_str"
|
||||
for _c in "${_weekly_arr[@]}"; do
|
||||
[[ -n "$_c" ]] && _exclude["$_c"]=1
|
||||
done
|
||||
unset _weekly_str _weekly_arr
|
||||
|
||||
# Weekly restart containers — restarted by docker_weekly_restart.sh after sync
|
||||
for _c in "${WEEKLY_RESTART_CONTAINERS[@]}"; do
|
||||
[[ -n "$_c" ]] && _exclude["$_c"]=1
|
||||
done
|
||||
|
||||
# Fallback coverage containers — owned by the remote server's update cycle.
|
||||
# This server runs them during fallback but should never update them independently.
|
||||
# Updating them here risks version divergence: if remote's writeback after handback
|
||||
# encounters data written by a newer version, it may not handle it correctly.
|
||||
for _tier in 1 2 3 4; do
|
||||
_tier_var="FALLBACK_${MY_ID}_COVERS_${REMOTE_ID}_TIER${_tier}"
|
||||
eval "_tier_arr=(\"\${${_tier_var}[@]:-}\")" 2>/dev/null
|
||||
for _c in "${_tier_arr[@]}"; do
|
||||
[[ -n "$_c" ]] && _exclude["$_c"]=1
|
||||
done
|
||||
done
|
||||
unset _tier _tier_var _tier_arr _c
|
||||
|
||||
mapfile -t _all_running < <(docker ps --format '{{.Names}}' | sort)
|
||||
TARGET_CONTAINERS=()
|
||||
for _c in "${_all_running[@]}"; do
|
||||
[[ -z "${_exclude[$_c]+x}" ]] && TARGET_CONTAINERS+=("$_c")
|
||||
done
|
||||
unset _all_running _exclude _c
|
||||
else
|
||||
if [[ "${DAILY_CONTAINER_UPDATES:-true}" != "true" ]]; then
|
||||
echo "DAILY_CONTAINER_UPDATES=false — skipping container updates"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ ${#DAILY_RESTART_CONTAINERS[@]} -eq 0 ]]; then
|
||||
warn "DAILY_RESTART_CONTAINERS is empty for $MY_ID — nothing to update"
|
||||
warn "Check HOST${MY_ID#HOST}_DAILY_RESTART_CONTAINERS in host*.conf"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
TARGET_CONTAINERS=("${DAILY_RESTART_CONTAINERS[@]}")
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_GEAR Mode: $([[ "$REMAINDER_MODE" == true ]] && echo "remainder" || echo "normal (daily)")"
|
||||
if [[ "$REMAINDER_MODE" == true ]]; then
|
||||
echo "$ICON_CONTAINERS Containers: ${#TARGET_CONTAINERS[@]} running (excluding daily, weekly sync, weekly restart, and fallback)"
|
||||
for _c in "${TARGET_CONTAINERS[@]}"; do echo " $_c"; done
|
||||
else
|
||||
echo "$ICON_CONTAINERS Containers: ${TARGET_CONTAINERS[*]}"
|
||||
echo "$ICON_GEAR Enabled: ${DAILY_CONTAINER_UPDATES:-true}"
|
||||
fi
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no images will be pulled"
|
||||
|
||||
if [[ ${#TARGET_CONTAINERS[@]} -eq 0 ]]; then
|
||||
echo "No containers to update"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Pull Updates ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
if [[ "$REMAINDER_MODE" == true ]]; then
|
||||
echo "━━━ $ICON_CONTAINERS Docker Update (remainder) — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo "$ICON_CONTAINERS Updating ${#TARGET_CONTAINERS[@]} container(s) (not in daily or weekly sync)"
|
||||
log "$ICON_CONTAINERS Remainder targets: ${TARGET_CONTAINERS[*]}"
|
||||
else
|
||||
echo "━━━ $ICON_CONTAINERS Docker Update — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
log "$ICON_CONTAINERS Containers: ${TARGET_CONTAINERS[*]}"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
START=$(date +%s)
|
||||
UPDATED=()
|
||||
UP_TO_DATE=()
|
||||
FAILED=()
|
||||
OLD_IMAGE_IDS=() # old image IDs to explicitly remove after rebuilds
|
||||
SKIPPED=()
|
||||
|
||||
for container in "${TARGET_CONTAINERS[@]}"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
log "━━━ $ICON_CONTAINERS $container ━━━"
|
||||
|
||||
if ! docker inspect "$container" &>/dev/null; then
|
||||
warn "$container — not found, skipping"
|
||||
SKIPPED+=("$container")
|
||||
continue
|
||||
fi
|
||||
|
||||
IMAGE=$(docker inspect --format='{{.Config.Image}}' "$container" 2>/dev/null)
|
||||
if [[ -z "$IMAGE" ]]; then
|
||||
warn "$container — could not determine image, skipping"
|
||||
SKIPPED+=("$container")
|
||||
continue
|
||||
fi
|
||||
|
||||
log "$container — image: $IMAGE"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would pull: $IMAGE"
|
||||
UPDATED+=("$container")
|
||||
continue
|
||||
fi
|
||||
|
||||
# Capture the image ID the container is currently running on, and the
|
||||
# image ID :latest points to before the pull. After pulling, we rebuild if
|
||||
# either a new digest landed OR the container is behind what :latest is now.
|
||||
CONTAINER_IMAGE_ID=$(docker inspect "$container" --format='{{.Image}}' 2>/dev/null || echo "")
|
||||
OLD_ID=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "")
|
||||
|
||||
log "$ICON_SYNC Pulling $IMAGE..."
|
||||
if [[ "$ENABLE_LOGGING" == "true" ]]; then
|
||||
docker pull "$IMAGE" 2>&1 | grep -E "^(Status:|Digest:|Error|error)" | sed 's/^/ /'
|
||||
_pull_rc=${PIPESTATUS[0]}
|
||||
else
|
||||
docker pull "$IMAGE" >/dev/null 2>&1
|
||||
_pull_rc=$?
|
||||
fi
|
||||
NEW_ID=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "")
|
||||
|
||||
if [[ $_pull_rc -eq 0 ]]; then
|
||||
_pull_new=$( [[ -n "$OLD_ID" && "$OLD_ID" != "$NEW_ID" ]] && echo true || echo false)
|
||||
_container_behind=$([[ -n "$CONTAINER_IMAGE_ID" && -n "$NEW_ID" && "$CONTAINER_IMAGE_ID" != "$NEW_ID" ]] && echo true || echo false)
|
||||
|
||||
if [[ "$_pull_new" == true || "$_container_behind" == true ]]; then
|
||||
[[ "$_pull_new" == true ]] && log "$ICON_DONE $container — new image (${OLD_ID:7:12} → ${NEW_ID:7:12})"
|
||||
[[ "$_container_behind" == true && "$_pull_new" == false ]] && log "$ICON_DONE $container — image already pulled, container behind (${CONTAINER_IMAGE_ID:7:12} → ${NEW_ID:7:12})"
|
||||
UPDATED+=("$container")
|
||||
OLD_IMAGE_IDS+=("$CONTAINER_IMAGE_ID")
|
||||
else
|
||||
log "$container — up to date (${NEW_ID:7:12})"
|
||||
UP_TO_DATE+=("$container")
|
||||
fi
|
||||
else
|
||||
warn "$container — pull failed ($IMAGE)"
|
||||
FAILED+=("$container")
|
||||
fi
|
||||
|
||||
done
|
||||
|
||||
# ── Recreate containers that received a new image ────────────────────────────
|
||||
# docker restart uses the image ID baked in at creation time — it never picks
|
||||
# up the new digest. rebuild_container reads the stored XML template, stops the
|
||||
# old container, recreates it (new image, same config), then prunes the old image.
|
||||
REBUILT=()
|
||||
REBUILD_FAILED=()
|
||||
if [[ ${#UPDATED[@]} -gt 0 ]]; then
|
||||
for container in "${UPDATED[@]}"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would rebuild $container from template"
|
||||
REBUILT+=("$container")
|
||||
continue
|
||||
fi
|
||||
log "$ICON_SYNC Rebuilding $container from template on new image..."
|
||||
if platform_rebuild_container "$container"; then
|
||||
log "$ICON_DONE $container rebuilt ✅"
|
||||
REBUILT+=("$container")
|
||||
else
|
||||
error "Failed to rebuild $container — will be picked up by docker_daily_restart.sh"
|
||||
notify "$container failed to rebuild after image update on $(hostname)" "Docker Update" "warning"
|
||||
REBUILD_FAILED+=("$container")
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
# ── Remove old images ────────────────────────────────────────────────────────
|
||||
# Explicitly rmi by the IDs captured before each pull. Tagged images are never
|
||||
# caught by dangling-only prune, so this is the only reliable cleanup path.
|
||||
# Fall through to dangling prune to catch any leftovers from other update paths.
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would remove ${#OLD_IMAGE_IDS[@]} old image(s) and prune dangling"
|
||||
PRUNED_SUMMARY="(dry run)"
|
||||
else
|
||||
for _old_id in "${OLD_IMAGE_IDS[@]}"; do
|
||||
docker rmi "$_old_id" >/dev/null 2>&1 || true
|
||||
done
|
||||
PRUNED_OUTPUT=$(docker image prune -f 2>&1)
|
||||
[[ "$ENABLE_LOGGING" == "true" ]] && echo "$PRUNED_OUTPUT" | sed 's/^/ /'
|
||||
PRUNED_SUMMARY=$(echo "$PRUNED_OUTPUT" | grep -E "^Total reclaimed" || echo "nothing reclaimed")
|
||||
fi
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$REMAINDER_MODE" == true ]]; then
|
||||
echo "━━━━━ $ICON_SUMMARY DOCKER UPDATE (REMAINDER) SUMMARY ━━━━━"
|
||||
else
|
||||
echo "━━━━━ $ICON_SUMMARY DOCKER UPDATE SUMMARY ━━━━━"
|
||||
fi
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
if [[ ${#UPDATED[@]} -gt 0 ]]; then
|
||||
echo "$ICON_DONE Updated: ${#UPDATED[@]}"
|
||||
log " ${UPDATED[*]}"
|
||||
fi
|
||||
[[ ${#REBUILT[@]} -gt 0 ]] && echo "$ICON_SYNC Rebuilt: ${#REBUILT[@]}"
|
||||
[[ ${#REBUILD_FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Rebuild fail:${REBUILD_FAILED[*]}"
|
||||
[[ ${#UP_TO_DATE[@]} -gt 0 ]] && log "$ICON_RUNNING Up to date: ${UP_TO_DATE[*]}"
|
||||
[[ ${#SKIPPED[@]} -gt 0 ]] && log "$ICON_WARN Skipped: ${SKIPPED[*]}"
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}"
|
||||
echo "$ICON_SYNC Pruned: ${PRUNED_SUMMARY:-none}"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no images pulled"
|
||||
elif [[ ${#FAILED[@]} -eq 0 ]]; then
|
||||
echo "$ICON_DONE Status: done ✅ — ${#UPDATED[@]} updated, ${#UP_TO_DATE[@]} current"
|
||||
else
|
||||
warn "Status: ${#FAILED[@]} pull(s) failed — restart will proceed with existing images"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
# Pull failures are non-fatal — restart proceeds regardless
|
||||
exit 0
|
||||
+411
@@ -0,0 +1,411 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ================================= Docker Update ==============================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Pulls the latest images for configured containers. Two modes: normal (daily)
|
||||
# and remainder (weekly).
|
||||
#
|
||||
# Normal mode is called by daily_sync_maintenance.sh before docker_daily_restart.sh.
|
||||
# Containers stay running during the pull — no extra downtime beyond what the
|
||||
# nightly restart already causes.
|
||||
#
|
||||
# Remainder mode is called by weekly_sync_maintenance.sh as the final update step.
|
||||
# It catches everything that normal mode and the weekly sync window did not already
|
||||
# update — derived automatically from docker ps, nothing to configure.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Normal mode (daily):
|
||||
# Targets DAILY_RESTART_CONTAINERS — same list used by docker_daily_restart.sh.
|
||||
# Pull → compare old vs new image ID → mark updated or already current.
|
||||
# docker_daily_restart.sh runs after — containers restart onto the fresh image.
|
||||
#
|
||||
# Remainder mode (weekly):
|
||||
# Targets all currently running containers NOT in:
|
||||
# DAILY_RESTART_CONTAINERS — already updated daily
|
||||
# emby + critical-data profiles — updated inline by the weekly sync window
|
||||
# FALLBACK_*_TIER* — owned by the remote server's update cycle
|
||||
# WEEKLY_RESTART_CONTAINERS are included — docker_weekly_restart.sh restarts
|
||||
# them but doesn't pull images; remainder is the only place they get updated.
|
||||
# Pull → compare → prune dangling images.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# DESIGN PRINCIPLES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Single List
|
||||
# Normal mode reuses DAILY_RESTART_CONTAINERS rather than maintaining a
|
||||
# separate update list. Adding or removing a container from the restart list
|
||||
# automatically updates the image pull list — one change, both places.
|
||||
#
|
||||
# Version Ownership
|
||||
# Fallback containers are excluded from remainder mode. This server only runs
|
||||
# them during a fallback. The remote server owns their version — if remainder
|
||||
# updates them independently and a handback occurs, the remote's older image
|
||||
# may not handle data written by the newer version.
|
||||
#
|
||||
# State Respect
|
||||
# Stopped containers are never targeted. Pulling while stopped adds no value
|
||||
# and a stopped container was likely halted intentionally.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Lock Acquisition
|
||||
# Prevents concurrent execution via acquire_lock(). Safe to call from
|
||||
# maintenance scripts without risk of overlap.
|
||||
#
|
||||
# Host Detection
|
||||
# detect_hosts() identifies which server is running the script and aliases
|
||||
# HOST*_DAILY_RESTART_CONTAINERS to the correct host's values.
|
||||
#
|
||||
# Root Enforcement
|
||||
# Docker operations require root privileges.
|
||||
#
|
||||
# DAILY_CONTAINER_UPDATES Toggle
|
||||
# Normal mode exits cleanly when disabled. docker_daily_restart.sh still runs
|
||||
# regardless — update and restart are independent operations.
|
||||
#
|
||||
# Fallback Exclusion
|
||||
# Remainder mode excludes containers owned by the remote server's update cycle
|
||||
# to prevent version divergence across the fallback boundary.
|
||||
#
|
||||
# Running-Only Filter
|
||||
# Stopped containers excluded from remainder mode — intentionally down.
|
||||
#
|
||||
# Image ID Comparison
|
||||
# Containers not restarted unless their image actually changed. Pulls that
|
||||
# result in "already up to date" produce no restart.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# master.conf
|
||||
#
|
||||
# DAILY_CONTAINER_UPDATES
|
||||
# Enable or disable normal mode. docker_daily_restart.sh runs regardless.
|
||||
# (default: true)
|
||||
#
|
||||
# WEEKLY_REMAINING_UPDATES
|
||||
# Enable or disable remainder mode. (default: true)
|
||||
#
|
||||
# PROFILE_CRITICAL_CONTAINER_NAMES[emby|critical-data]
|
||||
# Container names for emby and critical-data profiles — excluded from
|
||||
# remainder mode (already updated by the weekly sync window)
|
||||
#
|
||||
# host*.conf
|
||||
#
|
||||
# HOST*_DAILY_RESTART_CONTAINERS
|
||||
# Containers updated in normal mode. Aliased by detect_hosts() →
|
||||
# DAILY_RESTART_CONTAINERS
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# docker_update.sh
|
||||
# Normal mode — pull latest images for DAILY_RESTART_CONTAINERS
|
||||
# Called by daily_sync_maintenance.sh before docker_daily_restart.sh
|
||||
#
|
||||
# docker_update.sh --remainder
|
||||
# Remainder mode — pull all running containers not in managed lists,
|
||||
# restart those that received updates, prune dangling images
|
||||
# Called by weekly_sync_maintenance.sh at the end of the Sunday window
|
||||
#
|
||||
# docker_update.sh --dry-run
|
||||
# Preview which containers would be pulled without making changes
|
||||
#
|
||||
# docker_update.sh --status
|
||||
# Show configuration and container list for current mode
|
||||
#
|
||||
# docker_update.sh --log
|
||||
# Verbose per-container pull and comparison output
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
# Pre-parse --remainder before parse_args (unknown args pass through to PARSED_ARGS)
|
||||
REMAINDER_MODE=false
|
||||
_filtered_args=()
|
||||
for _arg in "$@"; do
|
||||
if [[ "$_arg" == "--remainder" ]]; then
|
||||
REMAINDER_MODE=true
|
||||
else
|
||||
_filtered_args+=("$_arg")
|
||||
fi
|
||||
done
|
||||
unset _arg
|
||||
|
||||
parse_args "${_filtered_args[@]}"
|
||||
unset _filtered_args
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
acquire_lock
|
||||
|
||||
if ! command -v docker &>/dev/null; then
|
||||
error "Docker command not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
detect_hosts
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Container Discovery ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$REMAINDER_MODE" == true ]]; then
|
||||
if [[ "${WEEKLY_REMAINING_UPDATES:-true}" != "true" ]]; then
|
||||
echo "WEEKLY_REMAINING_UPDATES=false — skipping remainder container updates"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
declare -A _exclude=()
|
||||
|
||||
# Daily containers — updated by docker_update.sh normal mode
|
||||
for _c in "${DAILY_RESTART_CONTAINERS[@]}"; do
|
||||
[[ -n "$_c" ]] && _exclude["$_c"]=1
|
||||
done
|
||||
|
||||
# Weekly sync-window containers (emby + critical-data) — updated inline by weekly_sync_maintenance.sh
|
||||
_weekly_str="${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-}"
|
||||
read -r -a _weekly_arr <<< "$_weekly_str"
|
||||
for _c in "${_weekly_arr[@]}"; do
|
||||
[[ -n "$_c" ]] && _exclude["$_c"]=1
|
||||
done
|
||||
unset _weekly_str _weekly_arr
|
||||
|
||||
# Fallback coverage containers — owned by the remote server's update cycle.
|
||||
# This server runs them during fallback but should never update them independently.
|
||||
# Updating them here risks version divergence: if remote's writeback after handback
|
||||
# encounters data written by a newer version, it may not handle it correctly.
|
||||
for _tier in 1 2 3 4; do
|
||||
_tier_var="FALLBACK_${MY_ID}_COVERS_${REMOTE_ID}_TIER${_tier}"
|
||||
eval "_tier_arr=(\"\${${_tier_var}[@]:-}\")" 2>/dev/null
|
||||
for _c in "${_tier_arr[@]}"; do
|
||||
[[ -n "$_c" ]] && _exclude["$_c"]=1
|
||||
done
|
||||
done
|
||||
unset _tier _tier_var _tier_arr _c
|
||||
|
||||
mapfile -t _all_running < <(docker ps --format '{{.Names}}' | sort)
|
||||
TARGET_CONTAINERS=()
|
||||
for _c in "${_all_running[@]}"; do
|
||||
[[ -z "${_exclude[$_c]+x}" ]] && TARGET_CONTAINERS+=("$_c")
|
||||
done
|
||||
unset _all_running _exclude _c
|
||||
else
|
||||
if [[ "${DAILY_CONTAINER_UPDATES:-true}" != "true" ]]; then
|
||||
echo "DAILY_CONTAINER_UPDATES=false — skipping container updates"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ ${#DAILY_RESTART_CONTAINERS[@]} -eq 0 ]]; then
|
||||
warn "DAILY_RESTART_CONTAINERS is empty for $MY_ID — nothing to update"
|
||||
warn "Check HOST${MY_ID#HOST}_DAILY_RESTART_CONTAINERS in host*.conf"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
TARGET_CONTAINERS=("${DAILY_RESTART_CONTAINERS[@]}")
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_GEAR Mode: $([[ "$REMAINDER_MODE" == true ]] && echo "remainder" || echo "normal (daily)")"
|
||||
if [[ "$REMAINDER_MODE" == true ]]; then
|
||||
echo "$ICON_CONTAINERS Containers: ${#TARGET_CONTAINERS[@]} running (excluding daily, weekly sync, and fallback)"
|
||||
for _c in "${TARGET_CONTAINERS[@]}"; do echo " $_c"; done
|
||||
else
|
||||
echo "$ICON_CONTAINERS Containers: ${TARGET_CONTAINERS[*]}"
|
||||
echo "$ICON_GEAR Enabled: ${DAILY_CONTAINER_UPDATES:-true}"
|
||||
fi
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no images will be pulled"
|
||||
|
||||
if [[ ${#TARGET_CONTAINERS[@]} -eq 0 ]]; then
|
||||
echo "No containers to update"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Pull Updates ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
if [[ "$REMAINDER_MODE" == true ]]; then
|
||||
echo "━━━ $ICON_CONTAINERS Docker Update (remainder) — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo "$ICON_CONTAINERS Updating ${#TARGET_CONTAINERS[@]} container(s) (not in daily or weekly sync)"
|
||||
log "$ICON_CONTAINERS Remainder targets: ${TARGET_CONTAINERS[*]}"
|
||||
else
|
||||
echo "━━━ $ICON_CONTAINERS Docker Update — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
log "$ICON_CONTAINERS Containers: ${TARGET_CONTAINERS[*]}"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
START=$(date +%s)
|
||||
UPDATED=()
|
||||
UP_TO_DATE=()
|
||||
FAILED=()
|
||||
OLD_IMAGE_IDS=() # old image IDs to explicitly remove after rebuilds
|
||||
SKIPPED=()
|
||||
|
||||
for container in "${TARGET_CONTAINERS[@]}"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
log "━━━ $ICON_CONTAINERS $container ━━━"
|
||||
|
||||
if ! docker inspect "$container" &>/dev/null; then
|
||||
warn "$container — not found, skipping"
|
||||
SKIPPED+=("$container")
|
||||
continue
|
||||
fi
|
||||
|
||||
IMAGE=$(docker inspect --format='{{.Config.Image}}' "$container" 2>/dev/null)
|
||||
if [[ -z "$IMAGE" ]]; then
|
||||
warn "$container — could not determine image, skipping"
|
||||
SKIPPED+=("$container")
|
||||
continue
|
||||
fi
|
||||
|
||||
log "$container — image: $IMAGE"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would pull: $IMAGE"
|
||||
UPDATED+=("$container")
|
||||
continue
|
||||
fi
|
||||
|
||||
# Capture the image ID the container is currently running on, and the
|
||||
# image ID :latest points to before the pull. After pulling, we rebuild if
|
||||
# either a new digest landed OR the container is behind what :latest is now.
|
||||
CONTAINER_IMAGE_ID=$(docker inspect "$container" --format='{{.Image}}' 2>/dev/null || echo "")
|
||||
OLD_ID=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "")
|
||||
|
||||
log "$ICON_SYNC Pulling $IMAGE..."
|
||||
if [[ "$ENABLE_LOGGING" == "true" ]]; then
|
||||
docker pull "$IMAGE" 2>&1 | grep -E "^(Status:|Digest:|Error|error)" | sed 's/^/ /'
|
||||
_pull_rc=${PIPESTATUS[0]}
|
||||
else
|
||||
docker pull "$IMAGE" >/dev/null 2>&1
|
||||
_pull_rc=$?
|
||||
fi
|
||||
NEW_ID=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "")
|
||||
|
||||
if [[ $_pull_rc -eq 0 ]]; then
|
||||
_pull_new=$( [[ -n "$OLD_ID" && "$OLD_ID" != "$NEW_ID" ]] && echo true || echo false)
|
||||
_container_behind=$([[ -n "$CONTAINER_IMAGE_ID" && -n "$NEW_ID" && "$CONTAINER_IMAGE_ID" != "$NEW_ID" ]] && echo true || echo false)
|
||||
|
||||
if [[ "$_pull_new" == true || "$_container_behind" == true ]]; then
|
||||
[[ "$_pull_new" == true ]] && log "$ICON_DONE $container — new image (${OLD_ID:7:12} → ${NEW_ID:7:12})"
|
||||
[[ "$_container_behind" == true && "$_pull_new" == false ]] && log "$ICON_DONE $container — image already pulled, container behind (${CONTAINER_IMAGE_ID:7:12} → ${NEW_ID:7:12})"
|
||||
UPDATED+=("$container")
|
||||
OLD_IMAGE_IDS+=("$CONTAINER_IMAGE_ID")
|
||||
else
|
||||
log "$container — up to date (${NEW_ID:7:12})"
|
||||
UP_TO_DATE+=("$container")
|
||||
fi
|
||||
else
|
||||
warn "$container — pull failed ($IMAGE)"
|
||||
FAILED+=("$container")
|
||||
fi
|
||||
|
||||
done
|
||||
|
||||
# ── Recreate containers that received a new image ────────────────────────────
|
||||
# docker restart uses the image ID baked in at creation time — it never picks
|
||||
# up the new digest. rebuild_container reads the stored XML template, stops the
|
||||
# old container, recreates it (new image, same config), then prunes the old image.
|
||||
REBUILT=()
|
||||
REBUILD_FAILED=()
|
||||
if [[ ${#UPDATED[@]} -gt 0 ]]; then
|
||||
for container in "${UPDATED[@]}"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would rebuild $container from template"
|
||||
REBUILT+=("$container")
|
||||
continue
|
||||
fi
|
||||
log "$ICON_SYNC Rebuilding $container from template on new image..."
|
||||
if platform_rebuild_container "$container"; then
|
||||
log "$ICON_DONE $container rebuilt ✅"
|
||||
REBUILT+=("$container")
|
||||
else
|
||||
error "Failed to rebuild $container — will be picked up by docker_daily_restart.sh"
|
||||
notify "$container failed to rebuild after image update on $(hostname)" "Docker Update" "warning"
|
||||
REBUILD_FAILED+=("$container")
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
# ── Remove old images ────────────────────────────────────────────────────────
|
||||
# Explicitly rmi by the IDs captured before each pull. Tagged images are never
|
||||
# caught by dangling-only prune, so this is the only reliable cleanup path.
|
||||
# Fall through to dangling prune to catch any leftovers from other update paths.
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would remove ${#OLD_IMAGE_IDS[@]} old image(s) and prune dangling"
|
||||
PRUNED_SUMMARY="(dry run)"
|
||||
else
|
||||
for _old_id in "${OLD_IMAGE_IDS[@]}"; do
|
||||
docker rmi "$_old_id" >/dev/null 2>&1 || true
|
||||
done
|
||||
PRUNED_OUTPUT=$(docker image prune -f 2>&1)
|
||||
[[ "$ENABLE_LOGGING" == "true" ]] && echo "$PRUNED_OUTPUT" | sed 's/^/ /'
|
||||
PRUNED_SUMMARY=$(echo "$PRUNED_OUTPUT" | grep -E "^Total reclaimed" || echo "nothing reclaimed")
|
||||
fi
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$REMAINDER_MODE" == true ]]; then
|
||||
echo "━━━━━ $ICON_SUMMARY DOCKER UPDATE (REMAINDER) SUMMARY ━━━━━"
|
||||
else
|
||||
echo "━━━━━ $ICON_SUMMARY DOCKER UPDATE SUMMARY ━━━━━"
|
||||
fi
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
if [[ ${#UPDATED[@]} -gt 0 ]]; then
|
||||
echo "$ICON_DONE Updated: ${#UPDATED[@]}"
|
||||
log " ${UPDATED[*]}"
|
||||
fi
|
||||
[[ ${#REBUILT[@]} -gt 0 ]] && echo "$ICON_SYNC Rebuilt: ${#REBUILT[@]}"
|
||||
[[ ${#REBUILD_FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Rebuild fail:${REBUILD_FAILED[*]}"
|
||||
[[ ${#UP_TO_DATE[@]} -gt 0 ]] && log "$ICON_RUNNING Up to date: ${UP_TO_DATE[*]}"
|
||||
[[ ${#SKIPPED[@]} -gt 0 ]] && log "$ICON_WARN Skipped: ${SKIPPED[*]}"
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}"
|
||||
echo "$ICON_SYNC Pruned: ${PRUNED_SUMMARY:-none}"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no images pulled"
|
||||
elif [[ ${#FAILED[@]} -eq 0 ]]; then
|
||||
echo "$ICON_DONE Status: done ✅ — ${#UPDATED[@]} updated, ${#UP_TO_DATE[@]} current"
|
||||
else
|
||||
warn "Status: ${#FAILED[@]} pull(s) failed — restart will proceed with existing images"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
# Pull failures are non-fatal — restart proceeds regardless
|
||||
exit 0
|
||||
+411
@@ -0,0 +1,411 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ================================= Docker Update ==============================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Pulls the latest images for configured containers. Two modes: normal (daily)
|
||||
# and remainder (weekly).
|
||||
#
|
||||
# Normal mode is called by daily_sync_maintenance.sh before docker_daily_restart.sh.
|
||||
# Containers stay running during the pull — no extra downtime beyond what the
|
||||
# nightly restart already causes.
|
||||
#
|
||||
# Remainder mode is called by weekly_sync_maintenance.sh as the final update step.
|
||||
# It catches everything that normal mode and the weekly sync window did not already
|
||||
# update — derived automatically from docker ps, nothing to configure.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Normal mode (daily):
|
||||
# Targets DAILY_RESTART_CONTAINERS — same list used by docker_daily_restart.sh.
|
||||
# Pull → compare old vs new image ID → mark updated or already current.
|
||||
# docker_daily_restart.sh runs after — containers restart onto the fresh image.
|
||||
#
|
||||
# Remainder mode (weekly):
|
||||
# Targets all currently running containers NOT in:
|
||||
# DAILY_RESTART_CONTAINERS — already updated daily
|
||||
# emby + critical-data profiles — updated inline by the weekly sync window
|
||||
# FALLBACK_*_TIER* — owned by the remote server's update cycle
|
||||
# WEEKLY_RESTART_CONTAINERS are included — docker_weekly_restart.sh restarts
|
||||
# them but doesn't pull images; remainder is the only place they get updated.
|
||||
# Pull → compare → prune dangling images.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# DESIGN PRINCIPLES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Single List
|
||||
# Normal mode reuses DAILY_RESTART_CONTAINERS rather than maintaining a
|
||||
# separate update list. Adding or removing a container from the restart list
|
||||
# automatically updates the image pull list — one change, both places.
|
||||
#
|
||||
# Version Ownership
|
||||
# Fallback containers are excluded from remainder mode. This server only runs
|
||||
# them during a fallback. The remote server owns their version — if remainder
|
||||
# updates them independently and a handback occurs, the remote's older image
|
||||
# may not handle data written by the newer version.
|
||||
#
|
||||
# State Respect
|
||||
# Stopped containers are never targeted. Pulling while stopped adds no value
|
||||
# and a stopped container was likely halted intentionally.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Lock Acquisition
|
||||
# Prevents concurrent execution via acquire_lock(). Safe to call from
|
||||
# maintenance scripts without risk of overlap.
|
||||
#
|
||||
# Host Detection
|
||||
# detect_hosts() identifies which server is running the script and aliases
|
||||
# HOST*_DAILY_RESTART_CONTAINERS to the correct host's values.
|
||||
#
|
||||
# Root Enforcement
|
||||
# Docker operations require root privileges.
|
||||
#
|
||||
# DAILY_CONTAINER_UPDATES Toggle
|
||||
# Normal mode exits cleanly when disabled. docker_daily_restart.sh still runs
|
||||
# regardless — update and restart are independent operations.
|
||||
#
|
||||
# Fallback Exclusion
|
||||
# Remainder mode excludes containers owned by the remote server's update cycle
|
||||
# to prevent version divergence across the fallback boundary.
|
||||
#
|
||||
# Running-Only Filter
|
||||
# Stopped containers excluded from remainder mode — intentionally down.
|
||||
#
|
||||
# Image ID Comparison
|
||||
# Containers not restarted unless their image actually changed. Pulls that
|
||||
# result in "already up to date" produce no restart.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# master.conf
|
||||
#
|
||||
# DAILY_CONTAINER_UPDATES
|
||||
# Enable or disable normal mode. docker_daily_restart.sh runs regardless.
|
||||
# (default: true)
|
||||
#
|
||||
# MONTHLY_REMAINING_UPDATES
|
||||
# Enable or disable remainder mode. (default: true)
|
||||
#
|
||||
# PROFILE_CRITICAL_CONTAINER_NAMES[emby|critical-data]
|
||||
# Container names for emby and critical-data profiles — excluded from
|
||||
# remainder mode (already updated by the weekly sync window)
|
||||
#
|
||||
# host*.conf
|
||||
#
|
||||
# HOST*_DAILY_RESTART_CONTAINERS
|
||||
# Containers updated in normal mode. Aliased by detect_hosts() →
|
||||
# DAILY_RESTART_CONTAINERS
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# docker_update.sh
|
||||
# Normal mode — pull latest images for DAILY_RESTART_CONTAINERS
|
||||
# Called by daily_sync_maintenance.sh before docker_daily_restart.sh
|
||||
#
|
||||
# docker_update.sh --remainder
|
||||
# Remainder mode — pull all running containers not in managed lists,
|
||||
# restart those that received updates, prune dangling images
|
||||
# Called by monthly_maintenance.sh
|
||||
#
|
||||
# docker_update.sh --dry-run
|
||||
# Preview which containers would be pulled without making changes
|
||||
#
|
||||
# docker_update.sh --status
|
||||
# Show configuration and container list for current mode
|
||||
#
|
||||
# docker_update.sh --log
|
||||
# Verbose per-container pull and comparison output
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
# Pre-parse --remainder before parse_args (unknown args pass through to PARSED_ARGS)
|
||||
REMAINDER_MODE=false
|
||||
_filtered_args=()
|
||||
for _arg in "$@"; do
|
||||
if [[ "$_arg" == "--remainder" ]]; then
|
||||
REMAINDER_MODE=true
|
||||
else
|
||||
_filtered_args+=("$_arg")
|
||||
fi
|
||||
done
|
||||
unset _arg
|
||||
|
||||
parse_args "${_filtered_args[@]}"
|
||||
unset _filtered_args
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
acquire_lock
|
||||
|
||||
if ! command -v docker &>/dev/null; then
|
||||
error "Docker command not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
detect_hosts
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Container Discovery ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$REMAINDER_MODE" == true ]]; then
|
||||
if [[ "${MONTHLY_REMAINING_UPDATES:-true}" != "true" ]]; then
|
||||
echo "MONTHLY_REMAINING_UPDATES=false — skipping remainder container updates"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
declare -A _exclude=()
|
||||
|
||||
# Daily containers — updated by docker_update.sh normal mode
|
||||
for _c in "${DAILY_RESTART_CONTAINERS[@]}"; do
|
||||
[[ -n "$_c" ]] && _exclude["$_c"]=1
|
||||
done
|
||||
|
||||
# Weekly sync-window containers (emby + critical-data) — updated inline by weekly_sync_maintenance.sh
|
||||
_weekly_str="${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-}"
|
||||
read -r -a _weekly_arr <<< "$_weekly_str"
|
||||
for _c in "${_weekly_arr[@]}"; do
|
||||
[[ -n "$_c" ]] && _exclude["$_c"]=1
|
||||
done
|
||||
unset _weekly_str _weekly_arr
|
||||
|
||||
# Fallback coverage containers — owned by the remote server's update cycle.
|
||||
# This server runs them during fallback but should never update them independently.
|
||||
# Updating them here risks version divergence: if remote's writeback after handback
|
||||
# encounters data written by a newer version, it may not handle it correctly.
|
||||
for _tier in 1 2 3 4; do
|
||||
_tier_var="FALLBACK_${MY_ID}_COVERS_${REMOTE_ID}_TIER${_tier}"
|
||||
eval "_tier_arr=(\"\${${_tier_var}[@]:-}\")" 2>/dev/null
|
||||
for _c in "${_tier_arr[@]}"; do
|
||||
[[ -n "$_c" ]] && _exclude["$_c"]=1
|
||||
done
|
||||
done
|
||||
unset _tier _tier_var _tier_arr _c
|
||||
|
||||
mapfile -t _all_running < <(docker ps --format '{{.Names}}' | sort)
|
||||
TARGET_CONTAINERS=()
|
||||
for _c in "${_all_running[@]}"; do
|
||||
[[ -z "${_exclude[$_c]+x}" ]] && TARGET_CONTAINERS+=("$_c")
|
||||
done
|
||||
unset _all_running _exclude _c
|
||||
else
|
||||
if [[ "${DAILY_CONTAINER_UPDATES:-true}" != "true" ]]; then
|
||||
echo "DAILY_CONTAINER_UPDATES=false — skipping container updates"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ ${#DAILY_RESTART_CONTAINERS[@]} -eq 0 ]]; then
|
||||
warn "DAILY_RESTART_CONTAINERS is empty for $MY_ID — nothing to update"
|
||||
warn "Check HOST${MY_ID#HOST}_DAILY_RESTART_CONTAINERS in host*.conf"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
TARGET_CONTAINERS=("${DAILY_RESTART_CONTAINERS[@]}")
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_GEAR Mode: $([[ "$REMAINDER_MODE" == true ]] && echo "remainder" || echo "normal (daily)")"
|
||||
if [[ "$REMAINDER_MODE" == true ]]; then
|
||||
echo "$ICON_CONTAINERS Containers: ${#TARGET_CONTAINERS[@]} running (excluding daily, weekly sync, and fallback)"
|
||||
for _c in "${TARGET_CONTAINERS[@]}"; do echo " $_c"; done
|
||||
else
|
||||
echo "$ICON_CONTAINERS Containers: ${TARGET_CONTAINERS[*]}"
|
||||
echo "$ICON_GEAR Enabled: ${DAILY_CONTAINER_UPDATES:-true}"
|
||||
fi
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no images will be pulled"
|
||||
|
||||
if [[ ${#TARGET_CONTAINERS[@]} -eq 0 ]]; then
|
||||
echo "No containers to update"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Pull Updates ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
if [[ "$REMAINDER_MODE" == true ]]; then
|
||||
echo "━━━ $ICON_CONTAINERS Docker Update (remainder) — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo "$ICON_CONTAINERS Updating ${#TARGET_CONTAINERS[@]} container(s) (not in daily or weekly sync)"
|
||||
log "$ICON_CONTAINERS Remainder targets: ${TARGET_CONTAINERS[*]}"
|
||||
else
|
||||
echo "━━━ $ICON_CONTAINERS Docker Update — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
log "$ICON_CONTAINERS Containers: ${TARGET_CONTAINERS[*]}"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
START=$(date +%s)
|
||||
UPDATED=()
|
||||
UP_TO_DATE=()
|
||||
FAILED=()
|
||||
OLD_IMAGE_IDS=() # old image IDs to explicitly remove after rebuilds
|
||||
SKIPPED=()
|
||||
|
||||
for container in "${TARGET_CONTAINERS[@]}"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
log "━━━ $ICON_CONTAINERS $container ━━━"
|
||||
|
||||
if ! docker inspect "$container" &>/dev/null; then
|
||||
warn "$container — not found, skipping"
|
||||
SKIPPED+=("$container")
|
||||
continue
|
||||
fi
|
||||
|
||||
IMAGE=$(docker inspect --format='{{.Config.Image}}' "$container" 2>/dev/null)
|
||||
if [[ -z "$IMAGE" ]]; then
|
||||
warn "$container — could not determine image, skipping"
|
||||
SKIPPED+=("$container")
|
||||
continue
|
||||
fi
|
||||
|
||||
log "$container — image: $IMAGE"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would pull: $IMAGE"
|
||||
UPDATED+=("$container")
|
||||
continue
|
||||
fi
|
||||
|
||||
# Capture the image ID the container is currently running on, and the
|
||||
# image ID :latest points to before the pull. After pulling, we rebuild if
|
||||
# either a new digest landed OR the container is behind what :latest is now.
|
||||
CONTAINER_IMAGE_ID=$(docker inspect "$container" --format='{{.Image}}' 2>/dev/null || echo "")
|
||||
OLD_ID=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "")
|
||||
|
||||
log "$ICON_SYNC Pulling $IMAGE..."
|
||||
if [[ "$ENABLE_LOGGING" == "true" ]]; then
|
||||
docker pull "$IMAGE" 2>&1 | grep -E "^(Status:|Digest:|Error|error)" | sed 's/^/ /'
|
||||
_pull_rc=${PIPESTATUS[0]}
|
||||
else
|
||||
docker pull "$IMAGE" >/dev/null 2>&1
|
||||
_pull_rc=$?
|
||||
fi
|
||||
NEW_ID=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "")
|
||||
|
||||
if [[ $_pull_rc -eq 0 ]]; then
|
||||
_pull_new=$( [[ -n "$OLD_ID" && "$OLD_ID" != "$NEW_ID" ]] && echo true || echo false)
|
||||
_container_behind=$([[ -n "$CONTAINER_IMAGE_ID" && -n "$NEW_ID" && "$CONTAINER_IMAGE_ID" != "$NEW_ID" ]] && echo true || echo false)
|
||||
|
||||
if [[ "$_pull_new" == true || "$_container_behind" == true ]]; then
|
||||
[[ "$_pull_new" == true ]] && log "$ICON_DONE $container — new image (${OLD_ID:7:12} → ${NEW_ID:7:12})"
|
||||
[[ "$_container_behind" == true && "$_pull_new" == false ]] && log "$ICON_DONE $container — image already pulled, container behind (${CONTAINER_IMAGE_ID:7:12} → ${NEW_ID:7:12})"
|
||||
UPDATED+=("$container")
|
||||
OLD_IMAGE_IDS+=("$CONTAINER_IMAGE_ID")
|
||||
else
|
||||
log "$container — up to date (${NEW_ID:7:12})"
|
||||
UP_TO_DATE+=("$container")
|
||||
fi
|
||||
else
|
||||
warn "$container — pull failed ($IMAGE)"
|
||||
FAILED+=("$container")
|
||||
fi
|
||||
|
||||
done
|
||||
|
||||
# ── Recreate containers that received a new image ────────────────────────────
|
||||
# docker restart uses the image ID baked in at creation time — it never picks
|
||||
# up the new digest. rebuild_container reads the stored XML template, stops the
|
||||
# old container, recreates it (new image, same config), then prunes the old image.
|
||||
REBUILT=()
|
||||
REBUILD_FAILED=()
|
||||
if [[ ${#UPDATED[@]} -gt 0 ]]; then
|
||||
for container in "${UPDATED[@]}"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would rebuild $container from template"
|
||||
REBUILT+=("$container")
|
||||
continue
|
||||
fi
|
||||
log "$ICON_SYNC Rebuilding $container from template on new image..."
|
||||
if platform_rebuild_container "$container"; then
|
||||
log "$ICON_DONE $container rebuilt ✅"
|
||||
REBUILT+=("$container")
|
||||
else
|
||||
error "Failed to rebuild $container — will be picked up by docker_daily_restart.sh"
|
||||
notify "$container failed to rebuild after image update on $(hostname)" "Docker Update" "warning"
|
||||
REBUILD_FAILED+=("$container")
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
# ── Remove old images ────────────────────────────────────────────────────────
|
||||
# Explicitly rmi by the IDs captured before each pull. Tagged images are never
|
||||
# caught by dangling-only prune, so this is the only reliable cleanup path.
|
||||
# Fall through to dangling prune to catch any leftovers from other update paths.
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would remove ${#OLD_IMAGE_IDS[@]} old image(s) and prune dangling"
|
||||
PRUNED_SUMMARY="(dry run)"
|
||||
else
|
||||
for _old_id in "${OLD_IMAGE_IDS[@]}"; do
|
||||
docker rmi "$_old_id" >/dev/null 2>&1 || true
|
||||
done
|
||||
PRUNED_OUTPUT=$(docker image prune -f 2>&1)
|
||||
[[ "$ENABLE_LOGGING" == "true" ]] && echo "$PRUNED_OUTPUT" | sed 's/^/ /'
|
||||
PRUNED_SUMMARY=$(echo "$PRUNED_OUTPUT" | grep -E "^Total reclaimed" || echo "nothing reclaimed")
|
||||
fi
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$REMAINDER_MODE" == true ]]; then
|
||||
echo "━━━━━ $ICON_SUMMARY DOCKER UPDATE (REMAINDER) SUMMARY ━━━━━"
|
||||
else
|
||||
echo "━━━━━ $ICON_SUMMARY DOCKER UPDATE SUMMARY ━━━━━"
|
||||
fi
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
if [[ ${#UPDATED[@]} -gt 0 ]]; then
|
||||
echo "$ICON_DONE Updated: ${#UPDATED[@]}"
|
||||
log " ${UPDATED[*]}"
|
||||
fi
|
||||
[[ ${#REBUILT[@]} -gt 0 ]] && echo "$ICON_SYNC Rebuilt: ${#REBUILT[@]}"
|
||||
[[ ${#REBUILD_FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Rebuild fail:${REBUILD_FAILED[*]}"
|
||||
[[ ${#UP_TO_DATE[@]} -gt 0 ]] && log "$ICON_RUNNING Up to date: ${UP_TO_DATE[*]}"
|
||||
[[ ${#SKIPPED[@]} -gt 0 ]] && log "$ICON_WARN Skipped: ${SKIPPED[*]}"
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}"
|
||||
echo "$ICON_SYNC Pruned: ${PRUNED_SUMMARY:-none}"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no images pulled"
|
||||
elif [[ ${#FAILED[@]} -eq 0 ]]; then
|
||||
echo "$ICON_DONE Status: done ✅ — ${#UPDATED[@]} updated, ${#UP_TO_DATE[@]} current"
|
||||
else
|
||||
warn "Status: ${#FAILED[@]} pull(s) failed — restart will proceed with existing images"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
# Pull failures are non-fatal — restart proceeds regardless
|
||||
exit 0
|
||||
+458
@@ -0,0 +1,458 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ================================= Docker Update ==============================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Pulls the latest images for configured containers. Three modes:
|
||||
#
|
||||
# Normal mode — called by daily_sync_maintenance.sh before docker_daily_restart.sh.
|
||||
# Containers stay running during the pull — no extra downtime beyond what the
|
||||
# nightly restart already causes.
|
||||
#
|
||||
# Weekly mode — called by weekly_sync_maintenance.sh via WEEKLY_MAINTENANCE_SCRIPTS
|
||||
# before docker_weekly_restart.sh. Pulls WEEKLY_RESTART_CONTAINERS so the
|
||||
# subsequent restart lands on the fresh image.
|
||||
#
|
||||
# Remainder mode — called by monthly_maintenance.sh. Catches everything not
|
||||
# already owned by daily or weekly — derived from docker ps, nothing to configure.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Normal mode (daily):
|
||||
# Targets DAILY_RESTART_CONTAINERS — same list used by docker_daily_restart.sh.
|
||||
# Pull → compare old vs new image ID → mark updated or already current.
|
||||
# docker_daily_restart.sh runs after — containers restart onto the fresh image.
|
||||
#
|
||||
# Weekly mode:
|
||||
# Targets WEEKLY_RESTART_CONTAINERS — same list used by docker_weekly_restart.sh.
|
||||
# Pull → compare → rebuild if changed.
|
||||
# docker_weekly_restart.sh runs after — containers restart onto the fresh image.
|
||||
#
|
||||
# Remainder mode (monthly):
|
||||
# Targets all currently running containers NOT in:
|
||||
# DAILY_RESTART_CONTAINERS — already updated daily
|
||||
# WEEKLY_RESTART_CONTAINERS — already updated weekly
|
||||
# emby + critical-data profiles — updated inline by the weekly sync window
|
||||
# FALLBACK_*_TIER* — owned by the remote server's update cycle
|
||||
# Pull → compare → prune dangling images.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# DESIGN PRINCIPLES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Single List
|
||||
# Normal mode reuses DAILY_RESTART_CONTAINERS rather than maintaining a
|
||||
# separate update list. Adding or removing a container from the restart list
|
||||
# automatically updates the image pull list — one change, both places.
|
||||
#
|
||||
# Version Ownership
|
||||
# Fallback containers are excluded from remainder mode. This server only runs
|
||||
# them during a fallback. The remote server owns their version — if remainder
|
||||
# updates them independently and a handback occurs, the remote's older image
|
||||
# may not handle data written by the newer version.
|
||||
#
|
||||
# State Respect
|
||||
# Stopped containers are never targeted. Pulling while stopped adds no value
|
||||
# and a stopped container was likely halted intentionally.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Lock Acquisition
|
||||
# Prevents concurrent execution via acquire_lock(). Safe to call from
|
||||
# maintenance scripts without risk of overlap.
|
||||
#
|
||||
# Host Detection
|
||||
# detect_hosts() identifies which server is running the script and aliases
|
||||
# HOST*_DAILY_RESTART_CONTAINERS to the correct host's values.
|
||||
#
|
||||
# Root Enforcement
|
||||
# Docker operations require root privileges.
|
||||
#
|
||||
# DAILY_CONTAINER_UPDATES / WEEKLY_CONTAINER_UPDATES Toggles
|
||||
# Each mode exits cleanly when disabled. Restart scripts run regardless —
|
||||
# update and restart are independent operations.
|
||||
#
|
||||
# Fallback Exclusion
|
||||
# Remainder mode excludes containers owned by the remote server's update cycle
|
||||
# to prevent version divergence across the fallback boundary.
|
||||
#
|
||||
# Running-Only Filter
|
||||
# Stopped containers excluded from remainder mode — intentionally down.
|
||||
#
|
||||
# Image ID Comparison
|
||||
# Containers not restarted unless their image actually changed. Pulls that
|
||||
# result in "already up to date" produce no restart.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# master.conf
|
||||
#
|
||||
# DAILY_CONTAINER_UPDATES
|
||||
# Enable or disable normal mode. docker_daily_restart.sh runs regardless.
|
||||
# (default: true)
|
||||
#
|
||||
# WEEKLY_CONTAINER_UPDATES
|
||||
# Enable or disable weekly mode. docker_weekly_restart.sh runs regardless.
|
||||
# (default: true)
|
||||
#
|
||||
# MONTHLY_REMAINING_UPDATES
|
||||
# Enable or disable remainder mode. (default: true)
|
||||
#
|
||||
# PROFILE_CRITICAL_CONTAINER_NAMES[emby|critical-data]
|
||||
# Container names for emby and critical-data profiles — excluded from
|
||||
# remainder mode (already updated by the weekly sync window)
|
||||
#
|
||||
# host*.conf
|
||||
#
|
||||
# HOST*_DAILY_RESTART_CONTAINERS
|
||||
# Containers updated in normal mode. Aliased by detect_hosts() →
|
||||
# DAILY_RESTART_CONTAINERS
|
||||
#
|
||||
# HOST*_WEEKLY_RESTART_CONTAINERS
|
||||
# Containers updated in weekly mode. Aliased by detect_hosts() →
|
||||
# WEEKLY_RESTART_CONTAINERS
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# docker_update.sh
|
||||
# Normal mode — pull latest images for DAILY_RESTART_CONTAINERS
|
||||
# Called by daily_sync_maintenance.sh before docker_daily_restart.sh
|
||||
#
|
||||
# docker_update.sh --weekly
|
||||
# Weekly mode — pull latest images for WEEKLY_RESTART_CONTAINERS
|
||||
# Called by weekly_sync_maintenance.sh (WEEKLY_MAINTENANCE_SCRIPTS) before docker_weekly_restart.sh
|
||||
#
|
||||
# docker_update.sh --remainder
|
||||
# Remainder mode — pull all running containers not in managed lists,
|
||||
# restart those that received updates, prune dangling images
|
||||
# Called by monthly_maintenance.sh
|
||||
#
|
||||
# docker_update.sh --dry-run
|
||||
# Preview which containers would be pulled without making changes
|
||||
#
|
||||
# docker_update.sh --status
|
||||
# Show configuration and container list for current mode
|
||||
#
|
||||
# docker_update.sh --log
|
||||
# Verbose per-container pull and comparison output
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
# Pre-parse mode flags before parse_args (unknown args pass through to PARSED_ARGS)
|
||||
REMAINDER_MODE=false
|
||||
WEEKLY_MODE=false
|
||||
_filtered_args=()
|
||||
for _arg in "$@"; do
|
||||
case "$_arg" in
|
||||
--remainder) REMAINDER_MODE=true ;;
|
||||
--weekly) WEEKLY_MODE=true ;;
|
||||
*) _filtered_args+=("$_arg") ;;
|
||||
esac
|
||||
done
|
||||
unset _arg
|
||||
|
||||
parse_args "${_filtered_args[@]}"
|
||||
unset _filtered_args
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
acquire_lock
|
||||
|
||||
if ! command -v docker &>/dev/null; then
|
||||
error "Docker command not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
detect_hosts
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Container Discovery ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$REMAINDER_MODE" == true ]]; then
|
||||
if [[ "${MONTHLY_REMAINING_UPDATES:-true}" != "true" ]]; then
|
||||
echo "MONTHLY_REMAINING_UPDATES=false — skipping remainder container updates"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
declare -A _exclude=()
|
||||
|
||||
# Daily containers — updated by docker_update.sh normal mode
|
||||
for _c in "${DAILY_RESTART_CONTAINERS[@]}"; do
|
||||
[[ -n "$_c" ]] && _exclude["$_c"]=1
|
||||
done
|
||||
|
||||
# Weekly containers — updated by docker_update.sh --weekly
|
||||
for _c in "${WEEKLY_RESTART_CONTAINERS[@]}"; do
|
||||
[[ -n "$_c" ]] && _exclude["$_c"]=1
|
||||
done
|
||||
|
||||
# Weekly sync-window containers (emby + critical-data) — updated inline by weekly_sync_maintenance.sh
|
||||
_weekly_str="${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-}"
|
||||
read -r -a _weekly_arr <<< "$_weekly_str"
|
||||
for _c in "${_weekly_arr[@]}"; do
|
||||
[[ -n "$_c" ]] && _exclude["$_c"]=1
|
||||
done
|
||||
unset _weekly_str _weekly_arr
|
||||
|
||||
# Fallback coverage containers — owned by the remote server's update cycle.
|
||||
# This server runs them during fallback but should never update them independently.
|
||||
# Updating them here risks version divergence: if remote's writeback after handback
|
||||
# encounters data written by a newer version, it may not handle it correctly.
|
||||
for _tier in 1 2 3 4; do
|
||||
_tier_var="FALLBACK_${MY_ID}_COVERS_${REMOTE_ID}_TIER${_tier}"
|
||||
eval "_tier_arr=(\"\${${_tier_var}[@]:-}\")" 2>/dev/null
|
||||
for _c in "${_tier_arr[@]}"; do
|
||||
[[ -n "$_c" ]] && _exclude["$_c"]=1
|
||||
done
|
||||
done
|
||||
unset _tier _tier_var _tier_arr _c
|
||||
|
||||
mapfile -t _all_running < <(docker ps --format '{{.Names}}' | sort)
|
||||
TARGET_CONTAINERS=()
|
||||
for _c in "${_all_running[@]}"; do
|
||||
[[ -z "${_exclude[$_c]+x}" ]] && TARGET_CONTAINERS+=("$_c")
|
||||
done
|
||||
unset _all_running _exclude _c
|
||||
elif [[ "$WEEKLY_MODE" == true ]]; then
|
||||
if [[ "${WEEKLY_CONTAINER_UPDATES:-true}" != "true" ]]; then
|
||||
echo "WEEKLY_CONTAINER_UPDATES=false — skipping weekly container updates"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ ${#WEEKLY_RESTART_CONTAINERS[@]} -eq 0 ]]; then
|
||||
warn "WEEKLY_RESTART_CONTAINERS is empty for $MY_ID — nothing to update"
|
||||
warn "Check HOST${MY_ID#HOST}_WEEKLY_RESTART_CONTAINERS in host*.conf"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
TARGET_CONTAINERS=("${WEEKLY_RESTART_CONTAINERS[@]}")
|
||||
else
|
||||
if [[ "${DAILY_CONTAINER_UPDATES:-true}" != "true" ]]; then
|
||||
echo "DAILY_CONTAINER_UPDATES=false — skipping container updates"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ ${#DAILY_RESTART_CONTAINERS[@]} -eq 0 ]]; then
|
||||
warn "DAILY_RESTART_CONTAINERS is empty for $MY_ID — nothing to update"
|
||||
warn "Check HOST${MY_ID#HOST}_DAILY_RESTART_CONTAINERS in host*.conf"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
TARGET_CONTAINERS=("${DAILY_RESTART_CONTAINERS[@]}")
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
if [[ "$REMAINDER_MODE" == true ]]; then
|
||||
echo "$ICON_GEAR Mode: remainder (monthly)"
|
||||
echo "$ICON_CONTAINERS Containers: ${#TARGET_CONTAINERS[@]} running (excluding daily, weekly, sync-window, fallback)"
|
||||
for _c in "${TARGET_CONTAINERS[@]}"; do echo " $_c"; done
|
||||
elif [[ "$WEEKLY_MODE" == true ]]; then
|
||||
echo "$ICON_GEAR Mode: weekly"
|
||||
echo "$ICON_CONTAINERS Containers: ${TARGET_CONTAINERS[*]}"
|
||||
echo "$ICON_GEAR Enabled: ${WEEKLY_CONTAINER_UPDATES:-true}"
|
||||
else
|
||||
echo "$ICON_GEAR Mode: normal (daily)"
|
||||
echo "$ICON_CONTAINERS Containers: ${TARGET_CONTAINERS[*]}"
|
||||
echo "$ICON_GEAR Enabled: ${DAILY_CONTAINER_UPDATES:-true}"
|
||||
fi
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no images will be pulled"
|
||||
|
||||
if [[ ${#TARGET_CONTAINERS[@]} -eq 0 ]]; then
|
||||
echo "No containers to update"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Pull Updates ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
if [[ "$REMAINDER_MODE" == true ]]; then
|
||||
echo "━━━ $ICON_CONTAINERS Docker Update (remainder) — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo "$ICON_CONTAINERS Updating ${#TARGET_CONTAINERS[@]} container(s) (not in daily, weekly, or sync window)"
|
||||
log "$ICON_CONTAINERS Remainder targets: ${TARGET_CONTAINERS[*]}"
|
||||
elif [[ "$WEEKLY_MODE" == true ]]; then
|
||||
echo "━━━ $ICON_CONTAINERS Docker Update (weekly) — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
log "$ICON_CONTAINERS Containers: ${TARGET_CONTAINERS[*]}"
|
||||
else
|
||||
echo "━━━ $ICON_CONTAINERS Docker Update — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
log "$ICON_CONTAINERS Containers: ${TARGET_CONTAINERS[*]}"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
START=$(date +%s)
|
||||
UPDATED=()
|
||||
UP_TO_DATE=()
|
||||
FAILED=()
|
||||
OLD_IMAGE_IDS=() # old image IDs to explicitly remove after rebuilds
|
||||
SKIPPED=()
|
||||
|
||||
for container in "${TARGET_CONTAINERS[@]}"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
log "━━━ $ICON_CONTAINERS $container ━━━"
|
||||
|
||||
if ! docker inspect "$container" &>/dev/null; then
|
||||
warn "$container — not found, skipping"
|
||||
SKIPPED+=("$container")
|
||||
continue
|
||||
fi
|
||||
|
||||
IMAGE=$(docker inspect --format='{{.Config.Image}}' "$container" 2>/dev/null)
|
||||
if [[ -z "$IMAGE" ]]; then
|
||||
warn "$container — could not determine image, skipping"
|
||||
SKIPPED+=("$container")
|
||||
continue
|
||||
fi
|
||||
|
||||
log "$container — image: $IMAGE"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would pull: $IMAGE"
|
||||
UPDATED+=("$container")
|
||||
continue
|
||||
fi
|
||||
|
||||
# Capture the image ID the container is currently running on, and the
|
||||
# image ID :latest points to before the pull. After pulling, we rebuild if
|
||||
# either a new digest landed OR the container is behind what :latest is now.
|
||||
CONTAINER_IMAGE_ID=$(docker inspect "$container" --format='{{.Image}}' 2>/dev/null || echo "")
|
||||
OLD_ID=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "")
|
||||
|
||||
log "$ICON_SYNC Pulling $IMAGE..."
|
||||
if [[ "$ENABLE_LOGGING" == "true" ]]; then
|
||||
docker pull "$IMAGE" 2>&1 | grep -E "^(Status:|Digest:|Error|error)" | sed 's/^/ /'
|
||||
_pull_rc=${PIPESTATUS[0]}
|
||||
else
|
||||
docker pull "$IMAGE" >/dev/null 2>&1
|
||||
_pull_rc=$?
|
||||
fi
|
||||
NEW_ID=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "")
|
||||
|
||||
if [[ $_pull_rc -eq 0 ]]; then
|
||||
_pull_new=$( [[ -n "$OLD_ID" && "$OLD_ID" != "$NEW_ID" ]] && echo true || echo false)
|
||||
_container_behind=$([[ -n "$CONTAINER_IMAGE_ID" && -n "$NEW_ID" && "$CONTAINER_IMAGE_ID" != "$NEW_ID" ]] && echo true || echo false)
|
||||
|
||||
if [[ "$_pull_new" == true || "$_container_behind" == true ]]; then
|
||||
[[ "$_pull_new" == true ]] && log "$ICON_DONE $container — new image (${OLD_ID:7:12} → ${NEW_ID:7:12})"
|
||||
[[ "$_container_behind" == true && "$_pull_new" == false ]] && log "$ICON_DONE $container — image already pulled, container behind (${CONTAINER_IMAGE_ID:7:12} → ${NEW_ID:7:12})"
|
||||
UPDATED+=("$container")
|
||||
OLD_IMAGE_IDS+=("$CONTAINER_IMAGE_ID")
|
||||
else
|
||||
log "$container — up to date (${NEW_ID:7:12})"
|
||||
UP_TO_DATE+=("$container")
|
||||
fi
|
||||
else
|
||||
warn "$container — pull failed ($IMAGE)"
|
||||
FAILED+=("$container")
|
||||
fi
|
||||
|
||||
done
|
||||
|
||||
# ── Recreate containers that received a new image ────────────────────────────
|
||||
# docker restart uses the image ID baked in at creation time — it never picks
|
||||
# up the new digest. rebuild_container reads the stored XML template, stops the
|
||||
# old container, recreates it (new image, same config), then prunes the old image.
|
||||
REBUILT=()
|
||||
REBUILD_FAILED=()
|
||||
if [[ ${#UPDATED[@]} -gt 0 ]]; then
|
||||
for container in "${UPDATED[@]}"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would rebuild $container from template"
|
||||
REBUILT+=("$container")
|
||||
continue
|
||||
fi
|
||||
log "$ICON_SYNC Rebuilding $container from template on new image..."
|
||||
if platform_rebuild_container "$container"; then
|
||||
log "$ICON_DONE $container rebuilt ✅"
|
||||
REBUILT+=("$container")
|
||||
else
|
||||
error "Failed to rebuild $container — will be picked up by docker_daily_restart.sh"
|
||||
notify "$container failed to rebuild after image update on $(hostname)" "Docker Update" "warning"
|
||||
REBUILD_FAILED+=("$container")
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
# ── Remove old images ────────────────────────────────────────────────────────
|
||||
# Explicitly rmi by the IDs captured before each pull. Tagged images are never
|
||||
# caught by dangling-only prune, so this is the only reliable cleanup path.
|
||||
# Fall through to dangling prune to catch any leftovers from other update paths.
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would remove ${#OLD_IMAGE_IDS[@]} old image(s) and prune dangling"
|
||||
PRUNED_SUMMARY="(dry run)"
|
||||
else
|
||||
for _old_id in "${OLD_IMAGE_IDS[@]}"; do
|
||||
docker rmi "$_old_id" >/dev/null 2>&1 || true
|
||||
done
|
||||
PRUNED_OUTPUT=$(docker image prune -f 2>&1)
|
||||
[[ "$ENABLE_LOGGING" == "true" ]] && echo "$PRUNED_OUTPUT" | sed 's/^/ /'
|
||||
PRUNED_SUMMARY=$(echo "$PRUNED_OUTPUT" | grep -E "^Total reclaimed" || echo "nothing reclaimed")
|
||||
fi
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$REMAINDER_MODE" == true ]]; then
|
||||
echo "━━━━━ $ICON_SUMMARY DOCKER UPDATE (REMAINDER) SUMMARY ━━━━━"
|
||||
elif [[ "$WEEKLY_MODE" == true ]]; then
|
||||
echo "━━━━━ $ICON_SUMMARY DOCKER UPDATE (WEEKLY) SUMMARY ━━━━━"
|
||||
else
|
||||
echo "━━━━━ $ICON_SUMMARY DOCKER UPDATE SUMMARY ━━━━━"
|
||||
fi
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
if [[ ${#UPDATED[@]} -gt 0 ]]; then
|
||||
echo "$ICON_DONE Updated: ${#UPDATED[@]}"
|
||||
log " ${UPDATED[*]}"
|
||||
fi
|
||||
[[ ${#REBUILT[@]} -gt 0 ]] && echo "$ICON_SYNC Rebuilt: ${#REBUILT[@]}"
|
||||
[[ ${#REBUILD_FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Rebuild fail:${REBUILD_FAILED[*]}"
|
||||
[[ ${#UP_TO_DATE[@]} -gt 0 ]] && log "$ICON_RUNNING Up to date: ${UP_TO_DATE[*]}"
|
||||
[[ ${#SKIPPED[@]} -gt 0 ]] && log "$ICON_WARN Skipped: ${SKIPPED[*]}"
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}"
|
||||
echo "$ICON_SYNC Pruned: ${PRUNED_SUMMARY:-none}"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no images pulled"
|
||||
elif [[ ${#FAILED[@]} -eq 0 ]]; then
|
||||
echo "$ICON_DONE Status: done ✅ — ${#UPDATED[@]} updated, ${#UP_TO_DATE[@]} current"
|
||||
else
|
||||
warn "Status: ${#FAILED[@]} pull(s) failed — restart will proceed with existing images"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
# Pull failures are non-fatal — restart proceeds regardless
|
||||
exit 0
|
||||
+313
@@ -0,0 +1,313 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# =============================== Emby → Lidarr Sync ===========================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# One-shot tool. Finds artists played on Emby that are not tracked in Lidarr
|
||||
# and adds them. No scoring — if you played it, Lidarr should monitor it.
|
||||
#
|
||||
# Intended as a bootstrap / catch-up tool, not a scheduled script. Run it once
|
||||
# after Lidarr is set up, or any time you suspect gaps between what you listen
|
||||
# to and what Lidarr monitors.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# FLOW
|
||||
# ==============================================================================================
|
||||
#
|
||||
# 1. Pull play completions from Emby activity log (all time, or --days N)
|
||||
# 2. Fetch current Lidarr artist library
|
||||
# 3. For each played artist not in Lidarr → add to Lidarr
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Root Required
|
||||
# acquire_lock requires root. Script exits cleanly if not root.
|
||||
#
|
||||
# Dry-Run Mode
|
||||
# --dry-run shows all artists that would be added without making any Lidarr API calls.
|
||||
# Always run first when closing the gap after a fresh Lidarr install or database wipe.
|
||||
#
|
||||
# Add-Only
|
||||
# Only adds artists to Lidarr. Items already tracked (by name) are skipped without
|
||||
# modification. Safe to run multiple times — the second run finds nothing to add.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION (host*.conf)
|
||||
# ==============================================================================================
|
||||
#
|
||||
# HOST*_LIDARR_URL / HOST*_LIDARR_API_KEY — Lidarr connection (aliased by detect_hosts)
|
||||
# HOST*_EMBY_URL / HOST*_EMBY_API_KEY — Emby connection (aliased by detect_hosts)
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# emby_to_lidarr_sync.sh — add all album artists not in Lidarr
|
||||
# emby_to_lidarr_sync.sh --dry-run — show what would be added, no changes
|
||||
# emby_to_lidarr_sync.sh --log — verbose output
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
acquire_lock
|
||||
|
||||
detect_hosts
|
||||
|
||||
if ! command -v jq >/dev/null 2>&1; then
|
||||
error "jq not installed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ -z "${EMBY_URL:-}" || -z "${EMBY_API_KEY:-}" ]]; then
|
||||
error "EMBY_URL / EMBY_API_KEY not configured — check host*.conf"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ -z "${LIDARR_URL:-}" || -z "${LIDARR_API_KEY:-}" ]]; then
|
||||
error "LIDARR_URL / LIDARR_API_KEY not configured — check host*.conf"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
log "$ICON_GEAR Config: emby=${EMBY_URL} lidarr=${LIDARR_URL}"
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no artists will be added to Lidarr"
|
||||
|
||||
# ==============================================================================================
|
||||
# ── API HELPERS ───────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
|
||||
_lidarr_get() {
|
||||
curl -sf --max-time 30 \
|
||||
-H "X-Api-Key: $LIDARR_API_KEY" \
|
||||
"${LIDARR_URL}/api/v1/${1}" 2>/dev/null
|
||||
}
|
||||
|
||||
_lidarr_lookup() {
|
||||
curl -sf --max-time 20 --get \
|
||||
--data-urlencode "term=$1" \
|
||||
-H "X-Api-Key: $LIDARR_API_KEY" \
|
||||
"${LIDARR_URL}/api/v1/artist/lookup" 2>/dev/null
|
||||
}
|
||||
|
||||
_lidarr_post() {
|
||||
curl -sf --max-time 20 -X POST \
|
||||
-H "X-Api-Key: $LIDARR_API_KEY" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "$1" \
|
||||
"${LIDARR_URL}/api/v1/artist" 2>/dev/null
|
||||
}
|
||||
|
||||
_lidarr_command() {
|
||||
curl -sf --max-time 20 -X POST \
|
||||
-H "X-Api-Key: $LIDARR_API_KEY" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "$1" \
|
||||
"${LIDARR_URL}/api/v1/command" 2>/dev/null
|
||||
}
|
||||
|
||||
_is_placeholder_artist() {
|
||||
local a="${1,,}"
|
||||
[[ "$a" =~ ^(va|various|various artists|unknown artist|unknown|soundtrack|original soundtrack|ost)$ ]]
|
||||
}
|
||||
|
||||
# Dirty tag: comma-list, feat./ft., ampersand join, or "Artist - Album" in the AlbumArtist field
|
||||
_is_dirty_artist() {
|
||||
local lower="${1,,}"
|
||||
[[ "$1" == *", "* ]] && return 0
|
||||
[[ "$lower" == *" feat."* ]] && return 0
|
||||
[[ "$lower" == *" ft."* ]] && return 0
|
||||
[[ "$1" == *" & "* ]] && return 0
|
||||
[[ "$lower" == *" vs "* ]] && return 0
|
||||
[[ "$1" == *" - "* ]] && return 0
|
||||
[[ "$1" == *"?"* ]] && return 0 # ASCII ? — encoding corruption
|
||||
[[ "$1" == *$'\xef\xbf\xbd'* ]] && return 0 # U+FFFD replacement character — encoding corruption
|
||||
return 1
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Fetch Emby Music Artist Library ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY Emby → Lidarr Sync — $(date '+%Y-%m-%d %H:%M:%S') ━━━━━"
|
||||
echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no artists will be added"
|
||||
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Emby Music Library ━━━"
|
||||
|
||||
# Query MusicAlbum items and extract AlbumArtists — gives primary album artists only,
|
||||
# not the full tag credits (guest features, songwriters, etc.) that MusicArtist returns
|
||||
EMBY_ALBUMS_JSON=$(emby_api "Items?IncludeItemTypes=MusicAlbum&Recursive=true&Fields=AlbumArtists&Limit=10000") || {
|
||||
error "Could not fetch Emby music albums"
|
||||
exit 1
|
||||
}
|
||||
|
||||
declare -A PLAYED_ARTISTS
|
||||
|
||||
while IFS= read -r artist; do
|
||||
[[ -z "$artist" || "$artist" == "null" ]] && continue
|
||||
_is_placeholder_artist "$artist" && continue
|
||||
_is_dirty_artist "$artist" && { log " $ICON_SKIP Skipping dirty tag: $artist"; continue; }
|
||||
PLAYED_ARTISTS["$artist"]=1
|
||||
done < <(echo "$EMBY_ALBUMS_JSON" | jq -r '.Items[] | .AlbumArtists[]?.Name' 2>/dev/null)
|
||||
|
||||
PLAYED_COUNT=${#PLAYED_ARTISTS[@]}
|
||||
log "$PLAYED_COUNT album artists in Emby library"
|
||||
|
||||
if [[ "$PLAYED_COUNT" -eq 0 ]]; then
|
||||
warn "No album artists found in Emby library"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Fetch Lidarr Library ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Lidarr Library ━━━"
|
||||
|
||||
LIDARR_ARTISTS_JSON=$(_lidarr_get "artist") || { error "Could not fetch Lidarr artists"; exit 1; }
|
||||
LIDARR_NAMES=$(echo "$LIDARR_ARTISTS_JSON" | jq -r '.[].artistName' 2>/dev/null)
|
||||
LIDARR_COUNT=$(echo "$LIDARR_NAMES" | grep -c . 2>/dev/null || echo 0)
|
||||
log "$LIDARR_COUNT artists already in Lidarr"
|
||||
|
||||
_in_lidarr() {
|
||||
echo "$LIDARR_NAMES" | grep -iq "^${1}$"
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Find Gaps ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Comparing Libraries ━━━"
|
||||
|
||||
MISSING=()
|
||||
ALREADY=0
|
||||
|
||||
for artist in "${!PLAYED_ARTISTS[@]}"; do
|
||||
if _in_lidarr "$artist"; then
|
||||
(( ALREADY++ ))
|
||||
log " $ICON_SKIP Already tracked: $artist"
|
||||
else
|
||||
MISSING+=("$artist")
|
||||
log " $ICON_WARN Not in Lidarr: $artist"
|
||||
fi
|
||||
done
|
||||
|
||||
IFS=$'\n' MISSING=($(printf '%s\n' "${MISSING[@]}" | sort))
|
||||
|
||||
echo " Already tracked: $ALREADY | Missing from Lidarr: ${#MISSING[@]}"
|
||||
|
||||
if [[ "${#MISSING[@]}" -eq 0 ]]; then
|
||||
echo "Lidarr already tracks everything played on Emby"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "━━━ $ICON_SUMMARY Artists to add (${#MISSING[@]}) ━━━"
|
||||
for a in "${MISSING[@]}"; do log " $a"; done
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN complete — run without --dry-run to add these artists"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Add to Lidarr ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Adding to Lidarr ━━━"
|
||||
|
||||
LIDARR_ROOT=$(_lidarr_get "rootfolder" | jq -r 'first(.[] | select(.accessible == true)) | .path' 2>/dev/null)
|
||||
if [[ -z "$LIDARR_ROOT" ]]; then error "Could not determine Lidarr root folder"; exit 1; fi
|
||||
|
||||
QUALITY_PROFILES=$(_lidarr_get "qualityprofile") || { error "Could not fetch quality profiles"; exit 1; }
|
||||
METADATA_PROFILES=$(_lidarr_get "metadataprofile") || { error "Could not fetch metadata profiles"; exit 1; }
|
||||
|
||||
DEFAULT_QUALITY_ID=$(echo "$QUALITY_PROFILES" | jq -r '.[0].id' 2>/dev/null)
|
||||
DEFAULT_METADATA_ID=$(echo "$METADATA_PROFILES" | jq -r '
|
||||
first(.[] | select(.name | test("Standard"; "i")) | .id) // .[0].id' 2>/dev/null)
|
||||
|
||||
log "Quality profile: $DEFAULT_QUALITY_ID | Metadata profile: $DEFAULT_METADATA_ID"
|
||||
|
||||
ADDED=0
|
||||
FAILED=0
|
||||
SKIPPED=0
|
||||
|
||||
for artist in "${MISSING[@]}"; do
|
||||
LOOKUP=$(_lidarr_lookup "$artist")
|
||||
|
||||
if [[ -z "$LOOKUP" ]] || echo "$LOOKUP" | jq -e '. == [] or . == null' >/dev/null 2>&1; then
|
||||
warn " $ICON_WARN No match in Lidarr lookup: $artist"
|
||||
(( FAILED++ ))
|
||||
continue
|
||||
fi
|
||||
|
||||
ARTIST_DATA=$(echo "$LOOKUP" | jq '.[0]' 2>/dev/null)
|
||||
MBID=$(echo "$ARTIST_DATA" | jq -r '.foreignArtistId // ""' 2>/dev/null)
|
||||
LIDARR_NAME=$(echo "$ARTIST_DATA" | jq -r '.artistName // ""' 2>/dev/null)
|
||||
|
||||
if [[ -z "$MBID" ]]; then
|
||||
warn " $ICON_WARN No MusicBrainz ID for: $artist"
|
||||
(( FAILED++ ))
|
||||
continue
|
||||
fi
|
||||
|
||||
PAYLOAD=$(echo "$ARTIST_DATA" | jq \
|
||||
--arg root "$LIDARR_ROOT" \
|
||||
--argjson qid "$DEFAULT_QUALITY_ID" \
|
||||
--argjson mid "$DEFAULT_METADATA_ID" \
|
||||
'. + {
|
||||
rootFolderPath: $root,
|
||||
qualityProfileId: $qid,
|
||||
metadataProfileId: $mid,
|
||||
monitored: true,
|
||||
addOptions: {
|
||||
monitor: "all",
|
||||
searchForMissingAlbums: false
|
||||
}
|
||||
}' 2>/dev/null)
|
||||
|
||||
RESULT=$(_lidarr_post "$PAYLOAD")
|
||||
|
||||
if echo "$RESULT" | jq -e '.id' >/dev/null 2>&1; then
|
||||
ARTIST_ID=$(echo "$RESULT" | jq -r '.id')
|
||||
_lidarr_command "{\"name\":\"ArtistSearch\",\"artistId\":${ARTIST_ID}}" >/dev/null
|
||||
log " $ICON_DONE Added: $LIDARR_NAME"
|
||||
(( ADDED++ ))
|
||||
else
|
||||
warn " $ICON_WARN Failed to add: $artist"
|
||||
log " $(echo "$RESULT" | head -c 200)"
|
||||
(( FAILED++ ))
|
||||
fi
|
||||
done
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY SYNC COMPLETE ━━━━━"
|
||||
echo " $ICON_DONE Added: $ADDED"
|
||||
[[ "$FAILED" -gt 0 ]] && echo " $ICON_WARN Failed: $FAILED"
|
||||
echo " $ICON_SKIP Already tracked: $ALREADY"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ "$ADDED" -gt 0 ]] && notify \
|
||||
"$ADDED artist(s) added to Lidarr from Emby play history on $(hostname)" \
|
||||
"Emby → Lidarr Sync" "normal"
|
||||
|
||||
exit 0
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
+400
@@ -0,0 +1,400 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ============================= Weekly Sync Maintenance ========================================
|
||||
# ==============================================================================================
|
||||
# Weekly maintenance window orchestrator — clean sync, container updates, weekly restarts.
|
||||
# Schedule: 30 2 * * 0 (Sunday 2:30am — before Sunday 7am coffee report)
|
||||
#
|
||||
# ── EXECUTION ORDER ───────────────────────────────────────────────────────────────────────────
|
||||
# 1. Stop local containers — Emby + auth stack stopped locally
|
||||
# 2. Stop remote containers — Emby + auth stack stopped remotely via SSH
|
||||
# 3. Pull updates locally — if WEEKLY_SYNC_UPDATES=true (zero extra downtime)
|
||||
# 4. Pull updates remotely — if WEEKLY_SYNC_UPDATES_REMOTE=true
|
||||
# 5. rsync WEEKLY_SYNC_SHARES — full clean mirror, containers stopped both sides
|
||||
# 6. Start remote containers — correct order, delayed start respected
|
||||
# 7. Start local containers — correct order, delayed start respected
|
||||
# 8. WEEKLY_MAINTENANCE_SCRIPTS — weekly restarts etc. (docker_weekly_restart.sh)
|
||||
# 9. docker_update.sh --remainder — update all containers not in daily or weekly sync window
|
||||
#
|
||||
# ── WHY WEEKLY NOT NIGHTLY FOR EMBY ──────────────────────────────────────────────────────────
|
||||
# Emby builds a warm image cache on HOST2 throughout the week.
|
||||
# Syncing nightly resets cache — cold loads every morning for users.
|
||||
# Weekly sync: cache stays warm 6 days, resets Sunday night while users sleep.
|
||||
# emby-fallback dirty sync covers watch states + library every 30min between weekly syncs.
|
||||
#
|
||||
# ── CONTAINER UPDATES ─────────────────────────────────────────────────────────────────────────
|
||||
# Containers already stopped for sync — updates pull at zero extra downtime.
|
||||
# Both servers start on identical image versions after the window completes.
|
||||
# Toggle: WEEKLY_SYNC_UPDATES / WEEKLY_SYNC_UPDATES_REMOTE in master.conf
|
||||
#
|
||||
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
|
||||
# detect_hosts() sets MY_ID — used in banner, summary, and notifications.
|
||||
# WEEKLY_SYNC_SHARES and WEEKLY_MAINTENANCE_SCRIPTS configured in master.conf.
|
||||
# Same script runs correctly on both servers.
|
||||
#
|
||||
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
|
||||
# Root check — stop/start containers, rsync require root
|
||||
# acquire_lock — prevents concurrent weekly windows
|
||||
# check_connectivity — verifies remote before any remote operations
|
||||
# check_remote_rootfs — aborts if remote rootfs nearly full
|
||||
# DOCKER_TIMEOUT — all docker calls protected
|
||||
# SSH_TIMEOUT — all SSH calls protected
|
||||
# platform_require_cmd — notify validated before use
|
||||
# Silent on success — runs weekly, only failures warrant notification
|
||||
#
|
||||
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||||
# WEEKLY_SYNC_SHARES — shares synced during window
|
||||
# WEEKLY_MAINTENANCE_SCRIPTS — scripts run after sync
|
||||
# WEEKLY_SYNC_UPDATES — toggle local container updates
|
||||
# WEEKLY_SYNC_UPDATES_REMOTE — toggle remote container updates
|
||||
# WEEKLY_RSYNC_ENABLED — enable/disable rsync section
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# weekly_sync_maintenance.sh — normal run
|
||||
# weekly_sync_maintenance.sh --dry-run — preview without stopping containers or syncing
|
||||
# weekly_sync_maintenance.sh --log — verbose per-share/per-job output
|
||||
# weekly_sync_maintenance.sh --status — show configuration and exit
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
RSYNC_SCRIPT="$SCRIPT_DIR/../Rsync/rsync.sh"
|
||||
SCRIPTS_ROOT="$SCRIPT_DIR/.."
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
DOCKER_TIMEOUT=30 # container stop/start needs longer than normal
|
||||
SSH_TIMEOUT=30 # remote pulls can be slow
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
acquire_lock
|
||||
|
||||
if ! command -v docker &>/dev/null; then
|
||||
error "Docker command not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
detect_hosts
|
||||
resolve_remote_ip
|
||||
|
||||
WINDOW_START=$(date +%s)
|
||||
PASS=()
|
||||
FAIL=()
|
||||
JOB_PASS=()
|
||||
JOB_FAIL=()
|
||||
|
||||
# Load container lists from profile config
|
||||
read -r -a MAINTENANCE_CONTAINERS <<< \
|
||||
"${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-}"
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be stopped, no sync, no updates"
|
||||
|
||||
# ── Helper — run a post-sync maintenance script ────────────────────────────────────────────────
|
||||
run_job() {
|
||||
local script_entry="$1"
|
||||
local extra_dry=""
|
||||
[[ "$DRY_RUN" == true ]] && extra_dry="--dry-run"
|
||||
|
||||
read -r -a script_args <<< "$script_entry"
|
||||
local script_path="$SCRIPTS_ROOT/${script_args[0]}"
|
||||
local script_name
|
||||
script_name=$(basename "${script_args[0]}")
|
||||
local extra_args=("${script_args[@]:1}")
|
||||
|
||||
if [[ ! -f "$script_path" ]]; then
|
||||
error "$script_name — not found at $script_path"
|
||||
JOB_FAIL+=("$script_name")
|
||||
return 1
|
||||
fi
|
||||
|
||||
log "Running: $script_name ${extra_args[*]}"
|
||||
# shellcheck disable=SC2086
|
||||
if bash "$script_path" "${extra_args[@]}" $extra_dry; then
|
||||
log "$script_name — done ✅"
|
||||
JOB_PASS+=("$script_name ${extra_args[*]}")
|
||||
else
|
||||
error "$script_name — failed (exit $?)"
|
||||
JOB_FAIL+=("$script_name ${extra_args[*]}")
|
||||
fi
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY WEEKLY SYNC STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_NET Remote: $REMOTE_ID ($REMOTE_SERVER_NAME)"
|
||||
echo "$ICON_SYNC Rsync enabled: ${RSYNC_ENABLED:-true}"
|
||||
echo "$ICON_SYNC Weekly enabled: ${WEEKLY_RSYNC_ENABLED:-false}"
|
||||
echo "$ICON_GEAR Local updates: ${WEEKLY_SYNC_UPDATES:-false}"
|
||||
echo "$ICON_GEAR Remote updates: ${WEEKLY_SYNC_UPDATES_REMOTE:-false}"
|
||||
echo ""
|
||||
echo "━━━ Weekly Sync Shares ━━━"
|
||||
if [[ ${#WEEKLY_SYNC_SHARES[@]} -eq 0 ]]; then
|
||||
warn " No WEEKLY_SYNC_SHARES configured"
|
||||
else
|
||||
for share in "${WEEKLY_SYNC_SHARES[@]}"; do
|
||||
[[ -n "$share" ]] && echo " $ICON_SYNC $(basename "$share") ($share)"
|
||||
done
|
||||
fi
|
||||
echo ""
|
||||
echo "━━━ Weekly Maintenance Scripts ━━━"
|
||||
if [[ ${#WEEKLY_MAINTENANCE_SCRIPTS[@]} -eq 0 ]]; then
|
||||
echo " None configured"
|
||||
else
|
||||
for entry in "${WEEKLY_MAINTENANCE_SCRIPTS[@]}"; do
|
||||
[[ -n "$entry" ]] && echo " $ICON_GEAR ${entry##*/}"
|
||||
done
|
||||
fi
|
||||
echo ""
|
||||
echo "━━━ Containers (from profile config) ━━━"
|
||||
for c in "${MAINTENANCE_CONTAINERS[@]}"; do
|
||||
[[ -n "$c" ]] && echo " $ICON_CONTAINERS $c"
|
||||
done
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Pre-flight Checks ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Weekly Sync Maintenance — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo ""
|
||||
echo "━━━ $ICON_SHIELD Pre-flight ━━━"
|
||||
|
||||
check_connectivity
|
||||
check_remote_rootfs
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Stop Containers ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_STOP $ICON_CONTAINERS Stop Containers ━━━"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — containers will not be stopped"
|
||||
else
|
||||
# Load container lists for stop functions
|
||||
read -r -a CRITICAL_CONTAINER_NAMES <<< \
|
||||
"${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-}"
|
||||
read -r -a DELAYED_CONTAINERS <<< "${PROFILE_DELAYED_CONTAINERS[critical-data]:-}"
|
||||
CONTAINER_DELAY="${PROFILE_CONTAINER_DELAY[critical-data]:-15}"
|
||||
|
||||
stop_local_containers
|
||||
stop_containers
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Container Updates ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Container Updates ━━━"
|
||||
|
||||
if [[ "$WEEKLY_SYNC_UPDATES" == true ]]; then
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
for c in "${MAINTENANCE_CONTAINERS[@]}"; do
|
||||
[[ -n "$c" ]] && warn "DRY RUN — would pull: $c"
|
||||
done
|
||||
else
|
||||
log "Pulling local container updates..."
|
||||
for c in "${MAINTENANCE_CONTAINERS[@]}"; do
|
||||
[[ -z "$c" ]] && continue
|
||||
IMAGE=$(timeout "$DOCKER_TIMEOUT" docker inspect \
|
||||
"$c" --format '{{.Config.Image}}' 2>/dev/null)
|
||||
if [[ -z "$IMAGE" ]]; then
|
||||
log "$c — not found locally, skipping update"
|
||||
continue
|
||||
fi
|
||||
log "Pulling $IMAGE for $c..."
|
||||
if docker pull "$IMAGE" >/dev/null 2>&1; then
|
||||
log "$c — image updated ✅"
|
||||
else
|
||||
warn "$c — pull failed, will start on existing image"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
else
|
||||
echo "WEEKLY_SYNC_UPDATES=false — skipping local updates"
|
||||
fi
|
||||
|
||||
if [[ "$WEEKLY_SYNC_UPDATES_REMOTE" == true ]]; then
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would pull updates on $REMOTE_SERVER_NAME"
|
||||
else
|
||||
log "Pulling remote container updates on $REMOTE_SERVER_NAME..."
|
||||
for c in "${MAINTENANCE_CONTAINERS[@]}"; do
|
||||
[[ -z "$c" ]] && continue
|
||||
IMAGE=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" \
|
||||
root@"$REMOTE_SERVER" \
|
||||
"docker inspect $c --format '{{.Config.Image}}' 2>/dev/null" 2>/dev/null)
|
||||
if [[ -z "$IMAGE" ]]; then
|
||||
log "$c — not found on remote, skipping update"
|
||||
continue
|
||||
fi
|
||||
log "Pulling $IMAGE for $c on $REMOTE_SERVER_NAME..."
|
||||
if timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" \
|
||||
root@"$REMOTE_SERVER" \
|
||||
"docker pull $IMAGE" >/dev/null 2>&1; then
|
||||
log "$c — remote image updated ✅"
|
||||
else
|
||||
warn "$c — remote pull failed, will start on existing image"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
else
|
||||
echo "WEEKLY_SYNC_UPDATES_REMOTE=false — skipping remote updates"
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Critical Shares Sync ━━━
|
||||
# ==============================================================================================
|
||||
SHARE_COUNT=${#WEEKLY_SYNC_SHARES[@]}
|
||||
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Critical Shares Sync — $SHARE_COUNT share(s) ━━━"
|
||||
|
||||
SYNC_START=$(date +%s)
|
||||
JOB_NUM=0
|
||||
|
||||
if ! check_rsync_enabled "WEEKLY"; then
|
||||
warn "Weekly rsync disabled — skipping all $SHARE_COUNT sync job(s)"
|
||||
warn "Proceeding to container start and maintenance scripts..."
|
||||
elif [[ "$SHARE_COUNT" -eq 0 ]]; then
|
||||
warn "No WEEKLY_SYNC_SHARES configured — skipping sync"
|
||||
warn "Check WEEKLY_SYNC_SHARES in master.conf"
|
||||
else
|
||||
RSYNC_DRY=""
|
||||
[[ "$DRY_RUN" == true ]] && RSYNC_DRY="--dry-run"
|
||||
|
||||
for JOB in "${WEEKLY_SYNC_SHARES[@]}"; do
|
||||
(( JOB_NUM++ ))
|
||||
JOB_NAME=$(basename "$JOB")
|
||||
|
||||
echo ""
|
||||
echo "━━━ [$JOB_NUM/$SHARE_COUNT] $JOB_NAME ━━━"
|
||||
|
||||
JOB_START=$(date +%s)
|
||||
bash "$RSYNC_SCRIPT" "$JOB" $RSYNC_DRY
|
||||
EXIT_CODE=$?
|
||||
JOB_DUR=$(format_duration $(( $(date +%s) - JOB_START )))
|
||||
|
||||
if [[ "$EXIT_CODE" -eq 0 ]]; then
|
||||
PASS+=("$JOB_NAME")
|
||||
log "$JOB_NAME — done in $JOB_DUR ✅"
|
||||
else
|
||||
FAIL+=("$JOB_NAME")
|
||||
error "$JOB_NAME — failed after $JOB_DUR (exit $EXIT_CODE)"
|
||||
fi
|
||||
echo ""
|
||||
done
|
||||
fi
|
||||
|
||||
SYNC_END=$(date +%s)
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Start Containers ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_START $ICON_CONTAINERS Start Containers ━━━"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — containers will not be started"
|
||||
else
|
||||
start_containers
|
||||
start_local_containers
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Post-sync Jobs ━━━
|
||||
# ==============================================================================================
|
||||
if [[ ${#WEEKLY_MAINTENANCE_SCRIPTS[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Post-sync Jobs ━━━"
|
||||
for script_entry in "${WEEKLY_MAINTENANCE_SCRIPTS[@]}"; do
|
||||
[[ -z "$script_entry" ]] && continue
|
||||
echo ""
|
||||
run_job "$script_entry"
|
||||
done
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Remainder Container Updates ━━━
|
||||
# ==============================================================================================
|
||||
# Updates all running containers not already covered by daily or the weekly sync window.
|
||||
# Runs last — weekly sync-window containers are back up before this pulls their peers.
|
||||
echo ""
|
||||
echo "━━━ $ICON_CONTAINERS Remainder Container Updates ━━━"
|
||||
|
||||
DOCKER_UPDATE_SCRIPT="$SCRIPTS_ROOT/Docker_Essentials/docker_update.sh"
|
||||
if [[ ! -f "$DOCKER_UPDATE_SCRIPT" ]]; then
|
||||
warn "docker_update.sh not found — skipping remainder updates"
|
||||
JOB_FAIL+=("docker_update.sh --remainder")
|
||||
else
|
||||
_remainder_args=("--remainder")
|
||||
[[ "$DRY_RUN" == true ]] && _remainder_args+=("--dry-run")
|
||||
if bash "$DOCKER_UPDATE_SCRIPT" "${_remainder_args[@]}"; then
|
||||
echo "Remainder updates complete ✅"
|
||||
JOB_PASS+=("docker_update.sh --remainder")
|
||||
else
|
||||
warn "Remainder updates completed with errors"
|
||||
JOB_FAIL+=("docker_update.sh --remainder")
|
||||
fi
|
||||
unset _remainder_args
|
||||
fi
|
||||
|
||||
WINDOW_END=$(date +%s)
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY WEEKLY SYNC MAINTENANCE SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_TIME Window: $(date -d @"$WINDOW_START" '+%Y-%m-%d %H:%M:%S') → $(date -d @"$WINDOW_END" '+%H:%M:%S')"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( WINDOW_END - WINDOW_START )))"
|
||||
echo "$ICON_GEAR Updates: local=${WEEKLY_SYNC_UPDATES:-false} remote=${WEEKLY_SYNC_UPDATES_REMOTE:-false}"
|
||||
echo ""
|
||||
|
||||
echo "$ICON_SYNC Sync jobs ($SHARE_COUNT):"
|
||||
for job in "${PASS[@]}"; do echo " $ICON_DONE $job"; done
|
||||
for job in "${FAIL[@]}"; do echo " $ICON_ERROR $job"; done
|
||||
echo " Passed: ${#PASS[@]} Failed: ${#FAIL[@]}"
|
||||
|
||||
if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
echo "$ICON_GEAR Post-sync jobs:"
|
||||
for job in "${JOB_PASS[@]}"; do echo " $ICON_DONE $job"; done
|
||||
for job in "${JOB_FAIL[@]}"; do echo " $ICON_ERROR $job"; done
|
||||
fi
|
||||
|
||||
TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} ))
|
||||
|
||||
echo ""
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no changes made"
|
||||
elif [[ "$TOTAL_FAIL" -eq 0 ]]; then
|
||||
echo "$ICON_DONE Status: all complete ✅ — ${#PASS[@]} share(s) synced, ${#JOB_PASS[@]} job(s) run"
|
||||
else
|
||||
warn "Status: $TOTAL_FAIL failure(s)"
|
||||
notify "Weekly maintenance failed on $(hostname) ($MY_ID) — sync: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \
|
||||
"Weekly Maintenance" "warning"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ "$TOTAL_FAIL" -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
+438
@@ -0,0 +1,438 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ============================= Weekly Sync Maintenance ========================================
|
||||
# ==============================================================================================
|
||||
# Weekly maintenance window orchestrator — clean sync, container updates, weekly restarts.
|
||||
# Schedule: 30 2 * * 0 (Sunday 2:30am — before Sunday 7am coffee report)
|
||||
#
|
||||
# ── EXECUTION ORDER ───────────────────────────────────────────────────────────────────────────
|
||||
# 1. Stop local containers — Emby + auth stack stopped locally
|
||||
# 2. Stop remote containers — Emby + auth stack stopped remotely via SSH
|
||||
# 3. Pull updates locally — if WEEKLY_SYNC_UPDATES=true (zero extra downtime)
|
||||
# 4. Pull updates remotely — if WEEKLY_SYNC_UPDATES_REMOTE=true
|
||||
# 5. rsync WEEKLY_SYNC_SHARES — full clean mirror, containers stopped both sides
|
||||
# 6. Start remote containers — correct order, delayed start respected
|
||||
# 7. Start local containers — rebuild if new image pulled, docker start otherwise
|
||||
# 8. WEEKLY_MAINTENANCE_SCRIPTS — weekly restarts etc. (docker_weekly_restart.sh)
|
||||
# 9. docker_update.sh --remainder — update all containers not in daily or weekly sync window
|
||||
#
|
||||
# ── WHY WEEKLY NOT NIGHTLY FOR EMBY ──────────────────────────────────────────────────────────
|
||||
# Emby builds a warm image cache on HOST2 throughout the week.
|
||||
# Syncing nightly resets cache — cold loads every morning for users.
|
||||
# Weekly sync: cache stays warm 6 days, resets Sunday night while users sleep.
|
||||
# emby-fallback dirty sync covers watch states + library every 30min between weekly syncs.
|
||||
#
|
||||
# ── CONTAINER UPDATES ─────────────────────────────────────────────────────────────────────────
|
||||
# Containers already stopped for sync — updates pull at zero extra downtime.
|
||||
# Both servers start on identical image versions after the window completes.
|
||||
# Toggle: WEEKLY_SYNC_UPDATES / WEEKLY_SYNC_UPDATES_REMOTE in master.conf
|
||||
#
|
||||
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
|
||||
# detect_hosts() sets MY_ID — used in banner, summary, and notifications.
|
||||
# WEEKLY_SYNC_SHARES and WEEKLY_MAINTENANCE_SCRIPTS configured in master.conf.
|
||||
# Same script runs correctly on both servers.
|
||||
#
|
||||
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
|
||||
# Root check — stop/start containers, rsync require root
|
||||
# acquire_lock — prevents concurrent weekly windows
|
||||
# check_connectivity — verifies remote before any remote operations
|
||||
# check_remote_rootfs — aborts if remote rootfs nearly full
|
||||
# DOCKER_TIMEOUT — all docker calls protected
|
||||
# SSH_TIMEOUT — all SSH calls protected
|
||||
# platform_require_cmd — notify validated before use
|
||||
# Silent on success — runs weekly, only failures warrant notification
|
||||
#
|
||||
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||||
# WEEKLY_SYNC_SHARES — shares synced during window
|
||||
# WEEKLY_MAINTENANCE_SCRIPTS — scripts run after sync
|
||||
# WEEKLY_SYNC_UPDATES — toggle local container updates
|
||||
# WEEKLY_SYNC_UPDATES_REMOTE — toggle remote container updates
|
||||
# WEEKLY_RSYNC_ENABLED — enable/disable rsync section
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# weekly_sync_maintenance.sh — normal run
|
||||
# weekly_sync_maintenance.sh --dry-run — preview without stopping containers or syncing
|
||||
# weekly_sync_maintenance.sh --log — verbose per-share/per-job output
|
||||
# weekly_sync_maintenance.sh --status — show configuration and exit
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
RSYNC_SCRIPT="$SCRIPT_DIR/../Rsync/rsync.sh"
|
||||
SCRIPTS_ROOT="$SCRIPT_DIR/.."
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
DOCKER_TIMEOUT=30 # container stop/start needs longer than normal
|
||||
SSH_TIMEOUT=30 # remote pulls can be slow
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
acquire_lock
|
||||
|
||||
if ! command -v docker &>/dev/null; then
|
||||
error "Docker command not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
detect_hosts
|
||||
resolve_remote_ip
|
||||
|
||||
WINDOW_START=$(date +%s)
|
||||
PASS=()
|
||||
FAIL=()
|
||||
JOB_PASS=()
|
||||
JOB_FAIL=()
|
||||
|
||||
# Load container lists from profile config
|
||||
read -r -a MAINTENANCE_CONTAINERS <<< \
|
||||
"${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-}"
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be stopped, no sync, no updates"
|
||||
|
||||
# ── Helper — run a post-sync maintenance script ────────────────────────────────────────────────
|
||||
run_job() {
|
||||
local script_entry="$1"
|
||||
local extra_dry=""
|
||||
[[ "$DRY_RUN" == true ]] && extra_dry="--dry-run"
|
||||
|
||||
read -r -a script_args <<< "$script_entry"
|
||||
local script_path="$SCRIPTS_ROOT/${script_args[0]}"
|
||||
local script_name
|
||||
script_name=$(basename "${script_args[0]}")
|
||||
local extra_args=("${script_args[@]:1}")
|
||||
|
||||
if [[ ! -f "$script_path" ]]; then
|
||||
error "$script_name — not found at $script_path"
|
||||
JOB_FAIL+=("$script_name")
|
||||
return 1
|
||||
fi
|
||||
|
||||
log "Running: $script_name ${extra_args[*]}"
|
||||
# shellcheck disable=SC2086
|
||||
if bash "$script_path" "${extra_args[@]}" $extra_dry; then
|
||||
log "$script_name — done ✅"
|
||||
JOB_PASS+=("$script_name ${extra_args[*]}")
|
||||
else
|
||||
error "$script_name — failed (exit $?)"
|
||||
JOB_FAIL+=("$script_name ${extra_args[*]}")
|
||||
fi
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY WEEKLY SYNC STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_NET Remote: $REMOTE_ID ($REMOTE_SERVER_NAME)"
|
||||
echo "$ICON_SYNC Rsync enabled: ${RSYNC_ENABLED:-true}"
|
||||
echo "$ICON_SYNC Weekly enabled: ${WEEKLY_RSYNC_ENABLED:-false}"
|
||||
echo "$ICON_GEAR Local updates: ${WEEKLY_SYNC_UPDATES:-false}"
|
||||
echo "$ICON_GEAR Remote updates: ${WEEKLY_SYNC_UPDATES_REMOTE:-false}"
|
||||
echo ""
|
||||
echo "━━━ Weekly Sync Shares ━━━"
|
||||
if [[ ${#WEEKLY_SYNC_SHARES[@]} -eq 0 ]]; then
|
||||
warn " No WEEKLY_SYNC_SHARES configured"
|
||||
else
|
||||
for share in "${WEEKLY_SYNC_SHARES[@]}"; do
|
||||
[[ -n "$share" ]] && echo " $ICON_SYNC $(basename "$share") ($share)"
|
||||
done
|
||||
fi
|
||||
echo ""
|
||||
echo "━━━ Weekly Maintenance Scripts ━━━"
|
||||
if [[ ${#WEEKLY_MAINTENANCE_SCRIPTS[@]} -eq 0 ]]; then
|
||||
echo " None configured"
|
||||
else
|
||||
for entry in "${WEEKLY_MAINTENANCE_SCRIPTS[@]}"; do
|
||||
[[ -n "$entry" ]] && echo " $ICON_GEAR ${entry##*/}"
|
||||
done
|
||||
fi
|
||||
echo ""
|
||||
echo "━━━ Containers (from profile config) ━━━"
|
||||
for c in "${MAINTENANCE_CONTAINERS[@]}"; do
|
||||
[[ -n "$c" ]] && echo " $ICON_CONTAINERS $c"
|
||||
done
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Pre-flight Checks ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Weekly Sync Maintenance — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo ""
|
||||
echo "━━━ $ICON_SHIELD Pre-flight ━━━"
|
||||
|
||||
check_connectivity
|
||||
check_remote_rootfs
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Stop Containers ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_STOP $ICON_CONTAINERS Stop Containers ━━━"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — containers will not be stopped"
|
||||
else
|
||||
# Load container lists for stop functions
|
||||
read -r -a CRITICAL_CONTAINER_NAMES <<< \
|
||||
"${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-}"
|
||||
read -r -a DELAYED_CONTAINERS <<< "${PROFILE_DELAYED_CONTAINERS[critical-data]:-}"
|
||||
CONTAINER_DELAY="${PROFILE_CONTAINER_DELAY[critical-data]:-15}"
|
||||
|
||||
stop_local_containers
|
||||
stop_containers
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Container Updates ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Container Updates ━━━"
|
||||
|
||||
declare -A _weekly_needs_rebuild=()
|
||||
|
||||
if [[ "$WEEKLY_SYNC_UPDATES" == true ]]; then
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
for c in "${MAINTENANCE_CONTAINERS[@]}"; do
|
||||
[[ -n "$c" ]] && warn "DRY RUN — would pull: $c"
|
||||
done
|
||||
else
|
||||
log "Pulling local container updates..."
|
||||
for c in "${MAINTENANCE_CONTAINERS[@]}"; do
|
||||
[[ -z "$c" ]] && continue
|
||||
IMAGE=$(timeout "$DOCKER_TIMEOUT" docker inspect \
|
||||
"$c" --format '{{.Config.Image}}' 2>/dev/null)
|
||||
if [[ -z "$IMAGE" ]]; then
|
||||
log "$c — not found locally, skipping update"
|
||||
continue
|
||||
fi
|
||||
_old_id=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "")
|
||||
log "Pulling $IMAGE for $c..."
|
||||
if docker pull "$IMAGE" >/dev/null 2>&1; then
|
||||
_new_id=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "")
|
||||
if [[ -n "$_old_id" && "$_old_id" != "$_new_id" ]]; then
|
||||
log "$c — new image (${_old_id:7:12} → ${_new_id:7:12}) — will rebuild after sync"
|
||||
_weekly_needs_rebuild["$c"]=1
|
||||
else
|
||||
log "$c — already current"
|
||||
fi
|
||||
else
|
||||
warn "$c — pull failed, will start on existing image"
|
||||
fi
|
||||
done
|
||||
unset _old_id _new_id
|
||||
fi
|
||||
else
|
||||
echo "WEEKLY_SYNC_UPDATES=false — skipping local updates"
|
||||
fi
|
||||
|
||||
if [[ "$WEEKLY_SYNC_UPDATES_REMOTE" == true ]]; then
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would pull updates on $REMOTE_SERVER_NAME"
|
||||
else
|
||||
log "Pulling remote container updates on $REMOTE_SERVER_NAME..."
|
||||
for c in "${MAINTENANCE_CONTAINERS[@]}"; do
|
||||
[[ -z "$c" ]] && continue
|
||||
IMAGE=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" \
|
||||
root@"$REMOTE_SERVER" \
|
||||
"docker inspect $c --format '{{.Config.Image}}' 2>/dev/null" 2>/dev/null)
|
||||
if [[ -z "$IMAGE" ]]; then
|
||||
log "$c — not found on remote, skipping update"
|
||||
continue
|
||||
fi
|
||||
log "Pulling $IMAGE for $c on $REMOTE_SERVER_NAME..."
|
||||
if timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" \
|
||||
root@"$REMOTE_SERVER" \
|
||||
"docker pull $IMAGE" >/dev/null 2>&1; then
|
||||
log "$c — remote image updated ✅"
|
||||
else
|
||||
warn "$c — remote pull failed, will start on existing image"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
else
|
||||
echo "WEEKLY_SYNC_UPDATES_REMOTE=false — skipping remote updates"
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Critical Shares Sync ━━━
|
||||
# ==============================================================================================
|
||||
SHARE_COUNT=${#WEEKLY_SYNC_SHARES[@]}
|
||||
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Critical Shares Sync — $SHARE_COUNT share(s) ━━━"
|
||||
|
||||
SYNC_START=$(date +%s)
|
||||
JOB_NUM=0
|
||||
|
||||
if ! check_rsync_enabled "WEEKLY"; then
|
||||
warn "Weekly rsync disabled — skipping all $SHARE_COUNT sync job(s)"
|
||||
warn "Proceeding to container start and maintenance scripts..."
|
||||
elif [[ "$SHARE_COUNT" -eq 0 ]]; then
|
||||
warn "No WEEKLY_SYNC_SHARES configured — skipping sync"
|
||||
warn "Check WEEKLY_SYNC_SHARES in master.conf"
|
||||
else
|
||||
RSYNC_DRY=""
|
||||
[[ "$DRY_RUN" == true ]] && RSYNC_DRY="--dry-run"
|
||||
|
||||
for JOB in "${WEEKLY_SYNC_SHARES[@]}"; do
|
||||
(( JOB_NUM++ ))
|
||||
JOB_NAME=$(basename "$JOB")
|
||||
|
||||
echo ""
|
||||
echo "━━━ [$JOB_NUM/$SHARE_COUNT] $JOB_NAME ━━━"
|
||||
|
||||
JOB_START=$(date +%s)
|
||||
bash "$RSYNC_SCRIPT" "$JOB" $RSYNC_DRY
|
||||
EXIT_CODE=$?
|
||||
JOB_DUR=$(format_duration $(( $(date +%s) - JOB_START )))
|
||||
|
||||
if [[ "$EXIT_CODE" -eq 0 ]]; then
|
||||
PASS+=("$JOB_NAME")
|
||||
log "$JOB_NAME — done in $JOB_DUR ✅"
|
||||
else
|
||||
FAIL+=("$JOB_NAME")
|
||||
error "$JOB_NAME — failed after $JOB_DUR (exit $EXIT_CODE)"
|
||||
fi
|
||||
echo ""
|
||||
done
|
||||
fi
|
||||
|
||||
SYNC_END=$(date +%s)
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Start Containers ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_START $ICON_CONTAINERS Start Containers ━━━"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — containers will not be started"
|
||||
else
|
||||
start_containers
|
||||
|
||||
# Local start — rebuild containers that received a new image, docker start the rest
|
||||
if [[ ${#LOCAL_RUNNING_CONTAINERS[@]} -eq 0 ]]; then
|
||||
log "No local containers to restart."
|
||||
else
|
||||
for _c in "${LOCAL_RUNNING_CONTAINERS[@]}"; do
|
||||
[[ -z "$_c" ]] && continue
|
||||
_needs_delay=false
|
||||
for _d in "${DELAYED_CONTAINERS[@]}"; do
|
||||
[[ "$_c" == "$_d" ]] && _needs_delay=true && break
|
||||
done
|
||||
[[ "$_needs_delay" == true ]] && {
|
||||
info "Waiting ${CONTAINER_DELAY}s before starting $_c..."
|
||||
sleep "$CONTAINER_DELAY"
|
||||
}
|
||||
if [[ -n "${_weekly_needs_rebuild[$_c]:-}" ]]; then
|
||||
log "Rebuilding $_c on new image..."
|
||||
if platform_rebuild_container "$_c"; then
|
||||
log "$_c rebuilt on new image ✅"
|
||||
else
|
||||
warn "$_c rebuild failed — falling back to docker start"
|
||||
docker start "$_c" >/dev/null 2>&1 || error "Failed to start $_c"
|
||||
fi
|
||||
else
|
||||
docker start "$_c" >/dev/null 2>&1 && log "$_c started" || error "Failed to start $_c"
|
||||
fi
|
||||
done
|
||||
unset _c _d _needs_delay
|
||||
fi
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Post-sync Jobs ━━━
|
||||
# ==============================================================================================
|
||||
if [[ ${#WEEKLY_MAINTENANCE_SCRIPTS[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Post-sync Jobs ━━━"
|
||||
for script_entry in "${WEEKLY_MAINTENANCE_SCRIPTS[@]}"; do
|
||||
[[ -z "$script_entry" ]] && continue
|
||||
echo ""
|
||||
run_job "$script_entry"
|
||||
done
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Remainder Container Updates ━━━
|
||||
# ==============================================================================================
|
||||
# Updates all running containers not already covered by daily or the weekly sync window.
|
||||
# Runs last — weekly sync-window containers are back up before this pulls their peers.
|
||||
echo ""
|
||||
echo "━━━ $ICON_CONTAINERS Remainder Container Updates ━━━"
|
||||
|
||||
DOCKER_UPDATE_SCRIPT="$SCRIPTS_ROOT/Docker_Essentials/docker_update.sh"
|
||||
if [[ ! -f "$DOCKER_UPDATE_SCRIPT" ]]; then
|
||||
warn "docker_update.sh not found — skipping remainder updates"
|
||||
JOB_FAIL+=("docker_update.sh --remainder")
|
||||
else
|
||||
_remainder_args=("--remainder")
|
||||
[[ "$DRY_RUN" == true ]] && _remainder_args+=("--dry-run")
|
||||
if bash "$DOCKER_UPDATE_SCRIPT" "${_remainder_args[@]}"; then
|
||||
echo "Remainder updates complete ✅"
|
||||
JOB_PASS+=("docker_update.sh --remainder")
|
||||
else
|
||||
warn "Remainder updates completed with errors"
|
||||
JOB_FAIL+=("docker_update.sh --remainder")
|
||||
fi
|
||||
unset _remainder_args
|
||||
fi
|
||||
|
||||
WINDOW_END=$(date +%s)
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY WEEKLY SYNC MAINTENANCE SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_TIME Window: $(date -d @"$WINDOW_START" '+%Y-%m-%d %H:%M:%S') → $(date -d @"$WINDOW_END" '+%H:%M:%S')"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( WINDOW_END - WINDOW_START )))"
|
||||
echo "$ICON_GEAR Updates: local=${WEEKLY_SYNC_UPDATES:-false} remote=${WEEKLY_SYNC_UPDATES_REMOTE:-false}"
|
||||
echo ""
|
||||
|
||||
echo "$ICON_SYNC Sync jobs ($SHARE_COUNT):"
|
||||
for job in "${PASS[@]}"; do echo " $ICON_DONE $job"; done
|
||||
for job in "${FAIL[@]}"; do echo " $ICON_ERROR $job"; done
|
||||
echo " Passed: ${#PASS[@]} Failed: ${#FAIL[@]}"
|
||||
|
||||
if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
echo "$ICON_GEAR Post-sync jobs:"
|
||||
for job in "${JOB_PASS[@]}"; do echo " $ICON_DONE $job"; done
|
||||
for job in "${JOB_FAIL[@]}"; do echo " $ICON_ERROR $job"; done
|
||||
fi
|
||||
|
||||
TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} ))
|
||||
|
||||
echo ""
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no changes made"
|
||||
elif [[ "$TOTAL_FAIL" -eq 0 ]]; then
|
||||
echo "$ICON_DONE Status: all complete ✅ — ${#PASS[@]} share(s) synced, ${#JOB_PASS[@]} job(s) run"
|
||||
else
|
||||
warn "Status: $TOTAL_FAIL failure(s)"
|
||||
notify "Weekly maintenance failed on $(hostname) ($MY_ID) — sync: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \
|
||||
"Weekly Maintenance" "warning"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ "$TOTAL_FAIL" -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
+412
@@ -0,0 +1,412 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ============================= Weekly Sync Maintenance ========================================
|
||||
# ==============================================================================================
|
||||
# Weekly maintenance window orchestrator — clean sync, container updates, weekly restarts.
|
||||
# Schedule: 30 2 * * 0 (Sunday 2:30am — before Sunday 7am coffee report)
|
||||
#
|
||||
# ── EXECUTION ORDER ───────────────────────────────────────────────────────────────────────────
|
||||
# 1. Stop local containers — Emby + auth stack stopped locally
|
||||
# 2. Stop remote containers — Emby + auth stack stopped remotely via SSH
|
||||
# 3. Pull updates locally — if WEEKLY_SYNC_UPDATES=true (zero extra downtime)
|
||||
# 4. Pull updates remotely — if WEEKLY_SYNC_UPDATES_REMOTE=true
|
||||
# 5. rsync WEEKLY_SYNC_SHARES — full clean mirror, containers stopped both sides
|
||||
# 6. Start remote containers — correct order, delayed start respected
|
||||
# 7. Start local containers — rebuild if new image pulled, docker start otherwise
|
||||
# 8. WEEKLY_MAINTENANCE_SCRIPTS — weekly restarts etc. (docker_weekly_restart.sh)
|
||||
#
|
||||
# ── WHY WEEKLY NOT NIGHTLY FOR EMBY ──────────────────────────────────────────────────────────
|
||||
# Emby builds a warm image cache on HOST2 throughout the week.
|
||||
# Syncing nightly resets cache — cold loads every morning for users.
|
||||
# Weekly sync: cache stays warm 6 days, resets Sunday night while users sleep.
|
||||
# emby-fallback dirty sync covers watch states + library every 30min between weekly syncs.
|
||||
#
|
||||
# ── CONTAINER UPDATES ─────────────────────────────────────────────────────────────────────────
|
||||
# Containers already stopped for sync — updates pull at zero extra downtime.
|
||||
# Both servers start on identical image versions after the window completes.
|
||||
# Toggle: WEEKLY_SYNC_UPDATES / WEEKLY_SYNC_UPDATES_REMOTE in master.conf
|
||||
#
|
||||
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
|
||||
# detect_hosts() sets MY_ID — used in banner, summary, and notifications.
|
||||
# WEEKLY_SYNC_SHARES and WEEKLY_MAINTENANCE_SCRIPTS configured in master.conf.
|
||||
# Same script runs correctly on both servers.
|
||||
#
|
||||
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
|
||||
# Root check — stop/start containers, rsync require root
|
||||
# acquire_lock — prevents concurrent weekly windows
|
||||
# check_connectivity — verifies remote before any remote operations
|
||||
# check_remote_rootfs — aborts if remote rootfs nearly full
|
||||
# DOCKER_TIMEOUT — all docker calls protected
|
||||
# SSH_TIMEOUT — all SSH calls protected
|
||||
# platform_require_cmd — notify validated before use
|
||||
# Silent on success — runs weekly, only failures warrant notification
|
||||
#
|
||||
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||||
# WEEKLY_SYNC_SHARES — shares synced during window
|
||||
# WEEKLY_MAINTENANCE_SCRIPTS — scripts run after sync
|
||||
# WEEKLY_SYNC_UPDATES — toggle local container updates
|
||||
# WEEKLY_SYNC_UPDATES_REMOTE — toggle remote container updates
|
||||
# WEEKLY_RSYNC_ENABLED — enable/disable rsync section
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# weekly_sync_maintenance.sh — normal run
|
||||
# weekly_sync_maintenance.sh --dry-run — preview without stopping containers or syncing
|
||||
# weekly_sync_maintenance.sh --log — verbose per-share/per-job output
|
||||
# weekly_sync_maintenance.sh --status — show configuration and exit
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
RSYNC_SCRIPT="$SCRIPT_DIR/../Rsync/rsync.sh"
|
||||
SCRIPTS_ROOT="$SCRIPT_DIR/.."
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
DOCKER_TIMEOUT=30 # container stop/start needs longer than normal
|
||||
SSH_TIMEOUT=30 # remote pulls can be slow
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
acquire_lock
|
||||
|
||||
if ! command -v docker &>/dev/null; then
|
||||
error "Docker command not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
detect_hosts
|
||||
resolve_remote_ip
|
||||
|
||||
WINDOW_START=$(date +%s)
|
||||
PASS=()
|
||||
FAIL=()
|
||||
JOB_PASS=()
|
||||
JOB_FAIL=()
|
||||
|
||||
# Load container lists from profile config
|
||||
read -r -a MAINTENANCE_CONTAINERS <<< \
|
||||
"${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-}"
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be stopped, no sync, no updates"
|
||||
|
||||
# ── Helper — run a post-sync maintenance script ────────────────────────────────────────────────
|
||||
run_job() {
|
||||
local script_entry="$1"
|
||||
local extra_dry=""
|
||||
[[ "$DRY_RUN" == true ]] && extra_dry="--dry-run"
|
||||
|
||||
read -r -a script_args <<< "$script_entry"
|
||||
local script_path="$SCRIPTS_ROOT/${script_args[0]}"
|
||||
local script_name
|
||||
script_name=$(basename "${script_args[0]}")
|
||||
local extra_args=("${script_args[@]:1}")
|
||||
|
||||
if [[ ! -f "$script_path" ]]; then
|
||||
error "$script_name — not found at $script_path"
|
||||
JOB_FAIL+=("$script_name")
|
||||
return 1
|
||||
fi
|
||||
|
||||
log "Running: $script_name ${extra_args[*]}"
|
||||
# shellcheck disable=SC2086
|
||||
if bash "$script_path" "${extra_args[@]}" $extra_dry; then
|
||||
log "$script_name — done ✅"
|
||||
JOB_PASS+=("$script_name ${extra_args[*]}")
|
||||
else
|
||||
error "$script_name — failed (exit $?)"
|
||||
JOB_FAIL+=("$script_name ${extra_args[*]}")
|
||||
fi
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY WEEKLY SYNC STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_NET Remote: $REMOTE_ID ($REMOTE_SERVER_NAME)"
|
||||
echo "$ICON_SYNC Rsync enabled: ${RSYNC_ENABLED:-true}"
|
||||
echo "$ICON_SYNC Weekly enabled: ${WEEKLY_RSYNC_ENABLED:-false}"
|
||||
echo "$ICON_GEAR Local updates: ${WEEKLY_SYNC_UPDATES:-false}"
|
||||
echo "$ICON_GEAR Remote updates: ${WEEKLY_SYNC_UPDATES_REMOTE:-false}"
|
||||
echo ""
|
||||
echo "━━━ Weekly Sync Shares ━━━"
|
||||
if [[ ${#WEEKLY_SYNC_SHARES[@]} -eq 0 ]]; then
|
||||
warn " No WEEKLY_SYNC_SHARES configured"
|
||||
else
|
||||
for share in "${WEEKLY_SYNC_SHARES[@]}"; do
|
||||
[[ -n "$share" ]] && echo " $ICON_SYNC $(basename "$share") ($share)"
|
||||
done
|
||||
fi
|
||||
echo ""
|
||||
echo "━━━ Weekly Maintenance Scripts ━━━"
|
||||
if [[ ${#WEEKLY_MAINTENANCE_SCRIPTS[@]} -eq 0 ]]; then
|
||||
echo " None configured"
|
||||
else
|
||||
for entry in "${WEEKLY_MAINTENANCE_SCRIPTS[@]}"; do
|
||||
[[ -n "$entry" ]] && echo " $ICON_GEAR ${entry##*/}"
|
||||
done
|
||||
fi
|
||||
echo ""
|
||||
echo "━━━ Containers (from profile config) ━━━"
|
||||
for c in "${MAINTENANCE_CONTAINERS[@]}"; do
|
||||
[[ -n "$c" ]] && echo " $ICON_CONTAINERS $c"
|
||||
done
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Pre-flight Checks ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Weekly Sync Maintenance — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo ""
|
||||
echo "━━━ $ICON_SHIELD Pre-flight ━━━"
|
||||
|
||||
check_connectivity
|
||||
check_remote_rootfs
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Stop Containers ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_STOP $ICON_CONTAINERS Stop Containers ━━━"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — containers will not be stopped"
|
||||
else
|
||||
# Load container lists for stop functions
|
||||
read -r -a CRITICAL_CONTAINER_NAMES <<< \
|
||||
"${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-}"
|
||||
read -r -a DELAYED_CONTAINERS <<< "${PROFILE_DELAYED_CONTAINERS[critical-data]:-}"
|
||||
CONTAINER_DELAY="${PROFILE_CONTAINER_DELAY[critical-data]:-15}"
|
||||
|
||||
stop_local_containers
|
||||
stop_containers
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Container Updates ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Container Updates ━━━"
|
||||
|
||||
declare -A _weekly_needs_rebuild=()
|
||||
|
||||
if [[ "$WEEKLY_SYNC_UPDATES" == true ]]; then
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
for c in "${MAINTENANCE_CONTAINERS[@]}"; do
|
||||
[[ -n "$c" ]] && warn "DRY RUN — would pull: $c"
|
||||
done
|
||||
else
|
||||
log "Pulling local container updates..."
|
||||
for c in "${MAINTENANCE_CONTAINERS[@]}"; do
|
||||
[[ -z "$c" ]] && continue
|
||||
IMAGE=$(timeout "$DOCKER_TIMEOUT" docker inspect \
|
||||
"$c" --format '{{.Config.Image}}' 2>/dev/null)
|
||||
if [[ -z "$IMAGE" ]]; then
|
||||
log "$c — not found locally, skipping update"
|
||||
continue
|
||||
fi
|
||||
_old_id=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "")
|
||||
log "Pulling $IMAGE for $c..."
|
||||
if docker pull "$IMAGE" >/dev/null 2>&1; then
|
||||
_new_id=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "")
|
||||
if [[ -n "$_old_id" && "$_old_id" != "$_new_id" ]]; then
|
||||
log "$c — new image (${_old_id:7:12} → ${_new_id:7:12}) — will rebuild after sync"
|
||||
_weekly_needs_rebuild["$c"]=1
|
||||
else
|
||||
log "$c — already current"
|
||||
fi
|
||||
else
|
||||
warn "$c — pull failed, will start on existing image"
|
||||
fi
|
||||
done
|
||||
unset _old_id _new_id
|
||||
fi
|
||||
else
|
||||
echo "WEEKLY_SYNC_UPDATES=false — skipping local updates"
|
||||
fi
|
||||
|
||||
if [[ "$WEEKLY_SYNC_UPDATES_REMOTE" == true ]]; then
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would pull updates on $REMOTE_SERVER_NAME"
|
||||
else
|
||||
log "Pulling remote container updates on $REMOTE_SERVER_NAME..."
|
||||
for c in "${MAINTENANCE_CONTAINERS[@]}"; do
|
||||
[[ -z "$c" ]] && continue
|
||||
IMAGE=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" \
|
||||
root@"$REMOTE_SERVER" \
|
||||
"docker inspect $c --format '{{.Config.Image}}' 2>/dev/null" 2>/dev/null)
|
||||
if [[ -z "$IMAGE" ]]; then
|
||||
log "$c — not found on remote, skipping update"
|
||||
continue
|
||||
fi
|
||||
log "Pulling $IMAGE for $c on $REMOTE_SERVER_NAME..."
|
||||
if timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" \
|
||||
root@"$REMOTE_SERVER" \
|
||||
"docker pull $IMAGE" >/dev/null 2>&1; then
|
||||
log "$c — remote image updated ✅"
|
||||
else
|
||||
warn "$c — remote pull failed, will start on existing image"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
else
|
||||
echo "WEEKLY_SYNC_UPDATES_REMOTE=false — skipping remote updates"
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Critical Shares Sync ━━━
|
||||
# ==============================================================================================
|
||||
SHARE_COUNT=${#WEEKLY_SYNC_SHARES[@]}
|
||||
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Critical Shares Sync — $SHARE_COUNT share(s) ━━━"
|
||||
|
||||
SYNC_START=$(date +%s)
|
||||
JOB_NUM=0
|
||||
|
||||
if ! check_rsync_enabled "WEEKLY"; then
|
||||
warn "Weekly rsync disabled — skipping all $SHARE_COUNT sync job(s)"
|
||||
warn "Proceeding to container start and maintenance scripts..."
|
||||
elif [[ "$SHARE_COUNT" -eq 0 ]]; then
|
||||
warn "No WEEKLY_SYNC_SHARES configured — skipping sync"
|
||||
warn "Check WEEKLY_SYNC_SHARES in master.conf"
|
||||
else
|
||||
RSYNC_DRY=""
|
||||
[[ "$DRY_RUN" == true ]] && RSYNC_DRY="--dry-run"
|
||||
|
||||
for JOB in "${WEEKLY_SYNC_SHARES[@]}"; do
|
||||
(( JOB_NUM++ ))
|
||||
JOB_NAME=$(basename "$JOB")
|
||||
|
||||
echo ""
|
||||
echo "━━━ [$JOB_NUM/$SHARE_COUNT] $JOB_NAME ━━━"
|
||||
|
||||
JOB_START=$(date +%s)
|
||||
bash "$RSYNC_SCRIPT" "$JOB" $RSYNC_DRY
|
||||
EXIT_CODE=$?
|
||||
JOB_DUR=$(format_duration $(( $(date +%s) - JOB_START )))
|
||||
|
||||
if [[ "$EXIT_CODE" -eq 0 ]]; then
|
||||
PASS+=("$JOB_NAME")
|
||||
log "$JOB_NAME — done in $JOB_DUR ✅"
|
||||
else
|
||||
FAIL+=("$JOB_NAME")
|
||||
error "$JOB_NAME — failed after $JOB_DUR (exit $EXIT_CODE)"
|
||||
fi
|
||||
echo ""
|
||||
done
|
||||
fi
|
||||
|
||||
SYNC_END=$(date +%s)
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Start Containers ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_START $ICON_CONTAINERS Start Containers ━━━"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — containers will not be started"
|
||||
else
|
||||
start_containers
|
||||
|
||||
# Local start — rebuild containers that received a new image, docker start the rest
|
||||
if [[ ${#LOCAL_RUNNING_CONTAINERS[@]} -eq 0 ]]; then
|
||||
log "No local containers to restart."
|
||||
else
|
||||
for _c in "${LOCAL_RUNNING_CONTAINERS[@]}"; do
|
||||
[[ -z "$_c" ]] && continue
|
||||
_needs_delay=false
|
||||
for _d in "${DELAYED_CONTAINERS[@]}"; do
|
||||
[[ "$_c" == "$_d" ]] && _needs_delay=true && break
|
||||
done
|
||||
[[ "$_needs_delay" == true ]] && {
|
||||
info "Waiting ${CONTAINER_DELAY}s before starting $_c..."
|
||||
sleep "$CONTAINER_DELAY"
|
||||
}
|
||||
if [[ -n "${_weekly_needs_rebuild[$_c]:-}" ]]; then
|
||||
log "Rebuilding $_c on new image..."
|
||||
if platform_rebuild_container "$_c"; then
|
||||
log "$_c rebuilt on new image ✅"
|
||||
else
|
||||
warn "$_c rebuild failed — falling back to docker start"
|
||||
docker start "$_c" >/dev/null 2>&1 || error "Failed to start $_c"
|
||||
fi
|
||||
else
|
||||
docker start "$_c" >/dev/null 2>&1 && log "$_c started" || error "Failed to start $_c"
|
||||
fi
|
||||
done
|
||||
unset _c _d _needs_delay
|
||||
fi
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Post-sync Jobs ━━━
|
||||
# ==============================================================================================
|
||||
if [[ ${#WEEKLY_MAINTENANCE_SCRIPTS[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Post-sync Jobs ━━━"
|
||||
for script_entry in "${WEEKLY_MAINTENANCE_SCRIPTS[@]}"; do
|
||||
[[ -z "$script_entry" ]] && continue
|
||||
echo ""
|
||||
run_job "$script_entry"
|
||||
done
|
||||
fi
|
||||
|
||||
WINDOW_END=$(date +%s)
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY WEEKLY SYNC MAINTENANCE SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_TIME Window: $(date -d @"$WINDOW_START" '+%Y-%m-%d %H:%M:%S') → $(date -d @"$WINDOW_END" '+%H:%M:%S')"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( WINDOW_END - WINDOW_START )))"
|
||||
echo "$ICON_GEAR Updates: local=${WEEKLY_SYNC_UPDATES:-false} remote=${WEEKLY_SYNC_UPDATES_REMOTE:-false}"
|
||||
echo ""
|
||||
|
||||
echo "$ICON_SYNC Sync jobs ($SHARE_COUNT):"
|
||||
for job in "${PASS[@]}"; do echo " $ICON_DONE $job"; done
|
||||
for job in "${FAIL[@]}"; do echo " $ICON_ERROR $job"; done
|
||||
echo " Passed: ${#PASS[@]} Failed: ${#FAIL[@]}"
|
||||
|
||||
if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
echo "$ICON_GEAR Post-sync jobs:"
|
||||
for job in "${JOB_PASS[@]}"; do echo " $ICON_DONE $job"; done
|
||||
for job in "${JOB_FAIL[@]}"; do echo " $ICON_ERROR $job"; done
|
||||
fi
|
||||
|
||||
TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} ))
|
||||
|
||||
echo ""
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no changes made"
|
||||
elif [[ "$TOTAL_FAIL" -eq 0 ]]; then
|
||||
echo "$ICON_DONE Status: all complete ✅ — ${#PASS[@]} share(s) synced, ${#JOB_PASS[@]} job(s) run"
|
||||
else
|
||||
warn "Status: $TOTAL_FAIL failure(s)"
|
||||
notify "Weekly maintenance failed on $(hostname) ($MY_ID) — sync: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \
|
||||
"Weekly Maintenance" "warning"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ "$TOTAL_FAIL" -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
+164
@@ -0,0 +1,164 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ============================= Docker Prune Images ============================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Removes orphaned Docker images. Two modes:
|
||||
#
|
||||
# Default — dangling only (safe, fast):
|
||||
# Removes untagged images (no name, no container reference). These accumulate
|
||||
# after container updates pull a new image, leaving the old one untagged.
|
||||
# Running containers are never affected.
|
||||
#
|
||||
# --all — full orphan cleanup:
|
||||
# Step 1: removes stopped containers (exited/created state).
|
||||
# Step 2: removes all images not used by any running container.
|
||||
# Use this to clear tagged images left behind by removed or stopped apps.
|
||||
# CAUTION: also removes intentionally stopped containers — only run when you
|
||||
# know all stopped containers are safe to delete.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# docker_prune_images.sh
|
||||
# Remove dangling (untagged) images only.
|
||||
#
|
||||
# docker_prune_images.sh --all
|
||||
# Remove stopped containers, then remove all unused images.
|
||||
#
|
||||
# docker_prune_images.sh --dry-run
|
||||
# Show what would be removed without making changes.
|
||||
#
|
||||
# docker_prune_images.sh --status
|
||||
# Show dangling images and stopped containers. No changes.
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
# ── Handle --all before parse_args ────────────────────────────────────────────
|
||||
ALL_MODE=false
|
||||
FILTERED_ARGS=()
|
||||
for _arg in "$@"; do
|
||||
[[ "$_arg" == "--all" ]] && ALL_MODE=true || FILTERED_ARGS+=("$_arg")
|
||||
done
|
||||
parse_args "${FILTERED_ARGS[@]}"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
acquire_lock
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY DOCKER PRUNE IMAGES STATUS ━━━━━"
|
||||
|
||||
DANGLING=$(docker images -f "dangling=true" --format "{{.ID}}\t{{.Repository}}:{{.Tag}}\t{{.Size}}\t{{.CreatedSince}}" 2>/dev/null)
|
||||
STOPPED=$(docker ps -a --filter "status=exited" --filter "status=created" \
|
||||
--format "{{.ID}}\t{{.Names}}\t{{.Image}}\t{{.Status}}" 2>/dev/null)
|
||||
|
||||
echo ""
|
||||
echo "$ICON_CONTAINERS Dangling images (no tag, no container):"
|
||||
if [[ -z "$DANGLING" ]]; then
|
||||
echo " none"
|
||||
else
|
||||
echo "$DANGLING" | while IFS=$'\t' read -r id repo size age; do
|
||||
echo " $id $repo $size $age"
|
||||
done
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "$ICON_CONTAINERS Stopped containers (--all would remove these first):"
|
||||
if [[ -z "$STOPPED" ]]; then
|
||||
echo " none"
|
||||
else
|
||||
echo "$STOPPED" | while IFS=$'\t' read -r id name image status; do
|
||||
echo " $name ($image) $status"
|
||||
done
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Run ━━━
|
||||
# ==============================================================================================
|
||||
MODE_LABEL=$([[ "$ALL_MODE" == true ]] && echo "full orphan cleanup" || echo "dangling only")
|
||||
log "$ICON_GEAR Config: mode=${MODE_LABEL} dry-run=${DRY_RUN}"
|
||||
echo "━━━ $ICON_CONTAINERS Docker Prune Images ($MODE_LABEL) — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
|
||||
TOTAL_RECLAIMED=0
|
||||
|
||||
# ── Step 1 (--all only): remove stopped containers ────────────────────────────
|
||||
if [[ "$ALL_MODE" == true ]]; then
|
||||
STOPPED_IDS=$(docker ps -a --filter "status=exited" --filter "status=created" -q 2>/dev/null)
|
||||
STOPPED_COUNT=$(echo "$STOPPED_IDS" | grep -c . || echo 0)
|
||||
|
||||
if [[ "$STOPPED_COUNT" -eq 0 ]]; then
|
||||
log "No stopped containers"
|
||||
elif [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would remove $STOPPED_COUNT stopped container(s):"
|
||||
docker ps -a --filter "status=exited" --filter "status=created" \
|
||||
--format " {{.Names}} {{.Image}} {{.Status}}" 2>/dev/null
|
||||
else
|
||||
log "Removing $STOPPED_COUNT stopped container(s)..."
|
||||
docker container prune -f 2>&1 | grep -v "^Total\|^$" || true
|
||||
success "Removed $STOPPED_COUNT stopped container(s)"
|
||||
fi
|
||||
echo ""
|
||||
fi
|
||||
|
||||
# ── Step 2: prune images ───────────────────────────────────────────────────────
|
||||
PRUNE_FLAGS=$([[ "$ALL_MODE" == true ]] && echo "-a" || echo "")
|
||||
PRUNE_FILTER=$([[ "$ALL_MODE" == true ]] && echo "" || echo '-f "dangling=true"')
|
||||
|
||||
# Preview count
|
||||
if [[ "$ALL_MODE" == true ]]; then
|
||||
# Images not used by any running container
|
||||
RUNNING_IMAGES=$(docker ps --format "{{.Image}}" 2>/dev/null)
|
||||
UNUSED_COUNT=$(docker images --format "{{.Repository}}:{{.Tag}}" 2>/dev/null \
|
||||
| grep -vxF "$RUNNING_IMAGES" | grep -c . || echo 0)
|
||||
TARGET_LABEL="$UNUSED_COUNT unused image(s)"
|
||||
else
|
||||
DANGLING_IDS=$(docker images -f "dangling=true" -q 2>/dev/null)
|
||||
UNUSED_COUNT=$(echo "$DANGLING_IDS" | grep -c . || echo 0)
|
||||
TARGET_LABEL="$UNUSED_COUNT dangling image(s)"
|
||||
fi
|
||||
|
||||
if [[ "$UNUSED_COUNT" -eq 0 ]]; then
|
||||
success "No images to remove"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would remove $TARGET_LABEL:"
|
||||
if [[ "$ALL_MODE" == true ]]; then
|
||||
docker images --format " {{.Repository}}:{{.Tag}} {{.Size}} {{.CreatedSince}}" 2>/dev/null \
|
||||
| grep -vF "$(docker ps --format '{{.Image}}' 2>/dev/null)" || true
|
||||
else
|
||||
docker images -f "dangling=true" \
|
||||
--format " {{.ID}} {{.Size}} created {{.CreatedSince}}" 2>/dev/null
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
|
||||
log "Pruning $TARGET_LABEL..."
|
||||
OUTPUT=$(docker image prune $PRUNE_FLAGS -f 2>&1)
|
||||
[[ "$ENABLE_LOGGING" == "true" ]] && echo "$OUTPUT" | sed 's/^/ /'
|
||||
RECLAIMED=$(echo "$OUTPUT" | grep -E "^Total reclaimed" || echo "Total reclaimed space: unknown")
|
||||
success "Done — $RECLAIMED"
|
||||
@@ -0,0 +1,808 @@
|
||||
<?php
|
||||
require_once __DIR__ . '/config.php';
|
||||
require_once __DIR__ . '/unraid_api.php';
|
||||
|
||||
// Common helpers shared across all Varaverk pages.
|
||||
|
||||
function vv_system_info(): array {
|
||||
// ── Shared local reads (always needed regardless of API) ──────────────────
|
||||
$ident = @parse_ini_file('/boot/config/ident.cfg') ?: [];
|
||||
$var = [];
|
||||
foreach (@file('/var/local/emhttp/var.ini') ?: [] as $line) {
|
||||
if (preg_match('/^(\w+)="([^"]*)"/', $line, $m)) $var[$m[1]] = $m[2];
|
||||
}
|
||||
$version = trim(@file_get_contents('/etc/unraid-version') ?: '');
|
||||
|
||||
// ── API path ──────────────────────────────────────────────────────────────
|
||||
$api = vv_api_data();
|
||||
if ($api) {
|
||||
$os = $api['info']['os'] ?? [];
|
||||
$cpu = $api['info']['cpu'] ?? [];
|
||||
|
||||
// uptime is a String in this schema — try numeric (seconds) first, else display as-is
|
||||
$uptimeRaw = $os['uptime'] ?? '';
|
||||
if (is_numeric($uptimeRaw)) {
|
||||
$uptimeSec = (int)$uptimeRaw;
|
||||
$uptime = vv_format_uptime($uptimeSec);
|
||||
} else {
|
||||
$uptimeSec = 0;
|
||||
$uptime = $uptimeRaw ?: '—';
|
||||
}
|
||||
|
||||
$load = sys_getloadavg();
|
||||
return [
|
||||
'name' => $os['hostname'] ?? ($ident['NAME'] ?? gethostname()),
|
||||
'comment' => $ident['COMMENT'] ?? '',
|
||||
'timezone' => $ident['timeZone'] ?? 'UTC',
|
||||
'cpu_model' => $cpu['brand'] ?? ($ident['SYS_MODEL'] ?? ''),
|
||||
'cpu_threads' => (int)($cpu['threads'] ?? 0),
|
||||
'cpu_cores' => (int)($cpu['cores'] ?? 0),
|
||||
'reg_type' => 'Unraid OS ' . ($var['regTy'] ?? ''),
|
||||
'reg_to' => $var['regTo'] ?? '',
|
||||
'uptime' => $uptime,
|
||||
'uptime_sec' => $uptimeSec,
|
||||
'array_state' => strtoupper($api['array']['state'] ?? $var['mdState'] ?? 'UNKNOWN'),
|
||||
'version' => trim($os['release'] ?? '') ?: $version,
|
||||
'load_avg' => $load ? [round($load[0], 2), round($load[1], 2), round($load[2], 2)] : null,
|
||||
];
|
||||
}
|
||||
|
||||
// ── Local fallback ────────────────────────────────────────────────────────
|
||||
vv_api_record_fallback('system_info');
|
||||
$cpuModel = '';
|
||||
foreach (@file('/proc/cpuinfo') ?: [] as $line) {
|
||||
if (preg_match('/^model name\s*:\s*(.+)/', $line, $m)) { $cpuModel = trim($m[1]); break; }
|
||||
}
|
||||
$uptimeSec = (int)explode(' ', @file_get_contents('/proc/uptime') ?: '0')[0];
|
||||
$uptime = vv_format_uptime($uptimeSec);
|
||||
|
||||
$load = sys_getloadavg();
|
||||
return [
|
||||
'name' => $ident['NAME'] ?? gethostname(),
|
||||
'comment' => $ident['COMMENT'] ?? '',
|
||||
'timezone' => $ident['timeZone'] ?? 'UTC',
|
||||
'cpu_model' => $ident['SYS_MODEL'] ?? $cpuModel,
|
||||
'cpu_threads' => 0,
|
||||
'cpu_cores' => 0,
|
||||
'reg_type' => 'Unraid OS ' . ($var['regTy'] ?? ''),
|
||||
'reg_to' => $var['regTo'] ?? '',
|
||||
'uptime' => $uptime,
|
||||
'uptime_sec' => $uptimeSec,
|
||||
'array_state' => $var['mdState'] ?? 'UNKNOWN',
|
||||
'version' => $version,
|
||||
'load_avg' => $load ? [round($load[0], 2), round($load[1], 2), round($load[2], 2)] : null,
|
||||
];
|
||||
}
|
||||
|
||||
function vv_docker_containers(): array {
|
||||
$out = shell_exec('docker ps --format \'{"name":"{{.Names}}","status":"{{.Status}}","image":"{{.Image}}"}\' 2>/dev/null');
|
||||
$containers = [];
|
||||
foreach (explode("\n", trim($out ?? '')) as $line) {
|
||||
if (!$line) continue;
|
||||
$c = json_decode($line, true);
|
||||
if ($c) $containers[] = $c;
|
||||
}
|
||||
return $containers;
|
||||
}
|
||||
|
||||
function vv_docker_stopped(): array {
|
||||
$out = shell_exec('docker ps -a --filter "status=exited" --filter "status=created" --format \'{"name":"{{.Names}}","status":"{{.Status}}"}\' 2>/dev/null');
|
||||
$containers = [];
|
||||
foreach (explode("\n", trim($out ?? '')) as $line) {
|
||||
if (!$line) continue;
|
||||
$c = json_decode($line, true);
|
||||
if ($c) $containers[] = $c;
|
||||
}
|
||||
return $containers;
|
||||
}
|
||||
|
||||
function vv_gpu_stats(): array {
|
||||
$out = shell_exec('nvidia-smi --query-gpu=name,memory.used,memory.total,utilization.gpu,temperature.gpu,power.draw,utilization.encoder,utilization.decoder --format=csv,noheader,nounits 2>/dev/null');
|
||||
if (!$out) return ['available' => false];
|
||||
|
||||
$parts = array_map('trim', explode(',', $out));
|
||||
$power = is_numeric($parts[5] ?? '') ? round((float)$parts[5], 1) : null;
|
||||
return [
|
||||
'available' => true,
|
||||
'name' => $parts[0] ?? '',
|
||||
'memory_used' => (int)($parts[1] ?? 0),
|
||||
'memory_total' => (int)($parts[2] ?? 0),
|
||||
'utilization' => (int)($parts[3] ?? 0),
|
||||
'temperature' => (int)($parts[4] ?? 0),
|
||||
'power_w' => $power,
|
||||
'enc_pct' => (int)($parts[6] ?? 0),
|
||||
'dec_pct' => (int)($parts[7] ?? 0),
|
||||
];
|
||||
}
|
||||
|
||||
function vv_gpu_processes(): array {
|
||||
$out = shell_exec('nvidia-smi --query-compute-apps=pid,used_gpu_memory,name --format=csv,noheader,nounits 2>/dev/null');
|
||||
$procs = [];
|
||||
foreach (explode("\n", trim($out ?? '')) as $line) {
|
||||
if (!$line) continue;
|
||||
$parts = array_map('trim', explode(',', $line));
|
||||
$procs[] = [
|
||||
'pid' => $parts[0] ?? '',
|
||||
'memory_mb' => $parts[1] ?? '',
|
||||
'name' => $parts[2] ?? '',
|
||||
];
|
||||
}
|
||||
return $procs;
|
||||
}
|
||||
|
||||
function vv_system_resources(): array {
|
||||
$mem = [];
|
||||
foreach (file('/proc/meminfo') ?: [] as $line) {
|
||||
if (preg_match('/^(MemTotal|MemAvailable):\s+(\d+)/', $line, $m))
|
||||
$mem[$m[1]] = (int)$m[2];
|
||||
}
|
||||
return [
|
||||
'ram_total_mb' => (int)(($mem['MemTotal'] ?? 0) / 1024),
|
||||
'ram_free_mb' => (int)(($mem['MemAvailable'] ?? 0) / 1024),
|
||||
'cache' => vv_df('/mnt/cache'),
|
||||
];
|
||||
}
|
||||
|
||||
function vv_cpu_per_core(): array {
|
||||
// Parse /proc/stat — [user, nice, system, idle, iowait, irq, softirq]
|
||||
$raw = [];
|
||||
foreach (file('/proc/stat') ?: [] as $line) {
|
||||
if (!preg_match('/^(cpu\d*)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)/', $line, $m)) continue;
|
||||
$raw[$m[1]] = [(int)$m[2],(int)$m[3],(int)$m[4],(int)$m[5],(int)$m[6],(int)$m[7],(int)$m[8]];
|
||||
}
|
||||
|
||||
$stateFile = VV_CACHE_DIR . '/vv_cpu_stat.json';
|
||||
$prev = file_exists($stateFile) ? (json_decode(file_get_contents($stateFile), true) ?: []) : [];
|
||||
// Atomic write — concurrent fast/slow polls read a consistent snapshot
|
||||
$tmp = $stateFile . '.tmp';
|
||||
file_put_contents($tmp, json_encode($raw));
|
||||
rename($tmp, $stateFile);
|
||||
|
||||
$usage = function(array $c, ?array $p): int {
|
||||
if (!$p) return 0;
|
||||
$dt = array_sum($c) - array_sum($p);
|
||||
$di = ($c[3] + $c[4]) - ($p[3] + $p[4]);
|
||||
return $dt > 0 ? max(0, min(100, (int)round((1 - $di / $dt) * 100))) : 0;
|
||||
};
|
||||
|
||||
$overall = $usage($raw['cpu'] ?? [], $prev['cpu'] ?? null);
|
||||
$cores = [];
|
||||
foreach ($raw as $cpu => $c) {
|
||||
if ($cpu === 'cpu') continue;
|
||||
$num = (int)substr($cpu, 3);
|
||||
$freqKhz = (int)@file_get_contents("/sys/devices/system/cpu/$cpu/cpufreq/scaling_cur_freq");
|
||||
$maxKhz = (int)@file_get_contents("/sys/devices/system/cpu/$cpu/cpufreq/cpuinfo_max_freq");
|
||||
$minKhz = (int)@file_get_contents("/sys/devices/system/cpu/$cpu/cpufreq/cpuinfo_min_freq");
|
||||
$cores[] = [
|
||||
'core' => $num,
|
||||
'usage_pct' => $usage($c, $prev[$cpu] ?? null),
|
||||
'freq_mhz' => $freqKhz > 0 ? (int)round($freqKhz / 1000) : 0,
|
||||
'max_mhz' => $maxKhz > 0 ? (int)round($maxKhz / 1000) : 0,
|
||||
'min_mhz' => $minKhz > 0 ? (int)round($minKhz / 1000) : 0,
|
||||
];
|
||||
}
|
||||
usort($cores, fn($a, $b) => $a['core'] - $b['core']);
|
||||
return ['overall' => $overall, 'cores' => $cores];
|
||||
}
|
||||
|
||||
function vv_memory_breakdown(): array {
|
||||
$mem = [];
|
||||
foreach (file('/proc/meminfo') ?: [] as $line) {
|
||||
if (preg_match('/^(\w+):\s+(\d+)/', $line, $m)) $mem[$m[1]] = (int)$m[2];
|
||||
}
|
||||
$totalKb = $mem['MemTotal'] ?? 0;
|
||||
|
||||
// ZFS ARC
|
||||
$arcKb = 0;
|
||||
foreach (@file('/proc/spl/kstat/zfs/arcstats') ?: [] as $line) {
|
||||
if (preg_match('/^size\s+\d+\s+(\d+)/', $line, $m)) { $arcKb = (int)($m[1] / 1024); break; }
|
||||
}
|
||||
|
||||
// Docker — sum docker stats used memory per container (matches Unraid dashboard)
|
||||
$dockerKb = 0;
|
||||
$dsOut = shell_exec("docker stats --no-stream --format '{{.MemUsage}}' 2>/dev/null") ?: '';
|
||||
foreach (explode("\n", trim($dsOut)) as $line) {
|
||||
if (!preg_match('/^([0-9.]+)(GiB|MiB|KiB|B)\s*\//', trim($line), $m)) continue;
|
||||
$val = (float)$m[1];
|
||||
$dockerKb += match($m[2]) {
|
||||
'GiB' => (int)($val * 1048576),
|
||||
'MiB' => (int)($val * 1024),
|
||||
'KiB' => (int)$val,
|
||||
default => (int)($val / 1024),
|
||||
};
|
||||
}
|
||||
|
||||
// VM (QEMU/KVM RSS)
|
||||
$vmKb = 0;
|
||||
foreach (preg_split('/\s+/', trim(shell_exec('ps -C qemu-system-x86_64 -o rss= 2>/dev/null') ?: '')) as $rss) {
|
||||
if (is_numeric($rss) && $rss > 0) $vmKb += (int)$rss;
|
||||
}
|
||||
|
||||
$freeKb = max(0, $mem['MemAvailable'] ?? 0);
|
||||
$systemKb = max(0, $totalKb - $freeKb - $arcKb - $dockerKb - $vmKb);
|
||||
|
||||
// Top processes by RSS — group same-named procs, take top 5
|
||||
$grouped = [];
|
||||
$psOut = shell_exec("ps -eo comm,rss --sort=-rss 2>/dev/null | tail -n +2 | head -40") ?: '';
|
||||
foreach (explode("\n", trim($psOut)) as $line) {
|
||||
$parts = preg_split('/\s+/', trim($line), 2);
|
||||
if (count($parts) === 2 && is_numeric($parts[1]) && (int)$parts[1] > 0)
|
||||
$grouped[$parts[0]] = ($grouped[$parts[0]] ?? 0) + (int)$parts[1];
|
||||
}
|
||||
arsort($grouped);
|
||||
$topProcs = [];
|
||||
foreach (array_slice($grouped, 0, 3, true) as $name => $kb)
|
||||
$topProcs[] = ['name' => $name, 'kb' => $kb];
|
||||
|
||||
// Swap — from API metrics when available, else /proc/meminfo
|
||||
$swapTotalKb = 0; $swapUsedKb = 0;
|
||||
$apiMem = vv_api_data()['metrics']['memory'] ?? [];
|
||||
if (!empty($apiMem['swapTotal'])) {
|
||||
$swapTotalKb = (int)(((float)$apiMem['swapTotal']) / 1024);
|
||||
$swapUsedKb = (int)(((float)$apiMem['swapUsed']) / 1024);
|
||||
} else {
|
||||
$swapTotalKb = $mem['SwapTotal'] ?? 0;
|
||||
$swapUsedKb = ($mem['SwapTotal'] ?? 0) - ($mem['SwapFree'] ?? 0);
|
||||
}
|
||||
|
||||
return [
|
||||
'total_kb' => $totalKb,
|
||||
'system_kb' => $systemKb,
|
||||
'vm_kb' => $vmKb,
|
||||
'zfs_kb' => $arcKb,
|
||||
'docker_kb' => $dockerKb,
|
||||
'free_kb' => $freeKb,
|
||||
'swap_total_kb' => $swapTotalKb,
|
||||
'swap_used_kb' => $swapUsedKb,
|
||||
'top_procs' => $topProcs,
|
||||
];
|
||||
}
|
||||
|
||||
function vv_df(string $path): array {
|
||||
$out = shell_exec("df -BM --output=size,used,avail '$path' 2>/dev/null | tail -1");
|
||||
if (!$out) return ['available' => false, 'path' => $path];
|
||||
$parts = preg_split('/\s+/', trim($out));
|
||||
return [
|
||||
'available' => true,
|
||||
'path' => $path,
|
||||
'size_mb' => (int)$parts[0],
|
||||
'used_mb' => (int)$parts[1],
|
||||
'free_mb' => (int)$parts[2],
|
||||
];
|
||||
}
|
||||
|
||||
function vv_network_stats(): array {
|
||||
$iface = trim(shell_exec("ip route show default 2>/dev/null | awk 'NR==1{print \$5}'") ?: '');
|
||||
if (!$iface) {
|
||||
$best = ''; $bestBytes = 0;
|
||||
foreach (file('/proc/net/dev') ?: [] as $line) {
|
||||
if (!preg_match('/^\s*(\w+):\s+(\d+)/', $line, $m) || $m[1] === 'lo') continue;
|
||||
if ((int)$m[2] > $bestBytes) { $bestBytes = (int)$m[2]; $best = $m[1]; }
|
||||
}
|
||||
$iface = $best;
|
||||
}
|
||||
if (!$iface) return ['available' => false];
|
||||
|
||||
$rxBytes = $txBytes = 0;
|
||||
foreach (file('/proc/net/dev') ?: [] as $line) {
|
||||
if (!preg_match('/^\s*' . preg_quote($iface, '/') . ':\s+(.+)$/', $line, $m)) continue;
|
||||
$parts = preg_split('/\s+/', trim($m[1]));
|
||||
$rxBytes = (int)($parts[0] ?? 0);
|
||||
$txBytes = (int)($parts[8] ?? 0);
|
||||
break;
|
||||
}
|
||||
|
||||
$stateFile = VV_CACHE_DIR . '/vv_net_stat.json';
|
||||
$now = ['rx' => $rxBytes, 'tx' => $txBytes, 'ts' => microtime(true)];
|
||||
$prev = file_exists($stateFile) ? (json_decode(file_get_contents($stateFile), true) ?: []) : [];
|
||||
$tmp = $stateFile . '.tmp';
|
||||
file_put_contents($tmp, json_encode($now));
|
||||
rename($tmp, $stateFile);
|
||||
|
||||
$rxRate = $txRate = 0;
|
||||
if (!empty($prev['ts']) && ($dt = $now['ts'] - $prev['ts']) > 0.1) {
|
||||
$rxRate = max(0, (int)(($rxBytes - ($prev['rx'] ?? $rxBytes)) / $dt));
|
||||
$txRate = max(0, (int)(($txBytes - ($prev['tx'] ?? $txBytes)) / $dt));
|
||||
}
|
||||
|
||||
$speedMbps = (int)@file_get_contents("/sys/class/net/$iface/speed");
|
||||
|
||||
// Local IP — use primary iface
|
||||
$localIp = trim(shell_exec(
|
||||
"ip -4 addr show " . escapeshellarg($iface) . " 2>/dev/null | awk '/inet /{print \$2}' | cut -d/ -f1 | head -1"
|
||||
) ?: '');
|
||||
|
||||
// External IP — curl ifconfig.me, cached 5 min so we don't hammer it
|
||||
$extIp = '';
|
||||
$extData = vv_cache_read('ext_ip', 300);
|
||||
if ($extData) {
|
||||
$extIp = $extData['ip'] ?? '';
|
||||
} else {
|
||||
$fetched = trim(shell_exec('curl -sf --max-time 4 https://ifconfig.me 2>/dev/null') ?: '');
|
||||
if (preg_match('/^\d+\.\d+\.\d+\.\d+$/', $fetched)) {
|
||||
$extIp = $fetched;
|
||||
vv_cache_write('ext_ip', ['ip' => $extIp]);
|
||||
}
|
||||
}
|
||||
|
||||
// Tailscale IP — use `tailscale ip` CLI (interface name varies: tailscale0, tailscale1, etc.)
|
||||
$tsIp = trim(shell_exec('tailscale ip -4 2>/dev/null | head -1') ?: '');
|
||||
|
||||
return [
|
||||
'available' => true,
|
||||
'iface' => $iface,
|
||||
'speed_mbps' => $speedMbps > 0 ? $speedMbps : null,
|
||||
'rx_bps' => $rxRate,
|
||||
'tx_bps' => $txRate,
|
||||
'local_ip' => $localIp,
|
||||
'ext_ip' => $extIp,
|
||||
'ts_ip' => $tsIp,
|
||||
];
|
||||
}
|
||||
|
||||
function vv_disk_entry(array $d, string $key, string $role = 'data'): ?array {
|
||||
$name = $d['name'] ?? $key;
|
||||
$isParity = $role === 'parity';
|
||||
$mounted = ($d['fsStatus'] ?? '') === 'Mounted';
|
||||
// Parity has no filesystem — use raw size only
|
||||
$size_kb = (int)($isParity ? ($d['size'] ?? 0) : ($mounted ? ($d['fsSize'] ?? 0) : ($d['size'] ?? 0)));
|
||||
$used_kb = (int)($isParity ? 0 : ($mounted ? ($d['fsUsed'] ?? 0) : 0));
|
||||
if ($size_kb <= 0) return null;
|
||||
$tempRaw = trim($d['temp'] ?? '');
|
||||
return [
|
||||
'name' => $name,
|
||||
'device' => $d['device'] ?? $key,
|
||||
'role' => $role,
|
||||
'size_gb' => round($size_kb / 1048576, 1),
|
||||
'used_gb' => round($used_kb / 1048576, 1),
|
||||
'pct' => (!$isParity && $size_kb > 0) ? round($used_kb / $size_kb * 100, 1) : null,
|
||||
'temp' => is_numeric($tempRaw) ? (int)$tempRaw : null,
|
||||
'transport' => $d['transport'] ?? 'ata',
|
||||
'mounted' => $mounted,
|
||||
'status' => $d['status'] ?? '',
|
||||
];
|
||||
}
|
||||
|
||||
function vv_ups_stats(): array {
|
||||
$raw = shell_exec('apcaccess 2>/dev/null') ?: '';
|
||||
if (!$raw) return ['available' => false];
|
||||
|
||||
$fields = [];
|
||||
foreach (explode("\n", $raw) as $line) {
|
||||
if (preg_match('/^(\w+)\s*:\s*(.+)$/', trim($line), $m)) {
|
||||
$fields[trim($m[1])] = trim($m[2]);
|
||||
}
|
||||
}
|
||||
if (empty($fields)) return ['available' => false];
|
||||
|
||||
$parse_num = fn(string $k) => isset($fields[$k]) ? (float)$fields[$k] : null;
|
||||
|
||||
$loadPct = $parse_num('LOADPCT');
|
||||
$nomPower = $parse_num('NOMPOWER');
|
||||
$watts = ($loadPct !== null && $nomPower !== null) ? round($loadPct / 100 * $nomPower) : null;
|
||||
|
||||
return [
|
||||
'available' => true,
|
||||
'model' => $fields['MODEL'] ?? '',
|
||||
'status' => trim(explode(' ', $fields['STATUS'] ?? 'UNKNOWN')[0]),
|
||||
'line_v' => $parse_num('LINEV'),
|
||||
'output_v' => $parse_num('OUTPUTV'),
|
||||
'load_pct' => $loadPct,
|
||||
'nom_power' => $nomPower,
|
||||
'watts' => $watts,
|
||||
'bcharge' => $parse_num('BCHARGE'),
|
||||
'timeleft' => $parse_num('TIMELEFT'),
|
||||
'num_xfers' => (int)($fields['NUMXFERS'] ?? 0),
|
||||
'on_batt_s' => $parse_num('CUMONBATT'),
|
||||
'selftest' => $fields['SELFTEST'] ?? '',
|
||||
];
|
||||
}
|
||||
|
||||
function vv_parity_status(): array {
|
||||
$var = [];
|
||||
foreach (@file('/var/local/emhttp/var.ini') ?: [] as $line) {
|
||||
if (preg_match('/^(\w+)="([^"]*)"/', $line, $m)) $var[$m[1]] = $m[2];
|
||||
}
|
||||
|
||||
$numDisabled = (int)($var['mdNumDisabled'] ?? 0);
|
||||
$numMissing = (int)($var['mdNumMissing'] ?? 0);
|
||||
$exitCode = (int)($var['sbSyncExit'] ?? 0);
|
||||
$errors = (int)($var['sbSyncErrs'] ?? 0);
|
||||
// Emulated (DISK_DSBL) disks are protected by parity and don't make parity invalid.
|
||||
// True invalidity: sync errors on the last check, or unprotectable missing slots.
|
||||
$isValid = $errors === 0 && $numMissing === 0;
|
||||
$inProgress = ($var['mdResync'] ?? '0') !== '0';
|
||||
$resyncPos = (int)($var['mdResyncPos'] ?? 0);
|
||||
$resyncSize = (int)($var['mdResyncSize'] ?? 1);
|
||||
$resyncPct = $resyncSize > 0 ? round($resyncPos / $resyncSize * 100, 1) : 0;
|
||||
|
||||
// Last check from log
|
||||
$lastDate = null; $lastDuration = 0; $lastSpeed = 0; $lastErrors = 0; $lastExit = 0;
|
||||
$logFile = '/boot/config/parity-checks.log';
|
||||
if (file_exists($logFile)) {
|
||||
$lines = file($logFile, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES) ?: [];
|
||||
if ($lines) {
|
||||
$p = explode('|', trim(end($lines)));
|
||||
$lastDate = trim($p[0] ?? '');
|
||||
$lastDuration = (int)($p[1] ?? 0);
|
||||
$lastSpeed = (int)($p[2] ?? 0);
|
||||
$lastExit = (int)($p[3] ?? 0);
|
||||
$lastErrors = (int)($p[4] ?? 0);
|
||||
}
|
||||
}
|
||||
|
||||
// Parse last date string to timestamp
|
||||
$lastTs = $lastDate ? strtotime($lastDate) : null;
|
||||
|
||||
// Next scheduled check from cron
|
||||
$nextTs = null;
|
||||
$cronFile = '/boot/config/plugins/dynamix/parity-check.cron';
|
||||
foreach (@file($cronFile) ?: [] as $line) {
|
||||
$line = trim($line);
|
||||
if ($line === '' || $line[0] === '#') continue;
|
||||
if (!str_contains($line, 'mdcmd')) continue;
|
||||
$p = preg_split('/\s+/', $line);
|
||||
// cron: min hour dom month dow command...
|
||||
if (count($p) >= 5 && is_numeric($p[0]) && is_numeric($p[1]) && is_numeric($p[2])) {
|
||||
$next = new DateTime('now');
|
||||
$next->setTime((int)$p[1], (int)$p[0], 0);
|
||||
$next->setDate((int)$next->format('Y'), (int)$next->format('n'), (int)$p[2]);
|
||||
if ($next->getTimestamp() <= time()) $next->modify('+1 month');
|
||||
$nextTs = $next->getTimestamp();
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
$exitMap = ['0' => 'Completed', '-4' => 'Aborted', '-5' => 'Cancelled'];
|
||||
return [
|
||||
'valid' => $isValid,
|
||||
'num_disabled' => $numDisabled,
|
||||
'num_missing' => $numMissing,
|
||||
'in_progress' => $inProgress,
|
||||
'resync_pct' => $resyncPct,
|
||||
'exit_code' => $exitCode,
|
||||
'exit_label' => $exitMap[(string)$lastExit] ?? 'Unknown',
|
||||
'errors' => $lastErrors,
|
||||
'last_date' => $lastDate,
|
||||
'last_ts' => $lastTs,
|
||||
'last_duration' => $lastDuration,
|
||||
'last_speed_mb' => $lastSpeed > 0 ? round($lastSpeed / 1048576, 1) : null,
|
||||
'next_ts' => $nextTs,
|
||||
];
|
||||
}
|
||||
|
||||
function vv_storage_pools(): array {
|
||||
// ── API path ──────────────────────────────────────────────────────────────
|
||||
$api = vv_api_data();
|
||||
if ($api && isset($api['array']['caches'])) {
|
||||
$out = [];
|
||||
foreach ($api['array']['caches'] as $d) {
|
||||
$entry = vv_api_disk_entry($d, 'data');
|
||||
if ($entry) $out[] = $entry;
|
||||
}
|
||||
if (!empty($out)) {
|
||||
usort($out, fn($a, $b) => strcmp($a['name'], $b['name']));
|
||||
return $out;
|
||||
}
|
||||
}
|
||||
|
||||
// ── Local fallback ────────────────────────────────────────────────────────
|
||||
vv_api_record_fallback('storage_pools');
|
||||
$ini = @parse_ini_file('/var/local/emhttp/disks.ini', true) ?: [];
|
||||
$out = [];
|
||||
foreach ($ini as $key => $d) {
|
||||
if (($d['type'] ?? '') !== 'Cache') continue;
|
||||
if (($d['fsStatus'] ?? '') !== 'Mounted') continue;
|
||||
$entry = vv_disk_entry($d, $key);
|
||||
if ($entry) $out[] = $entry;
|
||||
}
|
||||
usort($out, fn($a, $b) => strcmp($a['name'], $b['name']));
|
||||
return $out;
|
||||
}
|
||||
|
||||
function vv_array_disks(): array {
|
||||
// ── API path ──────────────────────────────────────────────────────────────
|
||||
$api = vv_api_data();
|
||||
if ($api && (isset($api['array']['parities']) || isset($api['array']['disks']))) {
|
||||
$parity = [];
|
||||
$data = [];
|
||||
foreach ($api['array']['parities'] ?? [] as $d) {
|
||||
$entry = vv_api_disk_entry($d, 'parity');
|
||||
if ($entry) $parity[] = $entry;
|
||||
}
|
||||
foreach ($api['array']['disks'] ?? [] as $d) {
|
||||
$entry = vv_api_disk_entry($d, 'data');
|
||||
if ($entry) $data[] = $entry;
|
||||
}
|
||||
if (!empty($parity) || !empty($data)) {
|
||||
usort($parity, fn($a, $b) => strnatcmp($a['name'], $b['name']));
|
||||
usort($data, fn($a, $b) => strnatcmp($a['name'], $b['name']));
|
||||
return array_merge($parity, $data);
|
||||
}
|
||||
}
|
||||
|
||||
// ── Local fallback ────────────────────────────────────────────────────────
|
||||
vv_api_record_fallback('array_disks');
|
||||
$ini = @parse_ini_file('/var/local/emhttp/disks.ini', true) ?: [];
|
||||
$parity = [];
|
||||
$data = [];
|
||||
foreach ($ini as $key => $d) {
|
||||
$type = $d['type'] ?? '';
|
||||
if ($type === 'Parity') {
|
||||
$entry = vv_disk_entry($d, $key, 'parity');
|
||||
if ($entry) $parity[] = $entry;
|
||||
} elseif ($type === 'Data') {
|
||||
$entry = vv_disk_entry($d, $key, 'data');
|
||||
if ($entry) $data[] = $entry;
|
||||
}
|
||||
}
|
||||
usort($parity, fn($a, $b) => strnatcmp($a['name'], $b['name']));
|
||||
usort($data, fn($a, $b) => strnatcmp($a['name'], $b['name']));
|
||||
return array_merge($parity, $data);
|
||||
}
|
||||
|
||||
function vv_disk_io_rates(): array {
|
||||
$snapFile = VV_CACHE_DIR . '/vv_diskio_snap.json';
|
||||
$now = microtime(true);
|
||||
|
||||
// Read current whole-disk stats from /proc/diskstats
|
||||
$current = [];
|
||||
foreach (@file('/proc/diskstats', FILE_IGNORE_NEW_LINES) ?: [] as $line) {
|
||||
$p = preg_split('/\s+/', trim($line));
|
||||
if (count($p) < 14) continue;
|
||||
$dev = $p[2];
|
||||
// Keep only whole disks: sda/sdb, nvme0n1, md*, not sda1/nvme0n1p1
|
||||
if (!preg_match('/^(sd[a-z]+|nvme\d+n\d+|md\d+)$/', $dev)) continue;
|
||||
$current[$dev] = [(int)$p[5], (int)$p[9]]; // [sectors_read, sectors_written]
|
||||
}
|
||||
|
||||
// Load previous snapshot
|
||||
$snap = @json_decode(@file_get_contents($snapFile) ?: '', true) ?: [];
|
||||
$prevTime = (float)($snap['t'] ?? $now);
|
||||
$prev = $snap['d'] ?? [];
|
||||
|
||||
// Save current snapshot
|
||||
@file_put_contents($snapFile, json_encode(['t' => $now, 'd' => $current], JSON_UNESCAPED_UNICODE));
|
||||
|
||||
$dt = max(0.5, $now - $prevTime);
|
||||
$out = [];
|
||||
foreach ($current as $dev => [$rs, $ws]) {
|
||||
$entry = [
|
||||
'tr' => round($rs * 512 / 1073741824, 2), // cumulative GB read
|
||||
'tw' => round($ws * 512 / 1073741824, 2), // cumulative GB written
|
||||
];
|
||||
if (isset($prev[$dev])) {
|
||||
[$prs, $pws] = $prev[$dev];
|
||||
$r = max(0.0, ($rs - $prs) * 512 / $dt / 1048576);
|
||||
$w = max(0.0, ($ws - $pws) * 512 / $dt / 1048576);
|
||||
if ($r > 0.01) $entry['r'] = round($r, 1);
|
||||
if ($w > 0.01) $entry['w'] = round($w, 1);
|
||||
}
|
||||
$out[$dev] = $entry;
|
||||
}
|
||||
return $out;
|
||||
}
|
||||
|
||||
function vv_disk_thresholds(): array {
|
||||
$cfg = @file_get_contents('/boot/config/plugins/dynamix/dynamix.cfg') ?: '';
|
||||
$get = function(string $key) use ($cfg): ?int {
|
||||
return preg_match('/^\s*' . preg_quote($key, '/') . '\s*=\s*"?(\d+)"?/m', $cfg, $m)
|
||||
? (int)$m[1] : null;
|
||||
};
|
||||
return [
|
||||
'util_warn' => $get('warning') ?? 70,
|
||||
'util_crit' => $get('critical') ?? 90,
|
||||
'hdd_warn' => $get('hot') ?? 45,
|
||||
'hdd_crit' => $get('max') ?? 55,
|
||||
'ssd_warn' => $get('hotssd') ?? 60,
|
||||
'ssd_crit' => $get('maxssd') ?? 70,
|
||||
];
|
||||
}
|
||||
|
||||
// Fetch a lightweight snapshot from each remote host that has an API key configured.
|
||||
// Results are cached in /tmp for 30 seconds so rapid monitor polls don't hammer remote hosts.
|
||||
function vv_remote_hosts_stats(): array {
|
||||
// Read ALL conf files — remote host keys live in their own host*.conf, not the current host's.
|
||||
$vars = vv_conf_vars();
|
||||
foreach (glob(CONF_DIR . '/host*.conf') ?: [] as $f) {
|
||||
$raw = file_get_contents($f) ?: '';
|
||||
preg_match_all('/^\s*([A-Z0-9_]+)\s*=\s*["\']?([^"\'#\n]*?)["\']?\s*(?:#.*)?$/m', $raw, $m);
|
||||
foreach ($m[1] as $i => $key) {
|
||||
if (!isset($vars[$key])) $vars[$key] = trim($m[2][$i]);
|
||||
}
|
||||
}
|
||||
$myHost = vv_detect_host();
|
||||
$hostIds = array_filter(array_keys($vars), fn($k) => preg_match('/^HOST\d+$/', $k) && ($vars[$k] ?? '') !== '');
|
||||
sort($hostIds);
|
||||
|
||||
$results = [];
|
||||
foreach ($hostIds as $id) {
|
||||
if (strtolower($id) === strtolower($myHost)) continue;
|
||||
// Background cache written by remote_arr_cache_writer.sh every 2h — use it if present.
|
||||
$bgCache = VV_CACHE_DIR . '/monitor_remote_' . strtolower($id) . '.json';
|
||||
if (file_exists($bgCache)) {
|
||||
$cached = json_decode(file_get_contents($bgCache), true);
|
||||
if ($cached) {
|
||||
$cached['cache_age'] = time() - (int)filemtime($bgCache);
|
||||
$results[$id] = $cached;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// No background cache yet — fall back to live call (uses 30s inline cache).
|
||||
$key = $vars[strtoupper($id) . '_UNRAID_API_KEY'] ?? '';
|
||||
if (!$key) {
|
||||
$results[$id] = ['available' => false, 'no_api_key' => true,
|
||||
'host_id' => $id, 'hostname' => $vars[$id]];
|
||||
continue;
|
||||
}
|
||||
|
||||
$cacheFile = VV_CACHE_DIR . "/vv_remote_{$id}.json";
|
||||
if (file_exists($cacheFile) && (time() - filemtime($cacheFile)) < 30) {
|
||||
$cached = json_decode(file_get_contents($cacheFile), true);
|
||||
if ($cached) { $results[$id] = $cached; continue; }
|
||||
}
|
||||
|
||||
$gql = '{
|
||||
info { os { hostname uptime release } cpu { brand threads cores } }
|
||||
metrics { cpu { percentTotal } memory { percentTotal total used available } }
|
||||
array {
|
||||
state
|
||||
disks { fsSize fsUsed temp }
|
||||
caches { fsSize fsUsed temp }
|
||||
parities { temp }
|
||||
}
|
||||
vms { domains { name } }
|
||||
}';
|
||||
$data = vv_unraid_api_query(strtolower($id), $gql, 4, $key);
|
||||
|
||||
if (!$data) {
|
||||
$entry = ['available' => false, 'host_id' => $id, 'hostname' => $vars[$id]];
|
||||
file_put_contents($cacheFile, json_encode($entry));
|
||||
$results[$id] = $entry;
|
||||
continue;
|
||||
}
|
||||
|
||||
$os = $data['info']['os'] ?? [];
|
||||
$cpu = $data['info']['cpu'] ?? [];
|
||||
$mMem = $data['metrics']['memory'] ?? [];
|
||||
|
||||
$memPct = round((float)($mMem['percentTotal'] ?? 0));
|
||||
if ($memPct === 0) {
|
||||
$totalBytes = (float)($mMem['total'] ?? 0);
|
||||
$availBytes = (float)($mMem['available'] ?? 0);
|
||||
$memPct = $totalBytes > 0 ? (int)round(($totalBytes - $availBytes) / $totalBytes * 100) : 0;
|
||||
}
|
||||
$memTotalGb = isset($mMem['total']) ? _vv_api_bytes_to_gb((float)$mMem['total']) : 0;
|
||||
|
||||
$uptimeRaw = $os['uptime'] ?? '';
|
||||
if (is_numeric($uptimeRaw)) {
|
||||
$uptimeSec = (int)$uptimeRaw;
|
||||
$days = intdiv($uptimeSec, 86400);
|
||||
$hours = intdiv($uptimeSec % 86400, 3600);
|
||||
$mins = intdiv($uptimeSec % 3600, 60);
|
||||
$uptime = ($days ? "{$days}d " : '') . ($hours ? "{$hours}h " : '') . "{$mins}m";
|
||||
} else {
|
||||
$uptimeSec = 0;
|
||||
$uptime = $uptimeRaw ?: '—';
|
||||
}
|
||||
|
||||
$nodeMetrics = vv_api_node_metrics($data);
|
||||
$entry = array_merge([
|
||||
'available' => true,
|
||||
'host_id' => $id,
|
||||
'hostname' => $os['hostname'] ?? $vars[$id],
|
||||
'version' => $os['release'] ?? '',
|
||||
'uptime' => $uptime,
|
||||
'uptime_sec' => $uptimeSec,
|
||||
'cpu_load' => $nodeMetrics['cpu_pct'] ?? 0,
|
||||
'cpu_threads' => (int)($cpu['threads'] ?? 0),
|
||||
'mem_total_gb' => $memTotalGb,
|
||||
'mem_used_pct' => $memPct,
|
||||
'array_state' => $data['array']['state'] ?? 'UNKNOWN',
|
||||
], $nodeMetrics);
|
||||
file_put_contents($cacheFile, json_encode($entry));
|
||||
$results[$id] = $entry;
|
||||
}
|
||||
return $results;
|
||||
}
|
||||
|
||||
function vv_log_tail(string $path, int $lines): string {
|
||||
$fp = @fopen($path, 'r');
|
||||
if (!$fp) return '';
|
||||
fseek($fp, 0, SEEK_END);
|
||||
$size = ftell($fp);
|
||||
if ($size <= 0) { fclose($fp); return ''; }
|
||||
$chunk = min($size, 4096);
|
||||
fseek($fp, -$chunk, SEEK_END);
|
||||
$data = fread($fp, $chunk);
|
||||
fclose($fp);
|
||||
$all = explode("\n", $data ?: '');
|
||||
return implode("\n", array_slice($all, -$lines));
|
||||
}
|
||||
|
||||
function vv_parse_bash_array(string $raw, string $varName): array {
|
||||
if (!preg_match('/^\s*' . preg_quote($varName, '/') . '\s*=\s*\(([^)]*)\)/ms', $raw, $m)) return [];
|
||||
$items = [];
|
||||
foreach (explode("\n", $m[1]) as $line) {
|
||||
$line = trim(preg_replace('/#.*$/', '', $line), " \t\"'");
|
||||
if ($line !== '') $items[] = $line;
|
||||
}
|
||||
return $items;
|
||||
}
|
||||
|
||||
function vv_transcode_sessions(): array {
|
||||
$v = vv_conf_vars();
|
||||
$stateDir = rtrim($v['STATE_DIR'] ?? STATE_DIR, '/');
|
||||
$stateFile = "$stateDir/transcode_state.db";
|
||||
if (!file_exists($stateFile)) return ['available' => false];
|
||||
|
||||
$raw = [];
|
||||
foreach (file($stateFile) ?: [] as $line) {
|
||||
[$k, $v] = array_pad(explode('=', trim($line), 2), 2, '');
|
||||
$raw[trim($k)] = trim($v);
|
||||
}
|
||||
|
||||
$target = $raw['current_target'] ?? '';
|
||||
$lastFlip = (int)($raw['last_flip_time'] ?? 0);
|
||||
$flipCount = (int)($raw['flip_count_hour'] ?? 0);
|
||||
$isRamdisk = str_contains($target, 'ramdisk');
|
||||
|
||||
// Count active session dirs in both known locations
|
||||
$ramdiskPath = '/mnt/ramdisk_transcodes/transcoding-temp';
|
||||
$ramSessions = count(glob("$ramdiskPath/*/", GLOB_ONLYDIR) ?: []);
|
||||
|
||||
// SSD path: first transcoding-temp mount that is not a RAM filesystem (tmpfs/ramfs)
|
||||
$ssdPath = '';
|
||||
$ssdSessions = 0;
|
||||
foreach (glob('/mnt/*/transcoding-temp/', GLOB_ONLYDIR) ?: [] as $p) {
|
||||
$parts = explode('/', rtrim($p, '/'));
|
||||
array_pop($parts);
|
||||
$mount = implode('/', $parts) ?: '/';
|
||||
$fsType = trim(shell_exec('findmnt -n -o FSTYPE ' . escapeshellarg($mount) . ' 2>/dev/null') ?: '');
|
||||
if ($fsType === 'tmpfs' || $fsType === 'ramfs') continue;
|
||||
$ssdPath = $p;
|
||||
break;
|
||||
}
|
||||
$ssd = ['available' => false];
|
||||
if ($ssdPath) {
|
||||
$ssdSessions = count(glob($ssdPath . '/*/', GLOB_ONLYDIR) ?: []);
|
||||
$parts = explode('/', rtrim($ssdPath, '/'));
|
||||
array_pop($parts);
|
||||
$ssdMount = implode('/', $parts) ?: '/';
|
||||
$ssd = vv_df($ssdMount);
|
||||
}
|
||||
|
||||
// Ramdisk disk usage
|
||||
$rd = vv_df('/mnt/ramdisk_transcodes');
|
||||
|
||||
// Last cleanup values from transcode management log
|
||||
$lastRdFreed = null;
|
||||
$lastSsdFreed = null;
|
||||
$logFile = LOG_DIR . '/Orchestrators/transcode_management.log';
|
||||
if (file_exists($logFile)) {
|
||||
$lines = file($logFile, FILE_IGNORE_NEW_LINES) ?: [];
|
||||
foreach (array_reverse($lines) as $line) {
|
||||
if ($lastRdFreed === null && preg_match('/Ramdisk freed:\s*(\S+)/u', $line, $m))
|
||||
$lastRdFreed = $m[1];
|
||||
if ($lastSsdFreed === null && preg_match('/SSD freed:\s*(\S+)/u', $line, $m))
|
||||
$lastSsdFreed = $m[1];
|
||||
if ($lastRdFreed !== null && $lastSsdFreed !== null) break;
|
||||
}
|
||||
}
|
||||
|
||||
return [
|
||||
'available' => true,
|
||||
'current_target' => $target,
|
||||
'is_ramdisk' => $isRamdisk,
|
||||
'flip_count_hour' => $flipCount,
|
||||
'last_flip_time' => $lastFlip,
|
||||
'last_flip_ago' => $lastFlip > 0 ? time() - $lastFlip : null,
|
||||
'ram_sessions' => $ramSessions,
|
||||
'ssd_sessions' => $ssdSessions,
|
||||
'ramdisk' => $rd,
|
||||
'ssd' => $ssd,
|
||||
'last_rd_freed' => $lastRdFreed,
|
||||
'last_ssd_freed' => $lastSsdFreed,
|
||||
];
|
||||
}
|
||||
@@ -0,0 +1,823 @@
|
||||
<?php
|
||||
require_once __DIR__ . '/config.php';
|
||||
require_once __DIR__ . '/unraid_api.php';
|
||||
|
||||
// Common helpers shared across all Varaverk pages.
|
||||
|
||||
function vv_system_info(): array {
|
||||
// ── Shared local reads (always needed regardless of API) ──────────────────
|
||||
$ident = @parse_ini_file('/boot/config/ident.cfg') ?: [];
|
||||
$var = [];
|
||||
foreach (@file('/var/local/emhttp/var.ini') ?: [] as $line) {
|
||||
if (preg_match('/^(\w+)="([^"]*)"/', $line, $m)) $var[$m[1]] = $m[2];
|
||||
}
|
||||
$version = trim(@file_get_contents('/etc/unraid-version') ?: '');
|
||||
|
||||
// ── API path ──────────────────────────────────────────────────────────────
|
||||
$api = vv_api_data();
|
||||
if ($api) {
|
||||
$os = $api['info']['os'] ?? [];
|
||||
$cpu = $api['info']['cpu'] ?? [];
|
||||
|
||||
// uptime is a String in this schema — try numeric (seconds) first, else display as-is
|
||||
$uptimeRaw = $os['uptime'] ?? '';
|
||||
if (is_numeric($uptimeRaw)) {
|
||||
$uptimeSec = (int)$uptimeRaw;
|
||||
$uptime = vv_format_uptime($uptimeSec);
|
||||
} else {
|
||||
$uptimeSec = 0;
|
||||
$uptime = $uptimeRaw ?: '—';
|
||||
}
|
||||
|
||||
$load = sys_getloadavg();
|
||||
return [
|
||||
'name' => $os['hostname'] ?? ($ident['NAME'] ?? gethostname()),
|
||||
'comment' => $ident['COMMENT'] ?? '',
|
||||
'timezone' => $ident['timeZone'] ?? 'UTC',
|
||||
'cpu_model' => $cpu['brand'] ?? ($ident['SYS_MODEL'] ?? ''),
|
||||
'cpu_threads' => (int)($cpu['threads'] ?? 0),
|
||||
'cpu_cores' => (int)($cpu['cores'] ?? 0),
|
||||
'reg_type' => 'Unraid OS ' . ($var['regTy'] ?? ''),
|
||||
'reg_to' => $var['regTo'] ?? '',
|
||||
'uptime' => $uptime,
|
||||
'uptime_sec' => $uptimeSec,
|
||||
'array_state' => strtoupper($api['array']['state'] ?? $var['mdState'] ?? 'UNKNOWN'),
|
||||
'version' => trim($os['release'] ?? '') ?: $version,
|
||||
'load_avg' => $load ? [round($load[0], 2), round($load[1], 2), round($load[2], 2)] : null,
|
||||
];
|
||||
}
|
||||
|
||||
// ── Local fallback ────────────────────────────────────────────────────────
|
||||
vv_api_record_fallback('system_info');
|
||||
$cpuModel = '';
|
||||
foreach (@file('/proc/cpuinfo') ?: [] as $line) {
|
||||
if (preg_match('/^model name\s*:\s*(.+)/', $line, $m)) { $cpuModel = trim($m[1]); break; }
|
||||
}
|
||||
$uptimeSec = (int)explode(' ', @file_get_contents('/proc/uptime') ?: '0')[0];
|
||||
$uptime = vv_format_uptime($uptimeSec);
|
||||
|
||||
$load = sys_getloadavg();
|
||||
return [
|
||||
'name' => $ident['NAME'] ?? gethostname(),
|
||||
'comment' => $ident['COMMENT'] ?? '',
|
||||
'timezone' => $ident['timeZone'] ?? 'UTC',
|
||||
'cpu_model' => $ident['SYS_MODEL'] ?? $cpuModel,
|
||||
'cpu_threads' => 0,
|
||||
'cpu_cores' => 0,
|
||||
'reg_type' => 'Unraid OS ' . ($var['regTy'] ?? ''),
|
||||
'reg_to' => $var['regTo'] ?? '',
|
||||
'uptime' => $uptime,
|
||||
'uptime_sec' => $uptimeSec,
|
||||
'array_state' => $var['mdState'] ?? 'UNKNOWN',
|
||||
'version' => $version,
|
||||
'load_avg' => $load ? [round($load[0], 2), round($load[1], 2), round($load[2], 2)] : null,
|
||||
];
|
||||
}
|
||||
|
||||
function vv_docker_containers(): array {
|
||||
$out = shell_exec('docker ps --format \'{"name":"{{.Names}}","status":"{{.Status}}","image":"{{.Image}}"}\' 2>/dev/null');
|
||||
$containers = [];
|
||||
foreach (explode("\n", trim($out ?? '')) as $line) {
|
||||
if (!$line) continue;
|
||||
$c = json_decode($line, true);
|
||||
if ($c) $containers[] = $c;
|
||||
}
|
||||
return $containers;
|
||||
}
|
||||
|
||||
function vv_docker_stopped(): array {
|
||||
$out = shell_exec('docker ps -a --filter "status=exited" --filter "status=created" --format \'{"name":"{{.Names}}","status":"{{.Status}}"}\' 2>/dev/null');
|
||||
$containers = [];
|
||||
foreach (explode("\n", trim($out ?? '')) as $line) {
|
||||
if (!$line) continue;
|
||||
$c = json_decode($line, true);
|
||||
if ($c) $containers[] = $c;
|
||||
}
|
||||
return $containers;
|
||||
}
|
||||
|
||||
function vv_gpu_stats(): array {
|
||||
$out = shell_exec('nvidia-smi --query-gpu=name,memory.used,memory.total,utilization.gpu,temperature.gpu,power.draw,utilization.encoder,utilization.decoder --format=csv,noheader,nounits 2>/dev/null');
|
||||
if (!$out) return ['available' => false];
|
||||
|
||||
$parts = array_map('trim', explode(',', $out));
|
||||
$power = is_numeric($parts[5] ?? '') ? round((float)$parts[5], 1) : null;
|
||||
return [
|
||||
'available' => true,
|
||||
'name' => $parts[0] ?? '',
|
||||
'memory_used' => (int)($parts[1] ?? 0),
|
||||
'memory_total' => (int)($parts[2] ?? 0),
|
||||
'utilization' => (int)($parts[3] ?? 0),
|
||||
'temperature' => (int)($parts[4] ?? 0),
|
||||
'power_w' => $power,
|
||||
'enc_pct' => (int)($parts[6] ?? 0),
|
||||
'dec_pct' => (int)($parts[7] ?? 0),
|
||||
];
|
||||
}
|
||||
|
||||
function vv_gpu_processes(): array {
|
||||
$out = shell_exec('nvidia-smi --query-compute-apps=pid,used_gpu_memory,name --format=csv,noheader,nounits 2>/dev/null');
|
||||
$procs = [];
|
||||
foreach (explode("\n", trim($out ?? '')) as $line) {
|
||||
if (!$line) continue;
|
||||
$parts = array_map('trim', explode(',', $line));
|
||||
$procs[] = [
|
||||
'pid' => $parts[0] ?? '',
|
||||
'memory_mb' => $parts[1] ?? '',
|
||||
'name' => $parts[2] ?? '',
|
||||
];
|
||||
}
|
||||
return $procs;
|
||||
}
|
||||
|
||||
function vv_system_resources(): array {
|
||||
$mem = [];
|
||||
foreach (file('/proc/meminfo') ?: [] as $line) {
|
||||
if (preg_match('/^(MemTotal|MemAvailable):\s+(\d+)/', $line, $m))
|
||||
$mem[$m[1]] = (int)$m[2];
|
||||
}
|
||||
return [
|
||||
'ram_total_mb' => (int)(($mem['MemTotal'] ?? 0) / 1024),
|
||||
'ram_free_mb' => (int)(($mem['MemAvailable'] ?? 0) / 1024),
|
||||
'cache' => vv_df('/mnt/cache'),
|
||||
];
|
||||
}
|
||||
|
||||
function vv_cpu_per_core(): array {
|
||||
// Parse /proc/stat — [user, nice, system, idle, iowait, irq, softirq]
|
||||
$raw = [];
|
||||
foreach (file('/proc/stat') ?: [] as $line) {
|
||||
if (!preg_match('/^(cpu\d*)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)/', $line, $m)) continue;
|
||||
$raw[$m[1]] = [(int)$m[2],(int)$m[3],(int)$m[4],(int)$m[5],(int)$m[6],(int)$m[7],(int)$m[8]];
|
||||
}
|
||||
|
||||
$stateFile = VV_CACHE_DIR . '/vv_cpu_stat.json';
|
||||
$prev = file_exists($stateFile) ? (json_decode(file_get_contents($stateFile), true) ?: []) : [];
|
||||
// Atomic write — concurrent fast/slow polls read a consistent snapshot
|
||||
$tmp = $stateFile . '.tmp';
|
||||
file_put_contents($tmp, json_encode($raw));
|
||||
rename($tmp, $stateFile);
|
||||
|
||||
$usage = function(array $c, ?array $p): int {
|
||||
if (!$p) return 0;
|
||||
$dt = array_sum($c) - array_sum($p);
|
||||
$di = ($c[3] + $c[4]) - ($p[3] + $p[4]);
|
||||
return $dt > 0 ? max(0, min(100, (int)round((1 - $di / $dt) * 100))) : 0;
|
||||
};
|
||||
|
||||
$overall = $usage($raw['cpu'] ?? [], $prev['cpu'] ?? null);
|
||||
$cores = [];
|
||||
foreach ($raw as $cpu => $c) {
|
||||
if ($cpu === 'cpu') continue;
|
||||
$num = (int)substr($cpu, 3);
|
||||
$freqKhz = (int)@file_get_contents("/sys/devices/system/cpu/$cpu/cpufreq/scaling_cur_freq");
|
||||
$maxKhz = (int)@file_get_contents("/sys/devices/system/cpu/$cpu/cpufreq/cpuinfo_max_freq");
|
||||
$minKhz = (int)@file_get_contents("/sys/devices/system/cpu/$cpu/cpufreq/cpuinfo_min_freq");
|
||||
$cores[] = [
|
||||
'core' => $num,
|
||||
'usage_pct' => $usage($c, $prev[$cpu] ?? null),
|
||||
'freq_mhz' => $freqKhz > 0 ? (int)round($freqKhz / 1000) : 0,
|
||||
'max_mhz' => $maxKhz > 0 ? (int)round($maxKhz / 1000) : 0,
|
||||
'min_mhz' => $minKhz > 0 ? (int)round($minKhz / 1000) : 0,
|
||||
];
|
||||
}
|
||||
usort($cores, fn($a, $b) => $a['core'] - $b['core']);
|
||||
return ['overall' => $overall, 'cores' => $cores];
|
||||
}
|
||||
|
||||
function vv_memory_breakdown(): array {
|
||||
$mem = [];
|
||||
foreach (file('/proc/meminfo') ?: [] as $line) {
|
||||
if (preg_match('/^(\w+):\s+(\d+)/', $line, $m)) $mem[$m[1]] = (int)$m[2];
|
||||
}
|
||||
$totalKb = $mem['MemTotal'] ?? 0;
|
||||
|
||||
// ZFS ARC
|
||||
$arcKb = 0;
|
||||
foreach (@file('/proc/spl/kstat/zfs/arcstats') ?: [] as $line) {
|
||||
if (preg_match('/^size\s+\d+\s+(\d+)/', $line, $m)) { $arcKb = (int)($m[1] / 1024); break; }
|
||||
}
|
||||
|
||||
// Docker — sum docker stats used memory per container (matches Unraid dashboard)
|
||||
$dockerKb = 0;
|
||||
$dsOut = shell_exec("docker stats --no-stream --format '{{.MemUsage}}' 2>/dev/null") ?: '';
|
||||
foreach (explode("\n", trim($dsOut)) as $line) {
|
||||
if (!preg_match('/^([0-9.]+)(GiB|MiB|KiB|B)\s*\//', trim($line), $m)) continue;
|
||||
$val = (float)$m[1];
|
||||
$dockerKb += match($m[2]) {
|
||||
'GiB' => (int)($val * 1048576),
|
||||
'MiB' => (int)($val * 1024),
|
||||
'KiB' => (int)$val,
|
||||
default => (int)($val / 1024),
|
||||
};
|
||||
}
|
||||
|
||||
// VM (QEMU/KVM RSS)
|
||||
$vmKb = 0;
|
||||
foreach (preg_split('/\s+/', trim(shell_exec('ps -C qemu-system-x86_64 -o rss= 2>/dev/null') ?: '')) as $rss) {
|
||||
if (is_numeric($rss) && $rss > 0) $vmKb += (int)$rss;
|
||||
}
|
||||
|
||||
$freeKb = max(0, $mem['MemAvailable'] ?? 0);
|
||||
$systemKb = max(0, $totalKb - $freeKb - $arcKb - $dockerKb - $vmKb);
|
||||
|
||||
// Top processes by RSS — group same-named procs, take top 5
|
||||
$grouped = [];
|
||||
$psOut = shell_exec("ps -eo comm,rss --sort=-rss 2>/dev/null | tail -n +2 | head -40") ?: '';
|
||||
foreach (explode("\n", trim($psOut)) as $line) {
|
||||
$parts = preg_split('/\s+/', trim($line), 2);
|
||||
if (count($parts) === 2 && is_numeric($parts[1]) && (int)$parts[1] > 0)
|
||||
$grouped[$parts[0]] = ($grouped[$parts[0]] ?? 0) + (int)$parts[1];
|
||||
}
|
||||
arsort($grouped);
|
||||
$topProcs = [];
|
||||
foreach (array_slice($grouped, 0, 3, true) as $name => $kb)
|
||||
$topProcs[] = ['name' => $name, 'kb' => $kb];
|
||||
|
||||
// Swap — from API metrics when available, else /proc/meminfo
|
||||
$swapTotalKb = 0; $swapUsedKb = 0;
|
||||
$apiMem = vv_api_data()['metrics']['memory'] ?? [];
|
||||
if (!empty($apiMem['swapTotal'])) {
|
||||
$swapTotalKb = (int)(((float)$apiMem['swapTotal']) / 1024);
|
||||
$swapUsedKb = (int)(((float)$apiMem['swapUsed']) / 1024);
|
||||
} else {
|
||||
$swapTotalKb = $mem['SwapTotal'] ?? 0;
|
||||
$swapUsedKb = ($mem['SwapTotal'] ?? 0) - ($mem['SwapFree'] ?? 0);
|
||||
}
|
||||
|
||||
return [
|
||||
'total_kb' => $totalKb,
|
||||
'system_kb' => $systemKb,
|
||||
'vm_kb' => $vmKb,
|
||||
'zfs_kb' => $arcKb,
|
||||
'docker_kb' => $dockerKb,
|
||||
'free_kb' => $freeKb,
|
||||
'swap_total_kb' => $swapTotalKb,
|
||||
'swap_used_kb' => $swapUsedKb,
|
||||
'top_procs' => $topProcs,
|
||||
];
|
||||
}
|
||||
|
||||
function vv_df(string $path): array {
|
||||
$out = shell_exec("df -BM --output=size,used,avail '$path' 2>/dev/null | tail -1");
|
||||
if (!$out) return ['available' => false, 'path' => $path];
|
||||
$parts = preg_split('/\s+/', trim($out));
|
||||
return [
|
||||
'available' => true,
|
||||
'path' => $path,
|
||||
'size_mb' => (int)$parts[0],
|
||||
'used_mb' => (int)$parts[1],
|
||||
'free_mb' => (int)$parts[2],
|
||||
];
|
||||
}
|
||||
|
||||
function vv_network_stats(): array {
|
||||
$iface = trim(shell_exec("ip route show default 2>/dev/null | awk 'NR==1{print \$5}'") ?: '');
|
||||
if (!$iface) {
|
||||
$best = ''; $bestBytes = 0;
|
||||
foreach (file('/proc/net/dev') ?: [] as $line) {
|
||||
if (!preg_match('/^\s*(\w+):\s+(\d+)/', $line, $m) || $m[1] === 'lo') continue;
|
||||
if ((int)$m[2] > $bestBytes) { $bestBytes = (int)$m[2]; $best = $m[1]; }
|
||||
}
|
||||
$iface = $best;
|
||||
}
|
||||
if (!$iface) return ['available' => false];
|
||||
|
||||
$rxBytes = $txBytes = 0;
|
||||
foreach (file('/proc/net/dev') ?: [] as $line) {
|
||||
if (!preg_match('/^\s*' . preg_quote($iface, '/') . ':\s+(.+)$/', $line, $m)) continue;
|
||||
$parts = preg_split('/\s+/', trim($m[1]));
|
||||
$rxBytes = (int)($parts[0] ?? 0);
|
||||
$txBytes = (int)($parts[8] ?? 0);
|
||||
break;
|
||||
}
|
||||
|
||||
$stateFile = VV_CACHE_DIR . '/vv_net_stat.json';
|
||||
$now = ['rx' => $rxBytes, 'tx' => $txBytes, 'ts' => microtime(true)];
|
||||
$prev = file_exists($stateFile) ? (json_decode(file_get_contents($stateFile), true) ?: []) : [];
|
||||
$tmp = $stateFile . '.tmp';
|
||||
file_put_contents($tmp, json_encode($now));
|
||||
rename($tmp, $stateFile);
|
||||
|
||||
$rxRate = $txRate = 0;
|
||||
if (!empty($prev['ts']) && ($dt = $now['ts'] - $prev['ts']) > 0.1) {
|
||||
$rxRate = max(0, (int)(($rxBytes - ($prev['rx'] ?? $rxBytes)) / $dt));
|
||||
$txRate = max(0, (int)(($txBytes - ($prev['tx'] ?? $txBytes)) / $dt));
|
||||
}
|
||||
|
||||
$speedMbps = (int)@file_get_contents("/sys/class/net/$iface/speed");
|
||||
|
||||
// Local IP — use primary iface
|
||||
$localIp = trim(shell_exec(
|
||||
"ip -4 addr show " . escapeshellarg($iface) . " 2>/dev/null | awk '/inet /{print \$2}' | cut -d/ -f1 | head -1"
|
||||
) ?: '');
|
||||
|
||||
// External IP — curl ifconfig.me, cached 5 min so we don't hammer it
|
||||
$extIp = '';
|
||||
$extData = vv_cache_read('ext_ip', 300);
|
||||
if ($extData) {
|
||||
$extIp = $extData['ip'] ?? '';
|
||||
} else {
|
||||
$fetched = trim(shell_exec('curl -sf --max-time 4 https://ifconfig.me 2>/dev/null') ?: '');
|
||||
if (preg_match('/^\d+\.\d+\.\d+\.\d+$/', $fetched)) {
|
||||
$extIp = $fetched;
|
||||
vv_cache_write('ext_ip', ['ip' => $extIp]);
|
||||
}
|
||||
}
|
||||
|
||||
// Tailscale IP — use `tailscale ip` CLI (interface name varies: tailscale0, tailscale1, etc.)
|
||||
$tsIp = trim(shell_exec('tailscale ip -4 2>/dev/null | head -1') ?: '');
|
||||
|
||||
return [
|
||||
'available' => true,
|
||||
'iface' => $iface,
|
||||
'speed_mbps' => $speedMbps > 0 ? $speedMbps : null,
|
||||
'rx_bps' => $rxRate,
|
||||
'tx_bps' => $txRate,
|
||||
'local_ip' => $localIp,
|
||||
'ext_ip' => $extIp,
|
||||
'ts_ip' => $tsIp,
|
||||
];
|
||||
}
|
||||
|
||||
function vv_disk_entry(array $d, string $key, string $role = 'data'): ?array {
|
||||
$name = $d['name'] ?? $key;
|
||||
$isParity = $role === 'parity';
|
||||
$mounted = ($d['fsStatus'] ?? '') === 'Mounted';
|
||||
// Parity has no filesystem — use raw size only
|
||||
$size_kb = (int)($isParity ? ($d['size'] ?? 0) : ($mounted ? ($d['fsSize'] ?? 0) : ($d['size'] ?? 0)));
|
||||
$used_kb = (int)($isParity ? 0 : ($mounted ? ($d['fsUsed'] ?? 0) : 0));
|
||||
if ($size_kb <= 0) return null;
|
||||
$tempRaw = trim($d['temp'] ?? '');
|
||||
return [
|
||||
'name' => $name,
|
||||
'device' => $d['device'] ?? $key,
|
||||
'role' => $role,
|
||||
'size_gb' => round($size_kb / 1048576, 1),
|
||||
'used_gb' => round($used_kb / 1048576, 1),
|
||||
'pct' => (!$isParity && $size_kb > 0) ? round($used_kb / $size_kb * 100, 1) : null,
|
||||
'temp' => is_numeric($tempRaw) ? (int)$tempRaw : null,
|
||||
'transport' => $d['transport'] ?? 'ata',
|
||||
'mounted' => $mounted,
|
||||
'status' => $d['status'] ?? '',
|
||||
];
|
||||
}
|
||||
|
||||
function vv_ups_stats(): array {
|
||||
$raw = shell_exec('apcaccess 2>/dev/null') ?: '';
|
||||
if (!$raw) return ['available' => false];
|
||||
|
||||
$fields = [];
|
||||
foreach (explode("\n", $raw) as $line) {
|
||||
if (preg_match('/^(\w+)\s*:\s*(.+)$/', trim($line), $m)) {
|
||||
$fields[trim($m[1])] = trim($m[2]);
|
||||
}
|
||||
}
|
||||
if (empty($fields)) return ['available' => false];
|
||||
|
||||
$parse_num = fn(string $k) => isset($fields[$k]) ? (float)$fields[$k] : null;
|
||||
|
||||
$loadPct = $parse_num('LOADPCT');
|
||||
$nomPower = $parse_num('NOMPOWER');
|
||||
$watts = ($loadPct !== null && $nomPower !== null) ? round($loadPct / 100 * $nomPower) : null;
|
||||
|
||||
return [
|
||||
'available' => true,
|
||||
'model' => $fields['MODEL'] ?? '',
|
||||
'status' => trim(explode(' ', $fields['STATUS'] ?? 'UNKNOWN')[0]),
|
||||
'line_v' => $parse_num('LINEV'),
|
||||
'output_v' => $parse_num('OUTPUTV'),
|
||||
'load_pct' => $loadPct,
|
||||
'nom_power' => $nomPower,
|
||||
'watts' => $watts,
|
||||
'bcharge' => $parse_num('BCHARGE'),
|
||||
'timeleft' => $parse_num('TIMELEFT'),
|
||||
'num_xfers' => (int)($fields['NUMXFERS'] ?? 0),
|
||||
'on_batt_s' => $parse_num('CUMONBATT'),
|
||||
'selftest' => $fields['SELFTEST'] ?? '',
|
||||
];
|
||||
}
|
||||
|
||||
function vv_parity_status(): array {
|
||||
$var = [];
|
||||
foreach (@file('/var/local/emhttp/var.ini') ?: [] as $line) {
|
||||
if (preg_match('/^(\w+)="([^"]*)"/', $line, $m)) $var[$m[1]] = $m[2];
|
||||
}
|
||||
|
||||
$numDisabled = (int)($var['mdNumDisabled'] ?? 0);
|
||||
$numMissing = (int)($var['mdNumMissing'] ?? 0);
|
||||
$exitCode = (int)($var['sbSyncExit'] ?? 0);
|
||||
$errors = (int)($var['sbSyncErrs'] ?? 0);
|
||||
// Emulated (DISK_DSBL) disks are protected by parity and don't make parity invalid.
|
||||
// True invalidity: sync errors on the last check, or unprotectable missing slots.
|
||||
$isValid = $errors === 0 && $numMissing === 0;
|
||||
$inProgress = ($var['mdResync'] ?? '0') !== '0';
|
||||
$resyncPos = (int)($var['mdResyncPos'] ?? 0);
|
||||
$resyncSize = (int)($var['mdResyncSize'] ?? 1);
|
||||
$resyncPct = $resyncSize > 0 ? round($resyncPos / $resyncSize * 100, 1) : 0;
|
||||
|
||||
// Last check from log
|
||||
$lastDate = null; $lastDuration = 0; $lastSpeed = 0; $lastErrors = 0; $lastExit = 0;
|
||||
$logFile = '/boot/config/parity-checks.log';
|
||||
if (file_exists($logFile)) {
|
||||
$lines = file($logFile, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES) ?: [];
|
||||
if ($lines) {
|
||||
$p = explode('|', trim(end($lines)));
|
||||
$lastDate = trim($p[0] ?? '');
|
||||
$lastDuration = (int)($p[1] ?? 0);
|
||||
$lastSpeed = (int)($p[2] ?? 0);
|
||||
$lastExit = (int)($p[3] ?? 0);
|
||||
$lastErrors = (int)($p[4] ?? 0);
|
||||
}
|
||||
}
|
||||
|
||||
// Parse last date string to timestamp
|
||||
$lastTs = $lastDate ? strtotime($lastDate) : null;
|
||||
|
||||
// Next scheduled check from cron
|
||||
$nextTs = null;
|
||||
$cronFile = '/boot/config/plugins/dynamix/parity-check.cron';
|
||||
foreach (@file($cronFile) ?: [] as $line) {
|
||||
$line = trim($line);
|
||||
if ($line === '' || $line[0] === '#') continue;
|
||||
if (!str_contains($line, 'mdcmd')) continue;
|
||||
$p = preg_split('/\s+/', $line);
|
||||
// cron: min hour dom month dow command...
|
||||
if (count($p) >= 5 && is_numeric($p[0]) && is_numeric($p[1]) && is_numeric($p[2])) {
|
||||
$next = new DateTime('now');
|
||||
$next->setTime((int)$p[1], (int)$p[0], 0);
|
||||
$next->setDate((int)$next->format('Y'), (int)$next->format('n'), (int)$p[2]);
|
||||
if ($next->getTimestamp() <= time()) $next->modify('+1 month');
|
||||
$nextTs = $next->getTimestamp();
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
$exitMap = ['0' => 'Completed', '-4' => 'Aborted', '-5' => 'Cancelled'];
|
||||
return [
|
||||
'valid' => $isValid,
|
||||
'num_disabled' => $numDisabled,
|
||||
'num_missing' => $numMissing,
|
||||
'in_progress' => $inProgress,
|
||||
'resync_pct' => $resyncPct,
|
||||
'exit_code' => $exitCode,
|
||||
'exit_label' => $exitMap[(string)$lastExit] ?? 'Unknown',
|
||||
'errors' => $lastErrors,
|
||||
'last_date' => $lastDate,
|
||||
'last_ts' => $lastTs,
|
||||
'last_duration' => $lastDuration,
|
||||
'last_speed_mb' => $lastSpeed > 0 ? round($lastSpeed / 1048576, 1) : null,
|
||||
'next_ts' => $nextTs,
|
||||
];
|
||||
}
|
||||
|
||||
function vv_storage_pools(): array {
|
||||
// ── API path ──────────────────────────────────────────────────────────────
|
||||
$api = vv_api_data();
|
||||
if ($api && isset($api['array']['caches'])) {
|
||||
$out = [];
|
||||
foreach ($api['array']['caches'] as $d) {
|
||||
$entry = vv_api_disk_entry($d, 'data');
|
||||
if ($entry) $out[] = $entry;
|
||||
}
|
||||
if (!empty($out)) {
|
||||
usort($out, fn($a, $b) => strcmp($a['name'], $b['name']));
|
||||
return $out;
|
||||
}
|
||||
}
|
||||
|
||||
// ── Local fallback ────────────────────────────────────────────────────────
|
||||
vv_api_record_fallback('storage_pools');
|
||||
$ini = @parse_ini_file('/var/local/emhttp/disks.ini', true) ?: [];
|
||||
$out = [];
|
||||
foreach ($ini as $key => $d) {
|
||||
if (($d['type'] ?? '') !== 'Cache') continue;
|
||||
if (($d['fsStatus'] ?? '') !== 'Mounted') continue;
|
||||
$entry = vv_disk_entry($d, $key);
|
||||
if ($entry) $out[] = $entry;
|
||||
}
|
||||
usort($out, fn($a, $b) => strcmp($a['name'], $b['name']));
|
||||
return $out;
|
||||
}
|
||||
|
||||
function vv_array_disks(): array {
|
||||
// ── API path ──────────────────────────────────────────────────────────────
|
||||
$api = vv_api_data();
|
||||
if ($api && (isset($api['array']['parities']) || isset($api['array']['disks']))) {
|
||||
$parity = [];
|
||||
$data = [];
|
||||
foreach ($api['array']['parities'] ?? [] as $d) {
|
||||
$entry = vv_api_disk_entry($d, 'parity');
|
||||
if ($entry) $parity[] = $entry;
|
||||
}
|
||||
foreach ($api['array']['disks'] ?? [] as $d) {
|
||||
$entry = vv_api_disk_entry($d, 'data');
|
||||
if ($entry) $data[] = $entry;
|
||||
}
|
||||
if (!empty($parity) || !empty($data)) {
|
||||
usort($parity, fn($a, $b) => strnatcmp($a['name'], $b['name']));
|
||||
usort($data, fn($a, $b) => strnatcmp($a['name'], $b['name']));
|
||||
return array_merge($parity, $data);
|
||||
}
|
||||
}
|
||||
|
||||
// ── Local fallback ────────────────────────────────────────────────────────
|
||||
vv_api_record_fallback('array_disks');
|
||||
$ini = @parse_ini_file('/var/local/emhttp/disks.ini', true) ?: [];
|
||||
$parity = [];
|
||||
$data = [];
|
||||
foreach ($ini as $key => $d) {
|
||||
$type = $d['type'] ?? '';
|
||||
if ($type === 'Parity') {
|
||||
$entry = vv_disk_entry($d, $key, 'parity');
|
||||
if ($entry) $parity[] = $entry;
|
||||
} elseif ($type === 'Data') {
|
||||
$entry = vv_disk_entry($d, $key, 'data');
|
||||
if ($entry) $data[] = $entry;
|
||||
}
|
||||
}
|
||||
usort($parity, fn($a, $b) => strnatcmp($a['name'], $b['name']));
|
||||
usort($data, fn($a, $b) => strnatcmp($a['name'], $b['name']));
|
||||
return array_merge($parity, $data);
|
||||
}
|
||||
|
||||
function vv_disk_io_rates(): array {
|
||||
$snapFile = VV_CACHE_DIR . '/vv_diskio_snap.json';
|
||||
$now = microtime(true);
|
||||
|
||||
// Read current whole-disk stats from /proc/diskstats
|
||||
$current = [];
|
||||
foreach (@file('/proc/diskstats', FILE_IGNORE_NEW_LINES) ?: [] as $line) {
|
||||
$p = preg_split('/\s+/', trim($line));
|
||||
if (count($p) < 14) continue;
|
||||
$dev = $p[2];
|
||||
// Keep only whole disks: sda/sdb, nvme0n1, md*, not sda1/nvme0n1p1
|
||||
if (!preg_match('/^(sd[a-z]+|nvme\d+n\d+|md\d+)$/', $dev)) continue;
|
||||
$current[$dev] = [(int)$p[5], (int)$p[9]]; // [sectors_read, sectors_written]
|
||||
}
|
||||
|
||||
// Load previous snapshot
|
||||
$snap = @json_decode(@file_get_contents($snapFile) ?: '', true) ?: [];
|
||||
$prevTime = (float)($snap['t'] ?? $now);
|
||||
$prev = $snap['d'] ?? [];
|
||||
|
||||
// Save current snapshot
|
||||
@file_put_contents($snapFile, json_encode(['t' => $now, 'd' => $current], JSON_UNESCAPED_UNICODE));
|
||||
|
||||
$dt = max(0.5, $now - $prevTime);
|
||||
$out = [];
|
||||
foreach ($current as $dev => [$rs, $ws]) {
|
||||
$entry = [
|
||||
'tr' => round($rs * 512 / 1073741824, 2), // cumulative GB read
|
||||
'tw' => round($ws * 512 / 1073741824, 2), // cumulative GB written
|
||||
];
|
||||
if (isset($prev[$dev])) {
|
||||
[$prs, $pws] = $prev[$dev];
|
||||
$r = max(0.0, ($rs - $prs) * 512 / $dt / 1048576);
|
||||
$w = max(0.0, ($ws - $pws) * 512 / $dt / 1048576);
|
||||
if ($r > 0.01) $entry['r'] = round($r, 1);
|
||||
if ($w > 0.01) $entry['w'] = round($w, 1);
|
||||
}
|
||||
$out[$dev] = $entry;
|
||||
}
|
||||
return $out;
|
||||
}
|
||||
|
||||
function vv_disk_thresholds(): array {
|
||||
$cfg = @file_get_contents('/boot/config/plugins/dynamix/dynamix.cfg') ?: '';
|
||||
$get = function(string $key) use ($cfg): ?int {
|
||||
return preg_match('/^\s*' . preg_quote($key, '/') . '\s*=\s*"?(\d+)"?/m', $cfg, $m)
|
||||
? (int)$m[1] : null;
|
||||
};
|
||||
return [
|
||||
'util_warn' => $get('warning') ?? 70,
|
||||
'util_crit' => $get('critical') ?? 90,
|
||||
'hdd_warn' => $get('hot') ?? 45,
|
||||
'hdd_crit' => $get('max') ?? 55,
|
||||
'ssd_warn' => $get('hotssd') ?? 60,
|
||||
'ssd_crit' => $get('maxssd') ?? 70,
|
||||
];
|
||||
}
|
||||
|
||||
// Fetch a lightweight snapshot from each remote host that has an API key configured.
|
||||
// Results are cached in /tmp for 30 seconds so rapid monitor polls don't hammer remote hosts.
|
||||
function vv_remote_hosts_stats(): array {
|
||||
// Read ALL conf files — remote host keys live in their own host*.conf, not the current host's.
|
||||
$vars = vv_conf_vars();
|
||||
foreach (glob(CONF_DIR . '/host*.conf') ?: [] as $f) {
|
||||
$raw = file_get_contents($f) ?: '';
|
||||
preg_match_all('/^\s*([A-Z0-9_]+)\s*=\s*["\']?([^"\'#\n]*?)["\']?\s*(?:#.*)?$/m', $raw, $m);
|
||||
foreach ($m[1] as $i => $key) {
|
||||
if (!isset($vars[$key])) $vars[$key] = trim($m[2][$i]);
|
||||
}
|
||||
}
|
||||
$myHost = vv_detect_host();
|
||||
$hostIds = array_filter(array_keys($vars), fn($k) => preg_match('/^HOST\d+$/', $k) && ($vars[$k] ?? '') !== '');
|
||||
sort($hostIds);
|
||||
|
||||
$results = [];
|
||||
foreach ($hostIds as $id) {
|
||||
if (strtolower($id) === strtolower($myHost)) continue;
|
||||
// Background cache written by remote_arr_cache_writer.sh every 2h — use it if present.
|
||||
$bgCache = VV_CACHE_DIR . '/monitor_remote_' . strtolower($id) . '.json';
|
||||
if (file_exists($bgCache)) {
|
||||
$cached = json_decode(file_get_contents($bgCache), true);
|
||||
if ($cached) {
|
||||
$cached['cache_age'] = time() - (int)filemtime($bgCache);
|
||||
$results[$id] = $cached;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// No background cache yet — fall back to live call (uses 30s inline cache).
|
||||
$key = $vars[strtoupper($id) . '_UNRAID_API_KEY'] ?? '';
|
||||
if (!$key) {
|
||||
$results[$id] = ['available' => false, 'no_api_key' => true,
|
||||
'host_id' => $id, 'hostname' => $vars[$id]];
|
||||
continue;
|
||||
}
|
||||
|
||||
$cacheFile = VV_CACHE_DIR . "/vv_remote_{$id}.json";
|
||||
if (file_exists($cacheFile) && (time() - filemtime($cacheFile)) < 30) {
|
||||
$cached = json_decode(file_get_contents($cacheFile), true);
|
||||
if ($cached) { $results[$id] = $cached; continue; }
|
||||
}
|
||||
|
||||
$gql = '{
|
||||
info { os { hostname uptime release } cpu { brand threads cores } }
|
||||
metrics { cpu { percentTotal } memory { percentTotal total used available } }
|
||||
array {
|
||||
state
|
||||
disks { fsSize fsUsed temp }
|
||||
caches { fsSize fsUsed temp }
|
||||
parities { temp }
|
||||
}
|
||||
vms { domains { name } }
|
||||
}';
|
||||
$data = vv_unraid_api_query(strtolower($id), $gql, 4, $key);
|
||||
|
||||
if (!$data) {
|
||||
$entry = ['available' => false, 'host_id' => $id, 'hostname' => $vars[$id]];
|
||||
file_put_contents($cacheFile, json_encode($entry));
|
||||
$results[$id] = $entry;
|
||||
continue;
|
||||
}
|
||||
|
||||
$os = $data['info']['os'] ?? [];
|
||||
$cpu = $data['info']['cpu'] ?? [];
|
||||
$mMem = $data['metrics']['memory'] ?? [];
|
||||
|
||||
$memPct = round((float)($mMem['percentTotal'] ?? 0));
|
||||
if ($memPct === 0) {
|
||||
$totalBytes = (float)($mMem['total'] ?? 0);
|
||||
$availBytes = (float)($mMem['available'] ?? 0);
|
||||
$memPct = $totalBytes > 0 ? (int)round(($totalBytes - $availBytes) / $totalBytes * 100) : 0;
|
||||
}
|
||||
$memTotalGb = isset($mMem['total']) ? _vv_api_bytes_to_gb((float)$mMem['total']) : 0;
|
||||
|
||||
$uptimeRaw = $os['uptime'] ?? '';
|
||||
if (is_numeric($uptimeRaw)) {
|
||||
$uptimeSec = (int)$uptimeRaw;
|
||||
$days = intdiv($uptimeSec, 86400);
|
||||
$hours = intdiv($uptimeSec % 86400, 3600);
|
||||
$mins = intdiv($uptimeSec % 3600, 60);
|
||||
$uptime = ($days ? "{$days}d " : '') . ($hours ? "{$hours}h " : '') . "{$mins}m";
|
||||
} else {
|
||||
$uptimeSec = 0;
|
||||
$uptime = $uptimeRaw ?: '—';
|
||||
}
|
||||
|
||||
$nodeMetrics = vv_api_node_metrics($data);
|
||||
$entry = array_merge([
|
||||
'available' => true,
|
||||
'host_id' => $id,
|
||||
'hostname' => $os['hostname'] ?? $vars[$id],
|
||||
'version' => $os['release'] ?? '',
|
||||
'uptime' => $uptime,
|
||||
'uptime_sec' => $uptimeSec,
|
||||
'cpu_load' => $nodeMetrics['cpu_pct'] ?? 0,
|
||||
'cpu_threads' => (int)($cpu['threads'] ?? 0),
|
||||
'mem_total_gb' => $memTotalGb,
|
||||
'mem_used_pct' => $memPct,
|
||||
'array_state' => $data['array']['state'] ?? 'UNKNOWN',
|
||||
], $nodeMetrics);
|
||||
file_put_contents($cacheFile, json_encode($entry));
|
||||
$results[$id] = $entry;
|
||||
}
|
||||
return $results;
|
||||
}
|
||||
|
||||
function vv_log_tail(string $path, int $lines): string {
|
||||
$fp = @fopen($path, 'r');
|
||||
if (!$fp) return '';
|
||||
fseek($fp, 0, SEEK_END);
|
||||
$size = ftell($fp);
|
||||
if ($size <= 0) { fclose($fp); return ''; }
|
||||
$chunk = min($size, 4096);
|
||||
fseek($fp, -$chunk, SEEK_END);
|
||||
$data = fread($fp, $chunk);
|
||||
fclose($fp);
|
||||
$all = explode("\n", $data ?: '');
|
||||
return implode("\n", array_slice($all, -$lines));
|
||||
}
|
||||
|
||||
function vv_parse_bash_array(string $raw, string $varName): array {
|
||||
if (!preg_match('/^\s*' . preg_quote($varName, '/') . '\s*=\s*\(([^)]*)\)/ms', $raw, $m)) return [];
|
||||
$items = [];
|
||||
foreach (explode("\n", $m[1]) as $line) {
|
||||
$line = trim(preg_replace('/#.*$/', '', $line), " \t\"'");
|
||||
if ($line !== '') $items[] = $line;
|
||||
}
|
||||
return $items;
|
||||
}
|
||||
|
||||
function vv_transcode_sessions(): array {
|
||||
$v = vv_conf_vars();
|
||||
$stateDir = rtrim($v['STATE_DIR'] ?? STATE_DIR, '/');
|
||||
$stateFile = "$stateDir/transcode_state.db";
|
||||
if (!file_exists($stateFile)) return ['available' => false];
|
||||
|
||||
$raw = [];
|
||||
foreach (file($stateFile) ?: [] as $line) {
|
||||
[$k, $v] = array_pad(explode('=', trim($line), 2), 2, '');
|
||||
$raw[trim($k)] = trim($v);
|
||||
}
|
||||
|
||||
$target = $raw['current_target'] ?? '';
|
||||
$lastFlip = (int)($raw['last_flip_time'] ?? 0);
|
||||
$flipCount = (int)($raw['flip_count_hour'] ?? 0);
|
||||
$isRamdisk = str_contains($target, 'ramdisk');
|
||||
|
||||
// Count active sessions: subdirs (legacy transcode) + unique hex prefixes (Live TV / Direct Stream HLS)
|
||||
$ramdiskPath = '/mnt/ramdisk_transcodes/transcoding-temp';
|
||||
$ramSessions = count(glob("$ramdiskPath/*/", GLOB_ONLYDIR) ?: []);
|
||||
$maxAge = (int)(vv_conf_vars()['TRANSCODE_MAX_AGE'] ?? 20);
|
||||
$activeFiles = 0;
|
||||
$cutoff = time() - $maxAge * 60;
|
||||
$flatPrefixes = [];
|
||||
if (is_dir($ramdiskPath)) {
|
||||
foreach (new DirectoryIterator($ramdiskPath) as $f) {
|
||||
if (!$f->isFile()) continue;
|
||||
if (preg_match('/^([0-9a-f]{16,})/', $f->getFilename(), $m)) {
|
||||
$flatPrefixes[$m[1]] = true;
|
||||
}
|
||||
if ($f->getMTime() >= $cutoff) $activeFiles++;
|
||||
}
|
||||
}
|
||||
$ramSessions += count($flatPrefixes);
|
||||
|
||||
// SSD path: first transcoding-temp mount that is not a RAM filesystem (tmpfs/ramfs)
|
||||
$ssdPath = '';
|
||||
$ssdSessions = 0;
|
||||
foreach (glob('/mnt/*/transcoding-temp/', GLOB_ONLYDIR) ?: [] as $p) {
|
||||
$parts = explode('/', rtrim($p, '/'));
|
||||
array_pop($parts);
|
||||
$mount = implode('/', $parts) ?: '/';
|
||||
$fsType = trim(shell_exec('findmnt -n -o FSTYPE ' . escapeshellarg($mount) . ' 2>/dev/null') ?: '');
|
||||
if ($fsType === 'tmpfs' || $fsType === 'ramfs') continue;
|
||||
$ssdPath = $p;
|
||||
break;
|
||||
}
|
||||
$ssd = ['available' => false];
|
||||
if ($ssdPath) {
|
||||
$ssdSessions = count(glob($ssdPath . '/*/', GLOB_ONLYDIR) ?: []);
|
||||
$parts = explode('/', rtrim($ssdPath, '/'));
|
||||
array_pop($parts);
|
||||
$ssdMount = implode('/', $parts) ?: '/';
|
||||
$ssd = vv_df($ssdMount);
|
||||
}
|
||||
|
||||
// Ramdisk disk usage
|
||||
$rd = vv_df('/mnt/ramdisk_transcodes');
|
||||
|
||||
// Last cleanup values from transcode management log
|
||||
$lastRdFreed = null;
|
||||
$lastSsdFreed = null;
|
||||
$logFile = LOG_DIR . '/Orchestrators/transcode_management.log';
|
||||
if (file_exists($logFile)) {
|
||||
$lines = file($logFile, FILE_IGNORE_NEW_LINES) ?: [];
|
||||
foreach (array_reverse($lines) as $line) {
|
||||
if ($lastRdFreed === null && preg_match('/Ramdisk freed:\s*(\S+)/u', $line, $m))
|
||||
$lastRdFreed = $m[1];
|
||||
if ($lastSsdFreed === null && preg_match('/SSD freed:\s*(\S+)/u', $line, $m))
|
||||
$lastSsdFreed = $m[1];
|
||||
if ($lastRdFreed !== null && $lastSsdFreed !== null) break;
|
||||
}
|
||||
}
|
||||
|
||||
return [
|
||||
'available' => true,
|
||||
'current_target' => $target,
|
||||
'is_ramdisk' => $isRamdisk,
|
||||
'flip_count_hour' => $flipCount,
|
||||
'last_flip_time' => $lastFlip,
|
||||
'last_flip_ago' => $lastFlip > 0 ? time() - $lastFlip : null,
|
||||
'ram_sessions' => $ramSessions,
|
||||
'ssd_sessions' => $ssdSessions,
|
||||
'active_files' => $activeFiles,
|
||||
'ramdisk' => $rd,
|
||||
'ssd' => $ssd,
|
||||
'last_rd_freed' => $lastRdFreed,
|
||||
'last_ssd_freed' => $lastSsdFreed,
|
||||
];
|
||||
}
|
||||
+514
@@ -0,0 +1,514 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ================================= Rsync Stop =================================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Stops rsync intelligently on both local and remote servers. Auto-detects
|
||||
# running orchestrators and chooses the safest stop strategy. If an
|
||||
# orchestrator is running, kills only the rsync subprocess so the orchestrator
|
||||
# exits cleanly after finishing the current share. Use --full-stop to kill
|
||||
# everything immediately.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Two Stop Modes
|
||||
# Default (smart):
|
||||
# Detects if an orchestrator (daily/weekly/critical sync) is running.
|
||||
# If orchestrator found → kills rsync subprocess only. Orchestrator sees
|
||||
# rsync exit → moves to next share or exits cleanly on its own.
|
||||
# If no orchestrator → kills rsync directly (standalone rsync.sh run).
|
||||
# Cleans stale lock files after kill.
|
||||
# Recovers containers left stopped by interrupted rsync (local only).
|
||||
#
|
||||
# --full-stop (nuclear):
|
||||
# Kills orchestrator first → then kills rsync.
|
||||
# Orchestrator will NOT continue to next share.
|
||||
# Use when everything needs to stop immediately.
|
||||
#
|
||||
# Orchestrator Detection
|
||||
# detect_rsync_parent() scans all lock files to find which running process
|
||||
# has rsync as a descendant. No hardcoded list — works for any orchestrator.
|
||||
# Returns "script_name:parent_pid" if found, empty if standalone.
|
||||
#
|
||||
# Remote Handling
|
||||
# Both local and remote handled in one run via SSH.
|
||||
# Remote containers left as-is — docker_watchdog.sh handles remote recovery.
|
||||
# If remote unreachable → skips remote cleanly, logs warning.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Root Required
|
||||
# pkill and docker require root.
|
||||
#
|
||||
# Single Instance Lock
|
||||
# acquire_lock prevents concurrent stop attempts racing each other.
|
||||
#
|
||||
# Timeout Protection
|
||||
# DOCKER_TIMEOUT (15s) on all docker calls — hung daemon doesn't block.
|
||||
# SSH_TIMEOUT (15s) on all remote SSH calls.
|
||||
#
|
||||
# SIGTERM → SIGKILL Sequence
|
||||
# Orchestrators receive SIGTERM first, SIGKILL only if still running after 2s.
|
||||
#
|
||||
# Container Recovery
|
||||
# Restarts local containers left stopped by the killed rsync session.
|
||||
# Remote containers deferred to docker_watchdog.sh.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# rsync_stop.sh
|
||||
# Auto-detect orchestrator. Kill rsync-only or full-stop accordingly.
|
||||
#
|
||||
# rsync_stop.sh --full-stop
|
||||
# Kill orchestrator first, then kill rsync. Nothing continues after this.
|
||||
#
|
||||
# rsync_stop.sh --rsync-only
|
||||
# Skip container recovery. Used when called by other scripts that handle
|
||||
# recovery themselves.
|
||||
#
|
||||
# rsync_stop.sh --dry-run
|
||||
# Show what would be killed without killing anything.
|
||||
#
|
||||
# rsync_stop.sh --status
|
||||
# Show local and remote rsync PIDs, running orchestrators, and lock files.
|
||||
#
|
||||
# rsync_stop.sh --full-stop --dry-run
|
||||
# Preview full-stop sequence without making any changes.
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
DOCKER_TIMEOUT=15
|
||||
SSH_TIMEOUT=15
|
||||
|
||||
# ── Parse special flags before parse_args ─────────────────────────────────────────────────────
|
||||
FULL_STOP=false
|
||||
RSYNC_ONLY_MODE=false
|
||||
FILTERED_ARGS=()
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--full-stop) FULL_STOP=true ;;
|
||||
--rsync-only) RSYNC_ONLY_MODE=true ;;
|
||||
*) FILTERED_ARGS+=("$arg") ;;
|
||||
esac
|
||||
done
|
||||
|
||||
parse_args "${FILTERED_ARGS[@]}"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root — pkill and docker require root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
acquire_lock
|
||||
|
||||
if ! command -v docker &>/dev/null; then
|
||||
error "Docker command not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
detect_hosts
|
||||
|
||||
# Soft IP resolution — rsync_stop continues local-only if remote unreachable
|
||||
REMOTE_REACHABLE=false
|
||||
REMOTE_SERVER=$(resolve_tailscale_ip "$REMOTE_SERVER_NAME")
|
||||
if [[ -z "$REMOTE_SERVER" ]]; then
|
||||
warn "$REMOTE_SERVER_NAME — cannot resolve Tailscale IP, remote operations will be skipped"
|
||||
elif timeout "$SSH_TIMEOUT" ping -c1 -W3 "$REMOTE_SERVER" &>/dev/null; then
|
||||
REMOTE_REACHABLE=true
|
||||
log "$REMOTE_SERVER_NAME reachable ✅"
|
||||
else
|
||||
warn "$REMOTE_SERVER_NAME unreachable — remote operations will be skipped"
|
||||
fi
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
[[ "$FULL_STOP" == true ]] && warn "FULL STOP mode — orchestrator + rsync will be killed"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY RSYNC STOP STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_NET Remote: $REMOTE_ID ($REMOTE_SERVER_NAME)"
|
||||
echo ""
|
||||
|
||||
LOCAL_PIDS=$(pgrep -x rsync 2>/dev/null | tr '\n' ' ')
|
||||
echo " $ICON_SYNC Local rsync PIDs: ${LOCAL_PIDS:-none}"
|
||||
|
||||
for lockfile in "$LOCK_DIR"/*.lock; do
|
||||
[[ -f "$lockfile" ]] || continue
|
||||
content=$(cat "$lockfile" 2>/dev/null)
|
||||
pid="${content%%:*}"
|
||||
name="${content##*:}"
|
||||
[[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null && \
|
||||
echo " $ICON_RUNNING Lock: $name (PID $pid)"
|
||||
done
|
||||
|
||||
if [[ "$REMOTE_REACHABLE" == true ]]; then
|
||||
REMOTE_PIDS=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" \
|
||||
root@"$REMOTE_SERVER" "pgrep -x rsync || true" 2>/dev/null | tr '\n' ' ')
|
||||
echo " $ICON_SYNC Remote rsync PIDs: ${REMOTE_PIDS:-none}"
|
||||
else
|
||||
echo " $ICON_WARN Remote: unreachable"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ── ORCHESTRATOR DETECTION ────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Scans lock files to find which running process has rsync as a descendant.
|
||||
# No hardcoded script names — detects any orchestrator automatically.
|
||||
|
||||
detect_rsync_parent() {
|
||||
local rsync_pids
|
||||
rsync_pids=$(pgrep -x rsync 2>/dev/null || true)
|
||||
[[ -z "$rsync_pids" ]] && echo "" && return
|
||||
|
||||
for lockfile in "$LOCK_DIR"/*.lock; do
|
||||
[[ -f "$lockfile" ]] || continue
|
||||
local content pid locked_name
|
||||
content=$(cat "$lockfile" 2>/dev/null)
|
||||
pid="${content%%:*}"
|
||||
locked_name="${content##*:}"
|
||||
[[ -z "$pid" ]] && continue
|
||||
! kill -0 "$pid" 2>/dev/null && continue
|
||||
[[ "$locked_name" == rsync_* ]] && continue
|
||||
|
||||
local all_descendants
|
||||
all_descendants=$(pgrep -P "$pid" 2>/dev/null || true)
|
||||
|
||||
while IFS= read -r rsync_pid; do
|
||||
[[ -z "$rsync_pid" ]] && continue
|
||||
local ppid
|
||||
ppid=$(awk '/^PPid:/{print $2}' /proc/"$rsync_pid"/status 2>/dev/null || echo "")
|
||||
if echo "$all_descendants" | grep -qw "$rsync_pid" 2>/dev/null || \
|
||||
[[ "$ppid" == "$pid" ]]; then
|
||||
echo "${locked_name}:${pid}"
|
||||
return
|
||||
fi
|
||||
done <<< "$rsync_pids"
|
||||
done
|
||||
echo ""
|
||||
}
|
||||
|
||||
detect_rsync_parent_remote() {
|
||||
[[ "$REMOTE_REACHABLE" != true ]] && echo "" && return
|
||||
|
||||
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" \
|
||||
root@"$REMOTE_SERVER" bash << 'REMOTE_SCRIPT' 2>/dev/null
|
||||
LOCK_DIR="/tmp/unraid_locks"
|
||||
rsync_pids=$(pgrep -x rsync 2>/dev/null || true)
|
||||
[[ -z "$rsync_pids" ]] && exit 0
|
||||
for lockfile in "$LOCK_DIR"/*.lock; do
|
||||
[[ -f "$lockfile" ]] || continue
|
||||
content=$(cat "$lockfile" 2>/dev/null)
|
||||
pid="${content%%:*}"
|
||||
locked_name="${content##*:}"
|
||||
[[ -z "$pid" ]] && continue
|
||||
! kill -0 "$pid" 2>/dev/null && continue
|
||||
[[ "$locked_name" == rsync_* ]] && continue
|
||||
all_descendants=$(pgrep -P "$pid" 2>/dev/null || true)
|
||||
while IFS= read -r rsync_pid; do
|
||||
[[ -z "$rsync_pid" ]] && continue
|
||||
ppid=$(awk '/^PPid:/{print $2}' /proc/"$rsync_pid"/status 2>/dev/null || echo "")
|
||||
if echo "$all_descendants" | grep -qw "$rsync_pid" 2>/dev/null || \
|
||||
[[ "$ppid" == "$pid" ]]; then
|
||||
echo "${locked_name}:${pid}"
|
||||
exit 0
|
||||
fi
|
||||
done <<< "$rsync_pids"
|
||||
done
|
||||
REMOTE_SCRIPT
|
||||
}
|
||||
|
||||
LOCAL_ORCH=$(detect_rsync_parent)
|
||||
REMOTE_ORCH=""
|
||||
[[ "$REMOTE_REACHABLE" == true ]] && REMOTE_ORCH=$(detect_rsync_parent_remote)
|
||||
|
||||
# Determine mode
|
||||
if [[ "$FULL_STOP" == true ]]; then
|
||||
MODE="full-stop"
|
||||
elif [[ -n "$LOCAL_ORCH" ]] || [[ -n "$REMOTE_ORCH" ]]; then
|
||||
MODE="rsync-only"
|
||||
[[ -n "$LOCAL_ORCH" ]] && \
|
||||
warn "Local orchestrator detected: ${LOCAL_ORCH%%:*} — rsync-only mode"
|
||||
[[ -n "$REMOTE_ORCH" ]] && \
|
||||
warn "Remote orchestrator detected: ${REMOTE_ORCH%%:*} — rsync-only mode"
|
||||
warn "Use --full-stop to also kill the orchestrator"
|
||||
else
|
||||
MODE="rsync-only"
|
||||
log "No orchestrator detected — killing rsync directly"
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ── Kill Orchestrators (full-stop only) ───────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
ORCHESTRATORS_KILLED=()
|
||||
REMOTE_ORCHESTRATORS_KILLED=()
|
||||
|
||||
kill_orchestrator() {
|
||||
local script_name="$1" pid="$2"
|
||||
local lockfile="$LOCK_DIR/${script_name}.lock"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would kill $script_name (PID $pid)"
|
||||
return 0
|
||||
fi
|
||||
|
||||
kill -TERM "$pid" 2>/dev/null
|
||||
sleep 2
|
||||
kill -0 "$pid" 2>/dev/null && kill -KILL "$pid" 2>/dev/null
|
||||
sleep 1
|
||||
|
||||
if ! kill -0 "$pid" 2>/dev/null; then
|
||||
warn "$script_name stopped (PID $pid) ✅"
|
||||
rm -f "$lockfile"
|
||||
return 0
|
||||
else
|
||||
error "Failed to kill $script_name (PID $pid)"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
if [[ "$MODE" == "full-stop" ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_STOP Kill Orchestrators ━━━"
|
||||
|
||||
if [[ -n "$LOCAL_ORCH" ]]; then
|
||||
local_name="${LOCAL_ORCH%%:*}"
|
||||
local_pid="${LOCAL_ORCH##*:}"
|
||||
warn "Killing local: $local_name (PID $local_pid)"
|
||||
kill_orchestrator "$local_name" "$local_pid" && \
|
||||
ORCHESTRATORS_KILLED+=("$local_name")
|
||||
else
|
||||
log "No local orchestrator running"
|
||||
fi
|
||||
|
||||
if [[ "$REMOTE_REACHABLE" == true ]] && [[ -n "$REMOTE_ORCH" ]]; then
|
||||
remote_name="${REMOTE_ORCH%%:*}"
|
||||
remote_pid="${REMOTE_ORCH##*:}"
|
||||
remote_lock="$LOCK_DIR/${remote_name}.lock"
|
||||
warn "Killing remote: $remote_name (PID $remote_pid)"
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" \
|
||||
root@"$REMOTE_SERVER" \
|
||||
"kill -TERM '$remote_pid' 2>/dev/null; sleep 2; \
|
||||
kill -0 '$remote_pid' 2>/dev/null && kill -KILL '$remote_pid' 2>/dev/null; \
|
||||
rm -f '$remote_lock'" 2>/dev/null
|
||||
warn "Remote $remote_name stopped ✅"
|
||||
REMOTE_ORCHESTRATORS_KILLED+=("$remote_name")
|
||||
else
|
||||
warn "DRY RUN — would kill remote $remote_name (PID $remote_pid)"
|
||||
fi
|
||||
elif [[ "$REMOTE_REACHABLE" == true ]]; then
|
||||
log "No remote orchestrator running"
|
||||
fi
|
||||
|
||||
[[ ${#ORCHESTRATORS_KILLED[@]} -gt 0 || \
|
||||
${#REMOTE_ORCHESTRATORS_KILLED[@]} -gt 0 ]] && sleep 3
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Local Rsync ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_STOP Local Rsync ━━━"
|
||||
|
||||
LOCAL_KILLED=false
|
||||
LOCAL_PIDS=$(pgrep -x rsync 2>/dev/null || true)
|
||||
|
||||
if [[ -z "$LOCAL_PIDS" ]]; then
|
||||
log "No rsync processes running locally"
|
||||
else
|
||||
warn "Found local rsync PIDs: $(echo "$LOCAL_PIDS" | tr '\n' ' ')"
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would kill local rsync"
|
||||
else
|
||||
pkill -x rsync 2>/dev/null && LOCAL_KILLED=true || \
|
||||
warn "pkill returned non-zero — rsync may have already exited"
|
||||
[[ "$LOCAL_KILLED" == true ]] && warn "Local rsync killed ✅"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Clean stale rsync lock files
|
||||
for lockfile in "$LOCK_DIR"/rsync_*.lock; do
|
||||
[[ -f "$lockfile" ]] || continue
|
||||
content=$(cat "$lockfile" 2>/dev/null)
|
||||
pid="${content%%:*}"
|
||||
if [[ -n "$pid" ]] && ! kill -0 "$pid" 2>/dev/null; then
|
||||
log "Cleaning stale lock: $(basename "$lockfile")"
|
||||
[[ "$DRY_RUN" == false ]] && rm -f "$lockfile"
|
||||
fi
|
||||
done
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Remote Rsync ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_STOP Remote Rsync — $REMOTE_SERVER_NAME ━━━"
|
||||
|
||||
REMOTE_KILLED=false
|
||||
|
||||
if [[ "$REMOTE_REACHABLE" == false ]]; then
|
||||
warn "Skipping — $REMOTE_SERVER_NAME unreachable"
|
||||
else
|
||||
REMOTE_PIDS=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" \
|
||||
root@"$REMOTE_SERVER" "pgrep -x rsync || true" 2>/dev/null || true)
|
||||
|
||||
if [[ -z "$REMOTE_PIDS" ]]; then
|
||||
log "No rsync running on $REMOTE_SERVER_NAME"
|
||||
else
|
||||
warn "Found remote rsync PIDs: $(echo "$REMOTE_PIDS" | tr '\n' ' ')"
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would kill remote rsync"
|
||||
else
|
||||
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" \
|
||||
root@"$REMOTE_SERVER" "pkill -x rsync || true" 2>/dev/null && \
|
||||
REMOTE_KILLED=true || \
|
||||
warn "Remote pkill returned non-zero — rsync may have already exited"
|
||||
[[ "$REMOTE_KILLED" == true ]] && warn "Remote rsync killed ✅"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Container Recovery ━━━
|
||||
# ==============================================================================================
|
||||
# Restart local containers left stopped by interrupted rsync.
|
||||
# Remote containers left for docker_watchdog.sh to recover.
|
||||
# Skipped with --rsync-only flag (called by other scripts that handle recovery themselves).
|
||||
CONTAINERS_RESTARTED=()
|
||||
CONTAINERS_FAILED=()
|
||||
|
||||
if [[ "$RSYNC_ONLY_MODE" == false ]] && \
|
||||
{ [[ "$LOCAL_KILLED" == true ]] || [[ ${#ORCHESTRATORS_KILLED[@]} -gt 0 ]]; }; then
|
||||
|
||||
echo ""
|
||||
echo "━━━ $ICON_START Container Recovery ━━━"
|
||||
log "Checking profile containers for recovery..."
|
||||
|
||||
declare -A SEEN
|
||||
ALL_CONTAINERS=()
|
||||
|
||||
for profile_containers in "${PROFILE_CRITICAL_CONTAINER_NAMES[@]:-}"; do
|
||||
read -r -a container_list <<< "$profile_containers"
|
||||
for c in "${container_list[@]:-}"; do
|
||||
[[ -z "$c" ]] && continue
|
||||
if [[ -z "${SEEN[$c]:-}" ]]; then
|
||||
SEEN[$c]=1
|
||||
ALL_CONTAINERS+=("$c")
|
||||
fi
|
||||
done
|
||||
done
|
||||
|
||||
if [[ ${#ALL_CONTAINERS[@]} -eq 0 ]]; then
|
||||
log "No profile containers defined — skipping recovery"
|
||||
else
|
||||
for c in "${ALL_CONTAINERS[@]}"; do
|
||||
STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
|
||||
'{{.State.Running}}' "$c" 2>/dev/null || echo "unknown")
|
||||
case "$STATUS" in
|
||||
true)
|
||||
log "$c — running ✅"
|
||||
;;
|
||||
false)
|
||||
warn "$c — stopped — restarting..."
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would restart $c"
|
||||
else
|
||||
if timeout "$DOCKER_TIMEOUT" docker start "$c" >/dev/null 2>&1; then
|
||||
warn "$c restarted ✅"
|
||||
CONTAINERS_RESTARTED+=("$c")
|
||||
else
|
||||
error "Failed to restart $c"
|
||||
CONTAINERS_FAILED+=("$c")
|
||||
fi
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
log "$c not found locally — skipping"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
fi
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY RSYNC STOP SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_GEAR Mode: $MODE"
|
||||
echo ""
|
||||
|
||||
echo "$ICON_HOST Local ($MY_ID):"
|
||||
[[ ${#ORCHESTRATORS_KILLED[@]} -gt 0 ]] && \
|
||||
warn " Orchestrators killed: ${ORCHESTRATORS_KILLED[*]}"
|
||||
if [[ "$LOCAL_KILLED" == true ]]; then
|
||||
warn " Rsync killed ✅"
|
||||
else
|
||||
log " No rsync was running"
|
||||
fi
|
||||
|
||||
echo "$ICON_NET Remote ($REMOTE_ID — $REMOTE_SERVER_NAME):"
|
||||
if [[ "$REMOTE_REACHABLE" == false ]]; then
|
||||
warn " Unreachable — skipped"
|
||||
else
|
||||
[[ ${#REMOTE_ORCHESTRATORS_KILLED[@]} -gt 0 ]] && \
|
||||
warn " Orchestrators killed: ${REMOTE_ORCHESTRATORS_KILLED[*]}"
|
||||
if [[ "$REMOTE_KILLED" == true ]]; then
|
||||
warn " Rsync killed ✅"
|
||||
else
|
||||
log " No rsync was running"
|
||||
fi
|
||||
fi
|
||||
|
||||
[[ ${#CONTAINERS_RESTARTED[@]} -gt 0 ]] && \
|
||||
warn "$ICON_CONTAINERS Containers recovered: ${CONTAINERS_RESTARTED[*]}"
|
||||
[[ ${#CONTAINERS_FAILED[@]} -gt 0 ]] && \
|
||||
echo "$ICON_ERROR Containers failed to restart: ${CONTAINERS_FAILED[*]}"
|
||||
|
||||
echo ""
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no changes made"
|
||||
else
|
||||
echo "$ICON_DONE Status: done ✅"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
# Notify if anything was actually killed or failed
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
if [[ ${#CONTAINERS_FAILED[@]} -gt 0 ]]; then
|
||||
notify "Rsync stop on $(hostname) ($MY_ID) — containers failed to restart: ${CONTAINERS_FAILED[*]}" \
|
||||
"Rsync Stop" "warning"
|
||||
elif [[ "$LOCAL_KILLED" == true || "$REMOTE_KILLED" == true || \
|
||||
${#ORCHESTRATORS_KILLED[@]} -gt 0 ]]; then
|
||||
notify "Rsync stopped on $(hostname) ($MY_ID) — mode: $MODE${CONTAINERS_RESTARTED:+ — recovered: ${CONTAINERS_RESTARTED[*]}}" \
|
||||
"Rsync Stop" "warning"
|
||||
fi
|
||||
fi
|
||||
+232
@@ -0,0 +1,232 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ============================= Critical Sync Maintenance ======================================
|
||||
# ==============================================================================================
|
||||
# Orchestrator for time-sensitive syncs that run every 30 minutes.
|
||||
# Keeps the mirror current between the less frequent daily and weekly windows.
|
||||
# Schedule: */30 * * * * (every 30 minutes via User Scripts plugin)
|
||||
#
|
||||
# ── EXECUTION ORDER ───────────────────────────────────────────────────────────────────────────
|
||||
# 1. Critical-Data rsync — auth stack, NPM config, certs (containers stopped both sides)
|
||||
# 2. emby-fallback rsync — dirty Emby sync (watch states, library — Emby stays running)
|
||||
# 3. CRITICAL_MAINTENANCE_SCRIPTS — any scripts configured for critical window
|
||||
# 4. partnership --check — read both state files, detect changes, act accordingly
|
||||
#
|
||||
# ── WHY EVERY 30 MINUTES ──────────────────────────────────────────────────────────────────────
|
||||
# Auth stack changes (new users, proxy rules, certs) propagate within 30min ✅
|
||||
# Emby watch states stay in sync — mirror users see correct playback position ✅
|
||||
# Partnership state changes detected and acted on quickly ✅
|
||||
# Lock prevents: daily rsync doing Critical-Data mid-critical window ✅
|
||||
#
|
||||
# ── RSYNC GATE ────────────────────────────────────────────────────────────────────────────────
|
||||
# RSYNC_ENABLED=false → skips all syncs (global gate)
|
||||
# CRITICAL_RSYNC_ENABLED=false → skips critical syncs only (per-orchestrator gate)
|
||||
# partnership --check always runs regardless — state check doesn't need rsync
|
||||
#
|
||||
# ── LOCK BEHAVIOUR ────────────────────────────────────────────────────────────────────────────
|
||||
# acquire_lock "strict" — if previous 30min run still going, skip this cycle entirely
|
||||
# Critical-Data taking > 30min is a problem worth knowing about
|
||||
# Strict mode prevents pile-up without waiting — log and move on ✅
|
||||
#
|
||||
# ── SILENT WHEN HEALTHY ───────────────────────────────────────────────────────────────────────
|
||||
# Runs 48 times per day — clean runs must produce zero output ✅
|
||||
# Only failures and notable events produce visible output
|
||||
#
|
||||
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||||
# CRITICAL_RSYNC_ENABLED — enable/disable rsync section
|
||||
# CRITICAL_SYNC_SHARES — shares synced every 30min (HOST*_CRITICAL_SYNC_SHARES)
|
||||
# CRITICAL_MAINTENANCE_SCRIPTS — scripts run in critical window (optional)
|
||||
# PARTNERSHIP_ENABLED — enable/disable partnership check
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# critical_sync_maintenance.sh — normal run
|
||||
# critical_sync_maintenance.sh --dry-run — preview syncs without transferring
|
||||
# critical_sync_maintenance.sh --log — verbose per-share output
|
||||
# critical_sync_maintenance.sh --status — show configuration and exit
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
acquire_lock "strict"
|
||||
|
||||
detect_hosts
|
||||
resolve_remote_ip
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY CRITICAL SYNC STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_NET Remote: $REMOTE_ID ($REMOTE_SERVER_NAME)"
|
||||
echo "$ICON_SYNC Rsync enabled: ${RSYNC_ENABLED:-true}"
|
||||
echo "$ICON_SYNC Critical enabled: ${CRITICAL_RSYNC_ENABLED:-false}"
|
||||
echo "$ICON_SHIELD Partnership: ${PARTNERSHIP_ENABLED:-false}"
|
||||
echo ""
|
||||
echo "━━━ Critical Sync Shares ━━━"
|
||||
if [[ ${#CRITICAL_SYNC_SHARES[@]} -eq 0 ]]; then
|
||||
warn " No CRITICAL_SYNC_SHARES configured"
|
||||
else
|
||||
for share in "${CRITICAL_SYNC_SHARES[@]}"; do
|
||||
[[ -z "$share" ]] && continue
|
||||
SHARE_PATH="${share%%|*}"
|
||||
SHARE_PROFILE="${share##*|}"
|
||||
SHARE_NAME=$(basename "$SHARE_PATH")
|
||||
[[ "$SHARE_PATH" == "$SHARE_PROFILE" ]] && \
|
||||
echo " $ICON_SYNC $SHARE_NAME — no profile" || \
|
||||
echo " $ICON_SYNC $SHARE_NAME — profile: $SHARE_PROFILE"
|
||||
done
|
||||
fi
|
||||
echo ""
|
||||
echo "━━━ Critical Maintenance Scripts ━━━"
|
||||
if [[ ${#CRITICAL_MAINTENANCE_SCRIPTS[@]} -eq 0 ]]; then
|
||||
echo " None configured"
|
||||
else
|
||||
for entry in "${CRITICAL_MAINTENANCE_SCRIPTS[@]}"; do
|
||||
[[ -z "$entry" || "$entry" == \#* ]] && continue
|
||||
echo " $ICON_GEAR $(basename "${entry%% *}")"
|
||||
done
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Critical Shares Sync ━━━
|
||||
# ==============================================================================================
|
||||
START=$(date +%s)
|
||||
RSYNC_OK=false
|
||||
PASS=()
|
||||
FAIL=()
|
||||
|
||||
log "$ICON_SYNC Critical shares (${#CRITICAL_SYNC_SHARES[@]}): $(for s in "${CRITICAL_SYNC_SHARES[@]}"; do printf '%s ' "$(basename "${s%%|*}")"; done)"
|
||||
[[ ${#CRITICAL_MAINTENANCE_SCRIPTS[@]} -gt 0 ]] && \
|
||||
log "$ICON_GEAR Maintenance scripts: $(for s in "${CRITICAL_MAINTENANCE_SCRIPTS[@]}"; do printf '%s ' "$(basename "${s%% *}")"; done)"
|
||||
|
||||
if ! check_rsync_enabled "CRITICAL"; then
|
||||
echo "Critical rsync disabled — skipping sync, running partnership check only"
|
||||
elif [[ ${#CRITICAL_SYNC_SHARES[@]} -eq 0 ]]; then
|
||||
warn "CRITICAL_RSYNC_ENABLED=true but CRITICAL_SYNC_SHARES is empty for $MY_ID"
|
||||
warn "Check HOST*_CRITICAL_SYNC_SHARES in host*.conf"
|
||||
else
|
||||
echo "Critical sync — $MY_ID → $REMOTE_ID — $(date '+%H:%M:%S')"
|
||||
|
||||
# Build dry-run flag to pass through
|
||||
RSYNC_DRY=""
|
||||
[[ "$DRY_RUN" == true ]] && RSYNC_DRY="--dry-run"
|
||||
|
||||
for share in "${CRITICAL_SYNC_SHARES[@]}"; do
|
||||
[[ -z "$share" ]] && continue
|
||||
|
||||
# Parse optional profile flag: "/path/to/share|profile-name"
|
||||
SHARE_PATH="${share%%|*}"
|
||||
SHARE_PROFILE="${share##*|}"
|
||||
SHARE_NAME=$(basename "$SHARE_PATH")
|
||||
|
||||
SHARE_START=$(date +%s)
|
||||
|
||||
if [[ "$SHARE_PATH" == "$SHARE_PROFILE" ]]; then
|
||||
bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$SHARE_PATH" $RSYNC_DRY
|
||||
else
|
||||
bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$SHARE_PATH" \
|
||||
--profile="$SHARE_PROFILE" $RSYNC_DRY
|
||||
fi
|
||||
|
||||
RSYNC_EXIT=$?
|
||||
SHARE_DUR=$(format_duration $(( $(date +%s) - SHARE_START )))
|
||||
|
||||
if [[ "$RSYNC_EXIT" -eq 0 ]]; then
|
||||
PASS+=("$SHARE_NAME")
|
||||
log "$SHARE_NAME — done in $SHARE_DUR ✅"
|
||||
RSYNC_OK=true
|
||||
else
|
||||
FAIL+=("$SHARE_NAME")
|
||||
error "$SHARE_NAME — failed after $SHARE_DUR (exit $RSYNC_EXIT)"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Critical Maintenance Scripts ━━━
|
||||
# ==============================================================================================
|
||||
if [[ ${#CRITICAL_MAINTENANCE_SCRIPTS[@]} -gt 0 ]]; then
|
||||
for script_entry in "${CRITICAL_MAINTENANCE_SCRIPTS[@]}"; do
|
||||
[[ -z "$script_entry" || "$script_entry" == \#* ]] && continue
|
||||
|
||||
SCRIPT_PATH="$SCRIPT_DIR/../${script_entry%% *}"
|
||||
SCRIPT_ARGS="${script_entry#* }"
|
||||
[[ "$SCRIPT_ARGS" == "$script_entry" ]] && SCRIPT_ARGS=""
|
||||
[[ "$DRY_RUN" == true ]] && SCRIPT_ARGS="$SCRIPT_ARGS --dry-run"
|
||||
|
||||
SCRIPT_NAME=$(basename "$SCRIPT_PATH")
|
||||
|
||||
if [[ ! -f "$SCRIPT_PATH" ]]; then
|
||||
warn "$SCRIPT_NAME not found at $SCRIPT_PATH — skipping"
|
||||
continue
|
||||
fi
|
||||
|
||||
log "Running: $SCRIPT_NAME"
|
||||
bash "$SCRIPT_PATH" $SCRIPT_ARGS
|
||||
EXIT_CODE=$?
|
||||
[[ "$EXIT_CODE" -ne 0 ]] && \
|
||||
warn "$SCRIPT_NAME exited with code $EXIT_CODE"
|
||||
done
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Partnership Check ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "${PARTNERSHIP_ENABLED:-false}" == true ]]; then
|
||||
PARTNER_DRY=""
|
||||
[[ "$DRY_RUN" == true ]] && PARTNER_DRY="--dry-run"
|
||||
|
||||
if [[ "$RSYNC_OK" == true ]]; then
|
||||
bash "$SCRIPT_DIR/../Partnership/partnership_manager.sh" \
|
||||
--check --remote-seen $PARTNER_DRY
|
||||
else
|
||||
bash "$SCRIPT_DIR/../Partnership/partnership_manager.sh" \
|
||||
--check --remote-unseen $PARTNER_DRY
|
||||
fi
|
||||
else
|
||||
echo "Partnership disabled — skipping check"
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
END=$(date +%s)
|
||||
DURATION=$(format_duration $(( END - START )))
|
||||
|
||||
# Silent when healthy — only show summary if there were failures or notable events
|
||||
if [[ ${#FAIL[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY CRITICAL SYNC SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_TIME Duration: $DURATION"
|
||||
[[ ${#PASS[@]} -gt 0 ]] && echo "Synced: ${PASS[*]}"
|
||||
echo "$ICON_ERROR Failed: ${FAIL[*]}"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
notify "Critical sync failed on $(hostname) ($MY_ID) — ${FAIL[*]}" \
|
||||
"Critical Sync" "warning"
|
||||
exit 1
|
||||
else
|
||||
echo "Critical sync complete — $MY_ID — ${DURATION} — ${#PASS[@]} share(s)"
|
||||
fi
|
||||
|
||||
exit 0
|
||||
+299
@@ -0,0 +1,299 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ============================= Recreate Shares ================================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Creates share directories on the correct disks after a fresh unRAID install
|
||||
# or disk rebuild. Reads all .cfg files from /boot/config/shares/ and creates
|
||||
# the corresponding directories on each disk listed in the shareInclude setting.
|
||||
# The array must be started before running — /mnt/user must be mounted.
|
||||
#
|
||||
# Typically run on HOST2 after a full disk replacement or fresh install where
|
||||
# share folders were lost but /boot/config/shares/*.cfg files were restored.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# For each share .cfg file:
|
||||
# 1. Reads shareInclude= to determine which disks own this share
|
||||
# 2. Creates /mnt/diskN/ShareName/ on each listed disk if it doesn't exist
|
||||
# 3. Places a .recovery marker file in /mnt/user/ShareName/ via the union filesystem
|
||||
#
|
||||
# .recovery Marker
|
||||
# Signals to rsync.sh that this is a fresh share with no existing data.
|
||||
# rsync.sh checks for .recovery before running with --delete:
|
||||
# .recovery present → rsync WITHOUT --delete (new files only, nothing removed)
|
||||
# .recovery absent → rsync WITH --delete (normal mirror mode)
|
||||
#
|
||||
# Self-cleaning: after the first successful rsync the source side has no .recovery
|
||||
# file, so the second nightly run deletes it from the mirror, restoring normal
|
||||
# --delete behaviour automatically. No manual cleanup needed.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Single Instance Lock
|
||||
# acquire_lock prevents duplicate runs placing duplicate .recovery markers.
|
||||
#
|
||||
# Root Required
|
||||
# mkdir on /mnt/diskN requires root.
|
||||
#
|
||||
# Array Mount Check
|
||||
# Exits cleanly if the array is not started — /mnt/user not mounted means
|
||||
# all share operations would fail silently.
|
||||
#
|
||||
# Per-Disk Guards
|
||||
# Missing disks are skipped with a warning and the rest continue — a single
|
||||
# offline disk does not abort the full run.
|
||||
#
|
||||
# Empty Config Guard
|
||||
# Warns if no share .cfg files are found — catches the case where
|
||||
# /boot/config/shares/ was not restored.
|
||||
#
|
||||
# Notification Validated
|
||||
# platform_require_cmd confirms the notify script is present before use.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# recreate_shares.sh
|
||||
# Read all .cfg files, create share directories, place .recovery markers.
|
||||
#
|
||||
# recreate_shares.sh --dry-run
|
||||
# Show what directories and markers would be created. No changes.
|
||||
#
|
||||
# recreate_shares.sh --log
|
||||
# Verbose output per share and per disk.
|
||||
#
|
||||
# recreate_shares.sh --status
|
||||
# Show which shares exist in config and which directories exist on disk.
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../../../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
SHARE_CFG_DIR="/boot/config/shares"
|
||||
MARKER_FILE=".recovery"
|
||||
|
||||
CREATED=()
|
||||
SKIPPED=()
|
||||
FAILED=()
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root — mkdir on /mnt/diskN requires root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
platform_require_cmd \
|
||||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||||
"" "" \
|
||||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||
|
||||
acquire_lock
|
||||
|
||||
# detect_hosts() sets MY_ID — used in summary
|
||||
detect_hosts
|
||||
|
||||
log "$ICON_GEAR Config: cfg-dir=${SHARE_CFG_DIR} marker=${MARKER_FILE}"
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no directories or markers will be created"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_DISK Cfg dir: $SHARE_CFG_DIR"
|
||||
echo "$ICON_DISK Marker: $MARKER_FILE"
|
||||
echo ""
|
||||
|
||||
if ! mountpoint -q /mnt/user; then
|
||||
warn "Array: NOT STARTED — /mnt/user not mounted"
|
||||
else
|
||||
echo " Array: started ✅"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "━━━ Share Config Files ━━━"
|
||||
CFG_COUNT=0
|
||||
for cfg in "$SHARE_CFG_DIR"/*.cfg; do
|
||||
[[ ! -f "$cfg" ]] && continue
|
||||
(( CFG_COUNT++ ))
|
||||
SHARE_NAME=$(basename "$cfg" .cfg)
|
||||
INCLUDE=$(grep '^shareInclude=' "$cfg" 2>/dev/null | cut -d'"' -f2)
|
||||
MARKER_EXISTS="no"
|
||||
[[ -f "/mnt/user/${SHARE_NAME}/${MARKER_FILE}" ]] && MARKER_EXISTS="yes"
|
||||
echo " $ICON_DISK $SHARE_NAME — disks: ${INCLUDE:-none} — recovery marker: $MARKER_EXISTS"
|
||||
done
|
||||
[[ "$CFG_COUNT" -eq 0 ]] && warn "No .cfg files found in $SHARE_CFG_DIR"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Pre-flight ━━━
|
||||
# ==============================================================================================
|
||||
# Array must be started — /mnt/user must be mounted
|
||||
if ! mountpoint -q /mnt/user; then
|
||||
error "Array is not started — /mnt/user is not mounted"
|
||||
warn "Start the array in the unRAID UI before running this script"
|
||||
notify "Recreate shares failed on $(hostname) — array is not started" \
|
||||
"Recreate Shares" "warning"
|
||||
exit 1
|
||||
fi
|
||||
log "Array is started — /mnt/user is mounted ✅"
|
||||
|
||||
# Check share cfg directory exists and has files
|
||||
if [[ ! -d "$SHARE_CFG_DIR" ]]; then
|
||||
error "Share config directory not found: $SHARE_CFG_DIR"
|
||||
error "Is /boot mounted? Is this the correct server?"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
CFG_FILES=("$SHARE_CFG_DIR"/*.cfg)
|
||||
if [[ ! -f "${CFG_FILES[0]}" ]]; then
|
||||
warn "No share .cfg files found in $SHARE_CFG_DIR"
|
||||
warn "Nothing to recreate — are share configs present on /boot?"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
log "Found ${#CFG_FILES[@]} share .cfg file(s) in $SHARE_CFG_DIR"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Recreate Shares ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_DISK Recreate Shares — $MY_ID ━━━"
|
||||
echo ""
|
||||
|
||||
for cfg in "${CFG_FILES[@]}"; do
|
||||
[[ ! -f "$cfg" ]] && continue
|
||||
SHARE_NAME=$(basename "$cfg" .cfg)
|
||||
INCLUDE=$(grep '^shareInclude=' "$cfg" 2>/dev/null | cut -d'"' -f2)
|
||||
|
||||
echo "━━━ $ICON_DISK $SHARE_NAME ━━━"
|
||||
|
||||
if [[ -z "$INCLUDE" ]]; then
|
||||
warn "$SHARE_NAME — no shareInclude in .cfg — skipping"
|
||||
SKIPPED+=("$SHARE_NAME")
|
||||
echo ""
|
||||
continue
|
||||
fi
|
||||
|
||||
log "$SHARE_NAME — disks: $INCLUDE"
|
||||
|
||||
SHARE_OK=true
|
||||
DIRS_CREATED=0
|
||||
DIRS_EXISTED=0
|
||||
|
||||
# Create directory on each listed disk
|
||||
IFS=',' read -ra DISKS <<< "$INCLUDE"
|
||||
for disk in "${DISKS[@]}"; do
|
||||
disk="${disk// /}" # trim whitespace
|
||||
[[ -z "$disk" ]] && continue
|
||||
|
||||
DISK_MOUNT="/mnt/${disk}"
|
||||
DISK_PATH="${DISK_MOUNT}/${SHARE_NAME}"
|
||||
|
||||
# Verify disk is mounted
|
||||
if ! mountpoint -q "$DISK_MOUNT" 2>/dev/null; then
|
||||
warn "$disk not mounted — skipping $DISK_PATH"
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ -d "$DISK_PATH" ]]; then
|
||||
log "$disk/$SHARE_NAME already exists — skipping"
|
||||
(( DIRS_EXISTED++ ))
|
||||
else
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would create: $DISK_PATH"
|
||||
(( DIRS_CREATED++ ))
|
||||
elif mkdir -p "$DISK_PATH"; then
|
||||
log "Created: $DISK_PATH ✅"
|
||||
(( DIRS_CREATED++ ))
|
||||
else
|
||||
error "Failed to create: $DISK_PATH"
|
||||
SHARE_OK=false
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
# Place .recovery marker via /mnt/user (union filesystem)
|
||||
MARKER_PATH="/mnt/user/${SHARE_NAME}/${MARKER_FILE}"
|
||||
|
||||
if [[ "$SHARE_OK" == true ]]; then
|
||||
if [[ -f "$MARKER_PATH" ]]; then
|
||||
log ".recovery marker already exists in $SHARE_NAME"
|
||||
CREATED+=("$SHARE_NAME")
|
||||
elif [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would place marker: $MARKER_PATH"
|
||||
CREATED+=("$SHARE_NAME")
|
||||
elif touch "$MARKER_PATH" 2>/dev/null; then
|
||||
log "Marker placed: $MARKER_PATH ✅"
|
||||
CREATED+=("$SHARE_NAME")
|
||||
else
|
||||
warn "$SHARE_NAME — could not place .recovery marker"
|
||||
warn "Share directory may not be visible via /mnt/user yet"
|
||||
warn "Try: touch /mnt/user/${SHARE_NAME}/.recovery manually after verifying share"
|
||||
SKIPPED+=("$SHARE_NAME")
|
||||
fi
|
||||
else
|
||||
FAILED+=("$SHARE_NAME")
|
||||
fi
|
||||
|
||||
[[ "$DIRS_CREATED" -gt 0 ]] && warn "$SHARE_NAME — created $DIRS_CREATED dir(s) on disk"
|
||||
[[ "$DIRS_EXISTED" -gt 0 ]] && log "$SHARE_NAME — $DIRS_EXISTED dir(s) already existed"
|
||||
echo ""
|
||||
done
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo "━━━━━ $ICON_SUMMARY RECREATE SHARES SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo ""
|
||||
|
||||
[[ ${#CREATED[@]} -gt 0 ]] && warn "Created + marked: ${CREATED[*]}"
|
||||
[[ ${#SKIPPED[@]} -gt 0 ]] && warn "Skipped: ${SKIPPED[*]}"
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}"
|
||||
|
||||
echo ""
|
||||
echo " Created: ${#CREATED[@]}"
|
||||
echo " Skipped: ${#SKIPPED[@]}"
|
||||
echo " Failed: ${#FAILED[@]}"
|
||||
|
||||
if [[ "$DRY_RUN" == false && ${#CREATED[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
echo "━━━ Next Steps ━━━"
|
||||
echo " 1. $ICON_HEALTH Verify shares are visible in unRAID UI"
|
||||
echo " 2. $ICON_SYNC Run initial rsync push from HOST1 → HOST2"
|
||||
echo " rsync.sh will detect .recovery markers and skip --delete"
|
||||
echo " Normal --delete mode restores automatically on second nightly run"
|
||||
echo " 3. $ICON_GEAR No manual config changes needed — markers self-clean ✅"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no changes made"
|
||||
elif [[ ${#FAILED[@]} -gt 0 ]]; then
|
||||
echo "$ICON_ERROR Status: completed with failures"
|
||||
notify "Recreate shares failed on $(hostname) ($MY_ID) — failed: ${FAILED[*]}" \
|
||||
"Recreate Shares" "warning"
|
||||
exit 1
|
||||
else
|
||||
echo "$ICON_DONE Status: done — ${#CREATED[@]} created, ${#SKIPPED[@]} skipped"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
+335
@@ -0,0 +1,335 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ================================= Server Reboot ==============================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Gracefully reboots the unRAID server with pre-flight checks, user warnings,
|
||||
# clean service shutdown, and disk sync. Use instead of raw /sbin/reboot —
|
||||
# gives users warning time and ensures services stop cleanly before the kernel
|
||||
# drops.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Shutdown Sequence
|
||||
# 1. Pre-flight warnings — rsync, mover, active Emby sessions (warn, not block)
|
||||
# 2. Wall message to all logged-in terminal users
|
||||
# 3. unRAID dashboard notification
|
||||
# 4. Wait REBOOT_SLEEP seconds — users time to save work
|
||||
# 5. array_stopping.sh — user scripts, rsync, mover, containers (verified stop)
|
||||
# 6. Graceful VM shutdown via virsh — ACPI signal, then wait REBOOT_VM_WAIT
|
||||
# 7. Stop libvirt (VM Manager)
|
||||
# 8. sync — filesystem buffers flushed to disk
|
||||
# 9. /sbin/reboot
|
||||
#
|
||||
# Pre-flight Warnings (informational — do not block)
|
||||
# rsync running → partial files possible if mid-transfer
|
||||
# mover running → files may be left mid-move on cache or array
|
||||
# Emby sessions → active streams/transcodes interrupted
|
||||
# Warnings do not block the reboot — you called this script, you know.
|
||||
#
|
||||
# VM Graceful Shutdown
|
||||
# virsh shutdown sends the ACPI power button signal — same as pressing the
|
||||
# physical power button. VM gets a chance to flush buffers and shut down.
|
||||
# After REBOOT_VM_WAIT seconds, libvirt stops anyway — reboot takes priority.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Root Required
|
||||
# /sbin/reboot requires root.
|
||||
#
|
||||
# Single Instance Lock
|
||||
# acquire_lock prevents concurrent reboot calls.
|
||||
#
|
||||
# Host Identity in All Messages
|
||||
# detect_hosts() sets MY_ID — wall and notifications show which server is
|
||||
# rebooting. Critical on a two-server setup.
|
||||
#
|
||||
# sync Before Reboot
|
||||
# filesystem buffers flushed to disk before reboot command.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# master.conf
|
||||
#
|
||||
# REBOOT_SLEEP
|
||||
# Seconds between warning and shutdown sequence start. (default: 30)
|
||||
#
|
||||
# REBOOT_VM_WAIT
|
||||
# Seconds to wait for VMs to shut down gracefully. (default: 30)
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# server_reboot.sh
|
||||
# Run pre-flight, warn users, stop services cleanly, then reboot.
|
||||
#
|
||||
# server_reboot.sh --dry-run
|
||||
# Walk through the entire shutdown sequence without stopping anything or rebooting.
|
||||
#
|
||||
# server_reboot.sh --status
|
||||
# Show running processes that would be affected: rsync, mover, VMs, containers.
|
||||
#
|
||||
# server_reboot.sh --reason="maintenance"
|
||||
# Include reason in wall message and notification. Defaults to "manual".
|
||||
#
|
||||
# server_reboot.sh --log
|
||||
# Verbose output — show each step of the shutdown sequence.
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
# ── Parse --reason flag before parse_args ─────────────────────────────────────────────────────
|
||||
REBOOT_REASON="manual"
|
||||
FILTERED_ARGS=()
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--reason=*) REBOOT_REASON="${arg#--reason=}" ;;
|
||||
*) FILTERED_ARGS+=("$arg") ;;
|
||||
esac
|
||||
done
|
||||
|
||||
parse_args "${FILTERED_ARGS[@]}"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root — reboot requires root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
validate_int REBOOT_SLEEP "$REBOOT_SLEEP"
|
||||
|
||||
acquire_lock
|
||||
|
||||
detect_hosts
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made, no reboot will occur"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY REBOOT STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_REBOOT Delay: ${REBOOT_SLEEP}s"
|
||||
echo "$ICON_GEAR VM wait: ${REBOOT_VM_WAIT:-30}s"
|
||||
echo "$ICON_GEAR Reason: $REBOOT_REASON"
|
||||
echo ""
|
||||
echo "━━━ Active Processes ━━━"
|
||||
|
||||
pgrep -x rsync >/dev/null 2>&1 && \
|
||||
warn " rsync: RUNNING — partial files if rebooted now" || \
|
||||
log " rsync: not running"
|
||||
|
||||
platform_is_mover_running && \
|
||||
warn " mover: RUNNING — files may be left mid-move" || \
|
||||
log " mover: not running"
|
||||
|
||||
if command -v virsh >/dev/null 2>&1; then
|
||||
VM_COUNT=$(virsh list --name 2>/dev/null | grep -c "." || echo 0)
|
||||
[[ "$VM_COUNT" -gt 0 ]] && \
|
||||
warn " VMs: $VM_COUNT running — will be gracefully shut down" || \
|
||||
log " VMs: none running"
|
||||
fi
|
||||
|
||||
if command -v docker >/dev/null 2>&1; then
|
||||
CONTAINER_COUNT=$(docker ps -q 2>/dev/null | wc -l || echo 0)
|
||||
log " Docker: $CONTAINER_COUNT container(s) running"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Pre-flight Warnings ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_SHIELD Pre-flight ━━━"
|
||||
WARNINGS=()
|
||||
|
||||
# rsync check — partial files if killed mid-transfer
|
||||
if pgrep -x rsync >/dev/null 2>&1; then
|
||||
RSYNC_PIDS=$(pgrep -x rsync | tr '\n' ' ')
|
||||
warn "rsync is running (PIDs: $RSYNC_PIDS) — partial files possible"
|
||||
warn "Consider: rsync_stop.sh before rebooting"
|
||||
WARNINGS+=("rsync running")
|
||||
fi
|
||||
|
||||
# mover check — files may be left mid-move
|
||||
if platform_is_mover_running; then
|
||||
warn "Mover is running — files may be left mid-move on cache or array"
|
||||
warn "Consider: mover_stop.sh before rebooting"
|
||||
WARNINGS+=("mover running")
|
||||
fi
|
||||
|
||||
# Emby sessions check — active streams interrupted
|
||||
if [[ -n "${EMBY_URL:-}" ]] && [[ -n "${EMBY_API_KEY:-}" ]]; then
|
||||
ACTIVE_STREAMS=$(curl -sf --max-time 5 \
|
||||
-H "X-Emby-Token: $EMBY_API_KEY" \
|
||||
"${EMBY_URL}/Sessions" 2>/dev/null | \
|
||||
grep -c "NowPlayingItem" 2>/dev/null || echo 0)
|
||||
ACTIVE_STREAMS="${ACTIVE_STREAMS//[^0-9]/}"
|
||||
if [[ "${ACTIVE_STREAMS:-0}" -gt 0 ]]; then
|
||||
warn "$ACTIVE_STREAMS active Emby stream(s) — will be interrupted"
|
||||
WARNINGS+=("${ACTIVE_STREAMS} Emby sessions")
|
||||
fi
|
||||
fi
|
||||
|
||||
CONTAINER_COUNT=$(docker ps -q 2>/dev/null | wc -l || echo 0)
|
||||
log "$ICON_CONTAINERS Docker: ${CONTAINER_COUNT} container(s) running"
|
||||
|
||||
if is_vm_manager_enabled && command -v virsh >/dev/null 2>&1; then
|
||||
VM_COUNT=$(virsh list --name 2>/dev/null | grep -c "." || echo 0)
|
||||
log "$ICON_GEAR VMs: ${VM_COUNT} running"
|
||||
fi
|
||||
|
||||
if [[ ${#WARNINGS[@]} -eq 0 ]]; then
|
||||
log "Pre-flight clean — no active processes to warn about"
|
||||
else
|
||||
warn "Proceeding with reboot despite warnings — ${WARNINGS[*]}"
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Notify and Wait ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_REBOOT Reboot Sequence — $MY_ID ━━━"
|
||||
echo " Reason: $REBOOT_REASON"
|
||||
echo " Delay: ${REBOOT_SLEEP}s"
|
||||
echo " Dry Run: $DRY_RUN"
|
||||
echo ""
|
||||
|
||||
START=$(date +%s)
|
||||
|
||||
if [[ "$REBOOT_SLEEP" -gt 0 ]]; then
|
||||
# Wall message — terminal users
|
||||
wall "$ICON_WARN $MY_ID ($LOCAL_SERVER_NAME) rebooting in ${REBOOT_SLEEP}s — reason: $REBOOT_REASON. Save your work now."
|
||||
|
||||
# unRAID notification — dashboard
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
notify "$MY_ID ($LOCAL_SERVER_NAME) rebooting in ${REBOOT_SLEEP}s — reason: $REBOOT_REASON${WARNINGS:+ — warnings: ${WARNINGS[*]}}" \
|
||||
"Server Reboot" "warning"
|
||||
fi
|
||||
|
||||
warn "Waiting ${REBOOT_SLEEP}s before shutdown sequence..."
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
sleep "$REBOOT_SLEEP"
|
||||
else
|
||||
warn "DRY RUN — skipping sleep"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Array Stop Orchestrator ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_STOP Array Stop Orchestrator ━━━"
|
||||
ARRAY_STOP_SCRIPT="$SCRIPT_DIR/../Orchestrators/array_stopping.sh"
|
||||
|
||||
if [[ ! -f "$ARRAY_STOP_SCRIPT" ]]; then
|
||||
warn "array_stopping.sh not found — skipping orchestrated stop"
|
||||
elif [[ "$DRY_RUN" == true ]]; then
|
||||
bash "$ARRAY_STOP_SCRIPT" --dry-run
|
||||
else
|
||||
if bash "$ARRAY_STOP_SCRIPT"; then
|
||||
log "Array stop complete ✅"
|
||||
else
|
||||
warn "array_stopping.sh reported failures — proceeding with reboot"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Graceful VM Shutdown ━━━
|
||||
# ==============================================================================================
|
||||
if is_vm_manager_enabled && command -v virsh >/dev/null 2>&1; then
|
||||
VM_LIST=$(virsh list --name 2>/dev/null | grep -v "^$" || true)
|
||||
if [[ -n "$VM_LIST" ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Graceful VM Shutdown ━━━"
|
||||
while IFS= read -r vm; do
|
||||
[[ -z "$vm" ]] && continue
|
||||
warn "Sending ACPI shutdown to VM: $vm"
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
virsh shutdown "$vm" >/dev/null 2>&1 || true
|
||||
else
|
||||
warn "DRY RUN — would virsh shutdown $vm"
|
||||
fi
|
||||
done <<< "$VM_LIST"
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
VM_WAIT="${REBOOT_VM_WAIT:-30}"
|
||||
log "Waiting ${VM_WAIT}s for VMs to shut down..."
|
||||
sleep "$VM_WAIT"
|
||||
fi
|
||||
else
|
||||
log "VM Manager enabled but no VMs running — skipping shutdown"
|
||||
fi
|
||||
else
|
||||
log "VM Manager not enabled — skipping VM shutdown"
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Stop VM Manager ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Stop VM Manager ━━━"
|
||||
if ! is_vm_manager_enabled; then
|
||||
log "VM Manager not enabled — skipping"
|
||||
elif [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would stop VM Manager (libvirt)"
|
||||
else
|
||||
if platform_stop_service libvirt; then
|
||||
warn "VM Manager stopped ✅"
|
||||
else
|
||||
warn "VM Manager stop returned non-zero — may already be stopped"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Sync Disks ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_DISK Sync Disks ━━━"
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would sync filesystem buffers"
|
||||
else
|
||||
sync
|
||||
log "Filesystem buffers flushed ✅"
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Reboot ━━━
|
||||
# ==============================================================================================
|
||||
END=$(date +%s)
|
||||
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY REBOOT SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_REBOOT Reason: $REBOOT_REASON"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
[[ ${#WARNINGS[@]} -gt 0 ]] && warn "Warnings: ${WARNINGS[*]}"
|
||||
echo ""
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — sequence complete, no reboot executed"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
else
|
||||
warn "$ICON_REBOOT Rebooting $MY_ID now..."
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
/sbin/reboot
|
||||
fi
|
||||
+557
@@ -0,0 +1,557 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ============================= Partnership Offboard ===========================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Cleanly ends a partnership. Role is detected automatically — run on either server.
|
||||
# Owner path runs the full sequence including remote cleanup and final sync.
|
||||
# Mirror path handles the local side and signals the owner to complete its own cleanup.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# OWNER PATH (10 steps)
|
||||
# Step 1: Stop rsync — halt any running sync before state changes
|
||||
# Step 2: Final sync — mirror leaves with current Critical-Data state
|
||||
# Step 3: Reconfigure WebUIs — mirror's auth WebUIs → localhost
|
||||
# Step 4: Disable sync — CRITICAL_RSYNC_ENABLED=false in master.conf
|
||||
# Step 5: Local cleanup — remove fallback coverage containers + appdata
|
||||
# Step 6: Restart own stack — bring up owner's own parked containers
|
||||
# Step 7: Remote cleanup — remove auth/arr stack + fallback containers from mirror
|
||||
# Step 8: Restart mirror — bring up mirror's own parked containers
|
||||
# Step 9: Revocation — Emby admin, SSH keys
|
||||
# Step 10: Write state — INACTIVE locally + pushed to mirror, mirror blocklisted
|
||||
# Tailscale — grace window then device removal (after state written)
|
||||
#
|
||||
# MIRROR PATH (8 steps)
|
||||
# Step 1: Stop rsync — halt any running sync
|
||||
# Step 2: Reconfigure WebUIs — local auth WebUIs → localhost
|
||||
# Step 3: Remote stack clean — remove owner-deployed containers locally (auth/arr stack)
|
||||
# Step 4: Fallback cleanup — remove fallback coverage containers
|
||||
# Step 5: Disable sync — CRITICAL_RSYNC_ENABLED=false in master.conf
|
||||
# Step 6: Revoke Emby admin — remove own admin account from local Emby instance
|
||||
# Step 7: Restart own stack — bring up own parked containers
|
||||
# Step 8: SSH revocation — revoke keys both directions, write state, signal owner
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# host*.conf
|
||||
#
|
||||
# HOST*_PARTNERSHIP_AUTH_STACK
|
||||
# Auth container XMLs to push during onboard — used on offboard to identify what
|
||||
# to remove. Owner's PARTNERSHIP_AUTH_STACK determines which containers get removed
|
||||
# from the mirror on both owner-initiated and mirror-initiated offboard.
|
||||
#
|
||||
# HOST*_PARTNERSHIP_ARR_STACK
|
||||
# Arr container XMLs — same cleanup logic as auth stack.
|
||||
#
|
||||
# HOST*_PARTNERSHIP_SERVICES_STACK
|
||||
# Shared services XMLs (Emby, Jellyfin, Seerr, SeerrFin) — same cleanup logic.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Partnership/partnership_offboard.sh
|
||||
# Full offboard — role detected automatically
|
||||
#
|
||||
# Partnership/partnership_offboard.sh --dry-run
|
||||
# Preview all steps without executing
|
||||
#
|
||||
# Partnership/partnership_offboard.sh --log
|
||||
# Verbose per-step output
|
||||
#
|
||||
# Partnership/partnership_offboard.sh --reason=<string>
|
||||
# Tag the offboard reason in state file and blocklist (default: manual)
|
||||
# Called by partnership_manager.sh --offboard (reason passed through)
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
SCRIPTS_ROOT="$SCRIPT_DIR/.."
|
||||
SSH_TIMEOUT=15
|
||||
|
||||
source "$SCRIPTS_ROOT/load_config.sh"
|
||||
source "$SCRIPTS_ROOT/Plugin/$PLATFORM/Partnership/containers.sh"
|
||||
|
||||
# ── Parse flags ───────────────────────────────────────────────────────────────────────────────
|
||||
REASON="manual"
|
||||
FILTERED_ARGS=()
|
||||
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--reason=*) REASON="${arg#--reason=}" ;;
|
||||
*) FILTERED_ARGS+=("$arg") ;;
|
||||
esac
|
||||
done
|
||||
|
||||
parse_args "${FILTERED_ARGS[@]}"
|
||||
|
||||
# ── Source partnership_manager.sh for shared helpers ──────────────────────────────────────────
|
||||
# PARTNERSHIP_LIB_MODE=1 skips mode dispatch — functions are defined, nothing is executed.
|
||||
PARTNERSHIP_LIB_MODE=1 source "$SCRIPT_DIR/partnership_manager.sh"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
[[ "$EUID" -ne 0 ]] && { error "Must be run as root"; exit 1; }
|
||||
|
||||
if ! command -v docker &>/dev/null; then
|
||||
error "Docker command not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
detect_hosts
|
||||
|
||||
OWNER_ID="${PARTNERSHIP_OWNER_HOST:-HOST1}"
|
||||
MIRROR_ID=$( [[ "$OWNER_ID" == "HOST1" ]] && echo "HOST2" || echo "HOST1" )
|
||||
OWNER="${!OWNER_ID}"
|
||||
MIRROR="${!MIRROR_ID}"
|
||||
MIRROR_SSH_KEY="$SSH_KEY"
|
||||
OWNER_SSH_KEY="$SSH_KEY"
|
||||
|
||||
AM_OWNER=false
|
||||
AM_MIRROR=false
|
||||
[[ "$MY_ID" == "$OWNER_ID" ]] && AM_OWNER=true
|
||||
[[ "$MY_ID" == "$MIRROR_ID" ]] && AM_MIRROR=true
|
||||
|
||||
LOCAL_STATE_FILE="${STATE_DIR}/partnership_${LOCAL_SERVER_NAME}.db"
|
||||
REMOTE_STATE_FILE="${STATE_DIR}/partnership_${REMOTE_SERVER_NAME}.db"
|
||||
OWNER_STATE_FILE="${STATE_DIR}/partnership_${OWNER}.db"
|
||||
MIRROR_STATE_FILE="${STATE_DIR}/partnership_${MIRROR}.db"
|
||||
OFFLINE_COUNTER="${STATE_DIR}/partnership_offline_days.db"
|
||||
|
||||
acquire_lock "strict"
|
||||
|
||||
trap _pm_trap_restart_stopped EXIT
|
||||
|
||||
# Check already offboarded
|
||||
if [[ -f "$LOCAL_STATE_FILE" ]]; then
|
||||
CURRENT_STATE=$(read_state_file "$LOCAL_STATE_FILE" "state")
|
||||
if [[ "$CURRENT_STATE" == "INACTIVE" ]]; then
|
||||
warn "Partnership already INACTIVE — use partnership_manager.sh --status to verify both servers agree"
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
|
||||
START=$(date +%s)
|
||||
|
||||
echo ""
|
||||
echo "━━━ $ICON_FALLBACK Partnership Offboard — $MY_ID ($LOCAL_SERVER_NAME) — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo ""
|
||||
echo " Role: $( [[ "$AM_OWNER" == true ]] && echo "OWNER" || echo "MIRROR" )"
|
||||
echo " This: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo " Partner: $( [[ "$AM_OWNER" == true ]] && echo "$MIRROR_ID ($MIRROR)" || echo "$OWNER_ID ($OWNER)" )"
|
||||
echo " Reason: $REASON"
|
||||
echo ""
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no permanent changes will be made"
|
||||
|
||||
# ==============================================================================================
|
||||
# ── HELPER: revoke own admin account from local Emby instance ────────────────────────────────
|
||||
#
|
||||
# Mirror-initiated path only. Called before start_own_stack so Emby is still running.
|
||||
# Uses local EMBY_API_KEY and the mirror's own short name as the username to delete.
|
||||
# ==============================================================================================
|
||||
revoke_local_emby_admin() {
|
||||
local emby_port="${PARTNERSHIP_EMBY_PORT:-8096}"
|
||||
local emby_url="http://127.0.0.1:${emby_port}"
|
||||
|
||||
echo ""
|
||||
echo "━━━ $ICON_EMBY Emby Admin Revocation ━━━"
|
||||
|
||||
if [[ "${PARTNERSHIP_PROVISION_EMBY_ADMIN:-false}" != true ]]; then
|
||||
log "PARTNERSHIP_PROVISION_EMBY_ADMIN=false — skipping"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [[ -z "${EMBY_API_KEY:-}" ]]; then
|
||||
warn "EMBY_API_KEY not set — skipping local Emby admin revocation"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# The account to revoke is this server's own short name (the mirror user's account)
|
||||
local username="${PARTNERSHIP_EMBY_ADMIN_USER:-$(derive_short_name "$LOCAL_SERVER_NAME")}"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would delete Emby admin '$username' at $emby_url"
|
||||
return 0
|
||||
fi
|
||||
|
||||
log "Looking up Emby user '$username' at $emby_url..."
|
||||
|
||||
local users_json user_id
|
||||
users_json=$(curl -sf --max-time 15 \
|
||||
-H "X-Emby-Authorization: MediaBrowser Token=\"$EMBY_API_KEY\"" \
|
||||
"${emby_url}/Users" 2>/dev/null)
|
||||
|
||||
user_id=$(echo "$users_json" | \
|
||||
grep -o "\"Id\":\"[^\"]*\"[^}]*\"Name\":\"${username}\"" | \
|
||||
grep -o '"Id":"[^"]*"' | cut -d'"' -f4 | head -1)
|
||||
|
||||
if [[ -z "$user_id" ]]; then
|
||||
warn "Emby user '$username' not found at $emby_url — may already be removed"
|
||||
return 0
|
||||
fi
|
||||
|
||||
local del_code
|
||||
del_code=$(curl -sf --max-time 15 -w "%{http_code}" -o /dev/null \
|
||||
-X DELETE \
|
||||
-H "X-Emby-Authorization: MediaBrowser Token=\"$EMBY_API_KEY\"" \
|
||||
"${emby_url}/Users/${user_id}" 2>/dev/null)
|
||||
|
||||
if [[ "$del_code" == "200" ]] || [[ "$del_code" == "204" ]] || [[ "$del_code" == "404" ]]; then
|
||||
echo "Emby admin '$username' removed ✅"
|
||||
else
|
||||
warn "Failed to delete Emby user '$username' (HTTP $del_code) — remove manually"
|
||||
fi
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ── MIRROR PATH ───────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
if [[ "$AM_MIRROR" == true ]]; then
|
||||
warn "$MIRROR_ID ($MIRROR) is initiating offboard"
|
||||
warn "Owner ($OWNER) will see INACTIVE state on its next --check cycle and finalize"
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
echo ""
|
||||
echo "You have 10 seconds to cancel (Ctrl+C)..."
|
||||
sleep 10
|
||||
fi
|
||||
|
||||
OWNER_IP=$(resolve_tailscale_ip "$OWNER")
|
||||
OWNER_REACHABLE=false
|
||||
[[ -n "$OWNER_IP" ]] && OWNER_REACHABLE=true
|
||||
|
||||
STEP_STOP_RSYNC_OK=true
|
||||
STEP_WEBUI_OK=true
|
||||
STEP_STACK_CLEANUP_OK=true
|
||||
STEP_FALLBACK_CLEANUP_OK=true
|
||||
STEP_DISABLE_RSYNC_OK=true
|
||||
STEP_EMBY_OK=true
|
||||
SSH_REVOKE_REMOTE_OK=false
|
||||
SSH_REVOKE_LOCAL_OK=false
|
||||
|
||||
# ── Step 1: Stop rsync ────────────────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_STOP Step 1/8 — Stop Rsync ━━━"
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
bash "$SCRIPTS_ROOT/Rsync/rsync_stop.sh" --rsync-only 2>/dev/null || true
|
||||
echo "Rsync stopped ✅"
|
||||
else
|
||||
warn "DRY RUN — would stop rsync"
|
||||
fi
|
||||
|
||||
# ── Step 2: Reconfigure local WebUIs → localhost ──────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_CONTAINERS Step 2/8 — Reconfigure Local WebUIs → localhost ━━━"
|
||||
|
||||
reconfigure_local_webuis "localhost" || STEP_WEBUI_OK=false
|
||||
|
||||
# ── Step 3: Remove owner-deployed containers (auth/arr stack) locally ─────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_CONTAINERS Step 3/8 — Remove Owner-Deployed Containers ━━━"
|
||||
|
||||
if [[ "$OWNER_REACHABLE" == true ]]; then
|
||||
cleanup_deployed_stack_locally "$OWNER_IP" "$OWNER_SSH_KEY" || STEP_STACK_CLEANUP_OK=false
|
||||
else
|
||||
warn "Owner unreachable — cannot read deployed stack list"
|
||||
warn "Auth/arr containers will remain — remove manually or re-run when owner is reachable"
|
||||
STEP_STACK_CLEANUP_OK=false
|
||||
fi
|
||||
|
||||
# ── Step 4: Remove fallback coverage containers ───────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_CONTAINERS Step 4/8 — Fallback Container Cleanup ━━━"
|
||||
|
||||
cleanup_partner_containers || STEP_FALLBACK_CLEANUP_OK=false
|
||||
|
||||
# ── Step 5: Disable critical sync ─────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Step 5/8 — Disable Critical Sync ━━━"
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
update_master_conf "CRITICAL_RSYNC_ENABLED" "false" && \
|
||||
warn "CRITICAL_RSYNC_ENABLED=false ✅" || \
|
||||
{ warn "Failed to update CRITICAL_RSYNC_ENABLED"; STEP_DISABLE_RSYNC_OK=false; }
|
||||
else
|
||||
warn "DRY RUN — would set CRITICAL_RSYNC_ENABLED=false"
|
||||
fi
|
||||
|
||||
# ── Step 6: Revoke Emby admin locally ─────────────────────────────────────────────────────
|
||||
revoke_local_emby_admin || STEP_EMBY_OK=false
|
||||
|
||||
# ── Step 7: Restart own stack ─────────────────────────────────────────────────────────────
|
||||
start_own_stack
|
||||
|
||||
# ── Step 8: SSH key revocation, write state, signal owner ────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_SHIELD Step 8/8 — SSH Revocation + State ━━━"
|
||||
|
||||
do_ssh_key_revocation "${OWNER_IP:-}"
|
||||
|
||||
NOW=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
write_state_file "$LOCAL_STATE_FILE" \
|
||||
"INACTIVE" "" "$NOW" "$LOCAL_SERVER_NAME" "$REASON"
|
||||
echo "Local state: INACTIVE ✅"
|
||||
add_to_blocklist "$OWNER" "$REASON"
|
||||
else
|
||||
warn "DRY RUN — would write INACTIVE state and blocklist $OWNER"
|
||||
fi
|
||||
|
||||
if [[ "$OWNER_REACHABLE" == true ]]; then
|
||||
push_state_to_remote "$LOCAL_STATE_FILE" "$OWNER_IP" "$OWNER_SSH_KEY"
|
||||
notify "Partnership offboard requested by $MIRROR — $OWNER will finalise on next check" \
|
||||
"Partnership" "normal"
|
||||
else
|
||||
warn "$OWNER unreachable — state written locally, owner will see it when reachable"
|
||||
fi
|
||||
|
||||
# ── Summary ───────────────────────────────────────────────────────────────────────────────
|
||||
END=$(date +%s)
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY OFFBOARD SUMMARY (Mirror) ━━━━━"
|
||||
echo " Mirror: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo " Owner: $OWNER_ID ($OWNER)"
|
||||
echo " Reason: $REASON"
|
||||
echo " Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
_ok() { [[ "$1" == true ]] && echo "✅" || echo "❌"; }
|
||||
_skip() { [[ "$1" == true ]] && echo "skipped" || echo "$(_ok "$2")"; }
|
||||
_revoke_status() {
|
||||
if [[ "${SSH_REVOKE_REMOTE_OK:-false}" == true && "${SSH_REVOKE_LOCAL_OK:-false}" == true ]]; then
|
||||
echo "both directions ✅"
|
||||
elif [[ "${SSH_REVOKE_LOCAL_OK:-false}" == true ]]; then
|
||||
echo "local only ✅ — remote failed (revoke manually on $OWNER)"
|
||||
else
|
||||
echo "⚠️ failed — check warnings above"
|
||||
fi
|
||||
}
|
||||
|
||||
echo " Step 1 — Stop rsync: $(_ok "$STEP_STOP_RSYNC_OK")"
|
||||
echo " Step 2 — WebUIs: $(_ok "$STEP_WEBUI_OK")"
|
||||
echo " Step 3 — Stack cleanup: $(_ok "$STEP_STACK_CLEANUP_OK")"
|
||||
echo " Step 4 — Fallback cleanup: $(_ok "$STEP_FALLBACK_CLEANUP_OK")"
|
||||
echo " Step 5 — Disable sync: $(_ok "$STEP_DISABLE_RSYNC_OK")"
|
||||
echo " Step 6 — Emby revoke: $(_ok "$STEP_EMBY_OK")"
|
||||
echo " Step 7 — Own stack: started"
|
||||
echo " Step 8 — Keys revoked: $(_revoke_status)"
|
||||
echo ""
|
||||
echo " State: INACTIVE ✅"
|
||||
echo " Blocklist: $OWNER blocked ✅"
|
||||
echo " Owner: will finalise + final sync on next --check"
|
||||
echo ""
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes made" || \
|
||||
warn "$ICON_DONE DONE — mirror separation complete ✅"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ── OWNER PATH ────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
warn "Offboarding $MIRROR_ID ($MIRROR) from partnership"
|
||||
warn "Final sync will run — mirror leaves with current state"
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
echo ""
|
||||
echo "You have 10 seconds to cancel (Ctrl+C)..."
|
||||
sleep 10
|
||||
echo "Proceeding..."
|
||||
fi
|
||||
|
||||
WEBUI_FAILURES=0
|
||||
STEP_STOP_OK=true
|
||||
STEP_SYNC_OK=true
|
||||
SSH_REVOKE_REMOTE_OK=false
|
||||
SSH_REVOKE_LOCAL_OK=false
|
||||
|
||||
# ── Step 1: Stop rsync ────────────────────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_STOP Step 1/10 — Stop Rsync ━━━"
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
bash "$SCRIPTS_ROOT/Rsync/rsync_stop.sh" --rsync-only 2>/dev/null || STEP_STOP_OK=false
|
||||
log "Rsync stopped ✅"
|
||||
else
|
||||
warn "DRY RUN — would stop rsync"
|
||||
fi
|
||||
|
||||
# ── Step 2: Final sync ────────────────────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Step 2/10 — Final Sync ━━━"
|
||||
|
||||
do_final_sync || STEP_SYNC_OK=false
|
||||
|
||||
# ── Step 3: Reconfigure mirror WebUIs → localhost ─────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_CONTAINERS Step 3/10 — Reconfigure Mirror WebUIs → localhost ━━━"
|
||||
|
||||
MIRROR_IP=$(resolve_tailscale_ip "$MIRROR")
|
||||
MIRROR_REACHABLE=false
|
||||
[[ -n "$MIRROR_IP" ]] && MIRROR_REACHABLE=true
|
||||
|
||||
if [[ "$MIRROR_REACHABLE" == true ]]; then
|
||||
for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]}"; do
|
||||
[[ -z "$entry" ]] && continue
|
||||
container="${entry%%|*}"
|
||||
port="${entry##*|}"
|
||||
reconfigure_webui "$container" "$port" "localhost" \
|
||||
"$MIRROR_SSH_KEY" "$MIRROR_IP" "$MIRROR" || (( WEBUI_FAILURES++ ))
|
||||
done
|
||||
else
|
||||
warn "$MIRROR unreachable — WebUI reconfiguration skipped"
|
||||
warn "$MIRROR will reconfigure its own WebUIs when it sees INACTIVE state on --check"
|
||||
(( WEBUI_FAILURES++ ))
|
||||
fi
|
||||
|
||||
# ── Step 4: Disable critical sync ─────────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Step 4/10 — Disable Critical Sync ━━━"
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
update_master_conf "CRITICAL_RSYNC_ENABLED" "false"
|
||||
warn "CRITICAL_RSYNC_ENABLED=false ✅"
|
||||
else
|
||||
warn "DRY RUN — would set CRITICAL_RSYNC_ENABLED=false"
|
||||
fi
|
||||
|
||||
# ── Step 5: Local container cleanup ───────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_CONTAINERS Step 5/10 — Local Container Cleanup ━━━"
|
||||
|
||||
cleanup_partner_containers
|
||||
|
||||
# ── Step 6: Restart own stack ─────────────────────────────────────────────────────────────────
|
||||
start_own_stack
|
||||
|
||||
# ── Step 7: Remote container cleanup ──────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_CONTAINERS Step 7/10 — Remote Container Cleanup ━━━"
|
||||
|
||||
if [[ "$MIRROR_REACHABLE" == true ]]; then
|
||||
# Remove auth/arr stack containers deployed during onboard (by config array)
|
||||
cleanup_deployed_stack_on_remote "$MIRROR_IP" "$MIRROR_SSH_KEY"
|
||||
# Remove fallback coverage containers (by *-owner_short naming pattern)
|
||||
cleanup_owner_containers_on_mirror "$MIRROR_IP"
|
||||
else
|
||||
warn "$MIRROR unreachable — remote container cleanup skipped"
|
||||
warn "Run 'partnership_offboard.sh' on $MIRROR to clean up manually"
|
||||
fi
|
||||
|
||||
# ── Step 8: Restart mirror's own stack ────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_START Step 8/10 — Restart Mirror Stack ━━━"
|
||||
|
||||
[[ "$MIRROR_REACHABLE" == true ]] && start_mirror_own_stack "$MIRROR_IP"
|
||||
|
||||
# ── Step 9: Revocation (Emby + SSH) ──────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_SHIELD Step 9/10 — Revocation ━━━"
|
||||
|
||||
# Emby admin — before SSH key revocation while Emby still reachable
|
||||
[[ "$MIRROR_REACHABLE" == true ]] && revoke_emby_admin "$MIRROR_IP"
|
||||
|
||||
# SSH key revocation — mutual, both directions; must run while Tailscale still active
|
||||
do_ssh_key_revocation "${MIRROR_IP:-}"
|
||||
|
||||
# ── Step 10: Write state, push to mirror, blocklist ───────────────────────────────────────────
|
||||
# State is written after container cleanup and SSH revocation so that:
|
||||
# • Re-running after a crash between steps 5–9 restarts from scratch (no early-exit on INACTIVE)
|
||||
# • --check sees INACTIVE during the Tailscale grace sleep and does not re-trigger offboard
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Step 10/10 — Write State ━━━"
|
||||
|
||||
NOW=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
write_state_file "$LOCAL_STATE_FILE" \
|
||||
"INACTIVE" "" "$NOW" "$LOCAL_SERVER_NAME" "$REASON"
|
||||
echo "Local state: INACTIVE ✅"
|
||||
add_to_blocklist "$MIRROR" "$REASON"
|
||||
[[ "$MIRROR_REACHABLE" == true ]] && \
|
||||
push_state_to_remote "$LOCAL_STATE_FILE" "$MIRROR_IP" "$MIRROR_SSH_KEY"
|
||||
else
|
||||
warn "DRY RUN — would write INACTIVE state, blocklist $MIRROR, push to remote"
|
||||
fi
|
||||
|
||||
# Tailscale removal — after state written so --check does not re-trigger offboard during grace sleep
|
||||
if [[ "${PARTNERSHIP_REMOVE_TAILSCALE:-true}" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_NET Tailscale Separation ━━━"
|
||||
if [[ "$MIRROR_REACHABLE" == true ]]; then
|
||||
grace_seconds=$(( ${PARTNERSHIP_GRACE_HOURS:-6} * 3600 ))
|
||||
warn "Waiting ${PARTNERSHIP_GRACE_HOURS:-6}hr grace — mirror can collect backups..."
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
trap 'warn "Offboard interrupted during grace sleep"; exit 0' SIGTERM SIGINT
|
||||
sleep "$grace_seconds"
|
||||
trap - SIGTERM SIGINT
|
||||
fi
|
||||
fi
|
||||
remove_tailscale_device "$MIRROR"
|
||||
fi
|
||||
|
||||
# Backup handover notification
|
||||
if [[ ${#PARTNERSHIP_MIRROR_BACKUPS[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_DISK Backup Handover ━━━"
|
||||
echo "Backups available for $MIRROR:"
|
||||
for path in "${PARTNERSHIP_MIRROR_BACKUPS[@]}"; do
|
||||
[[ -z "$path" ]] && continue
|
||||
echo " $path"
|
||||
done
|
||||
notify "$MIRROR offboard complete — backups available for ${PARTNERSHIP_GRACE_HOURS:-6}hr. Tailscale access expires then." \
|
||||
"Partnership" "warning"
|
||||
fi
|
||||
|
||||
# ── Summary ───────────────────────────────────────────────────────────────────────────────────
|
||||
END=$(date +%s)
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY OFFBOARD SUMMARY (Owner) ━━━━━"
|
||||
echo " Owner: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo " Mirror: $MIRROR_ID ($MIRROR)"
|
||||
echo " Reason: $REASON"
|
||||
echo " Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
_ok() { [[ "$1" == true ]] && echo "✅" || echo "❌"; }
|
||||
_revoke_status() {
|
||||
if [[ "${SSH_REVOKE_REMOTE_OK:-false}" == true && "${SSH_REVOKE_LOCAL_OK:-false}" == true ]]; then
|
||||
echo "both directions ✅"
|
||||
elif [[ "${SSH_REVOKE_LOCAL_OK:-false}" == true ]]; then
|
||||
echo "local only ✅ — remote failed (revoke manually on $MIRROR)"
|
||||
else
|
||||
echo "⚠️ failed — check warnings above"
|
||||
fi
|
||||
}
|
||||
|
||||
echo " Step 1 — Stop rsync: $(_ok "$STEP_STOP_OK")"
|
||||
echo " Step 2 — Final sync: $(_ok "$STEP_SYNC_OK")"
|
||||
echo " Step 3 — WebUI failures: $WEBUI_FAILURES"
|
||||
echo " Step 4 — Disable sync: ✅"
|
||||
echo " Step 5 — Local cleanup: ✅"
|
||||
echo " Step 6 — Own stack: started"
|
||||
echo " Step 7 — Remote cleanup: $( [[ "$MIRROR_REACHABLE" == true ]] && echo "✅" || echo "skipped (unreachable)" )"
|
||||
echo " Step 8 — Mirror stack: $( [[ "$MIRROR_REACHABLE" == true ]] && echo "started" || echo "skipped (unreachable)" )"
|
||||
echo " Step 9 — Keys revoked: $(_revoke_status)"
|
||||
echo " Step 10 — State: INACTIVE ✅"
|
||||
echo ""
|
||||
echo " Blocklist: $MIRROR blocked — re-onboard to permit access again ✅"
|
||||
[[ "${PARTNERSHIP_REMOVE_TAILSCALE:-true}" == true ]] && \
|
||||
echo " Tailscale: $MIRROR removed ✅"
|
||||
echo ""
|
||||
echo " $MIRROR leaves with:"
|
||||
echo " ✓ Current auth config (final sync)"
|
||||
echo " ✓ Auth WebUIs → localhost"
|
||||
echo " ✓ ${PARTNERSHIP_GRACE_HOURS:-6}hr to collect backups"
|
||||
echo ""
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes made" || \
|
||||
warn "$ICON_DONE DONE — clean separation complete ✅"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
Reference in New Issue
Block a user