massive update. Master conf split, now modular with a load sceriprt to drive all configs to scripts. with unraid scpecific safeguard tests , and improved standardized ux. including dynamic host detect, who am i who else it there. EVERY SINGLE SCRIPT UPDATED. DEBATING THAT THIS IS ACUALLY V2
This commit is contained in:
File diff suppressed because it is too large
Load Diff
+157
-51
@@ -1,53 +1,136 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- Array Start Orchestrator -----------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Single entry point for "At Startup of Array" in User Scripts plugin.
|
||||
# Launches everything configured in ARRAY_START_SCRIPTS in Master.conf.
|
||||
# ==============================================================================================
|
||||
# ================================= Array Start Orchestrator ===================================
|
||||
# ==============================================================================================
|
||||
# Single entry point for "At Startup of Array" in the User Scripts plugin.
|
||||
# Launches everything configured in ARRAY_START_SCRIPTS in master.conf.
|
||||
# This script exits after launching all scripts — unRAID sees it complete normally.
|
||||
#
|
||||
# What it launches (configured in Master.conf ARRAY_START_SCRIPTS):
|
||||
# Transcodes/ramdisk_setup.sh — creates tmpfs + symlink before Emby starts (one-shot)
|
||||
# unRAID_Essentials/docker_syslog_filter.sh — suppress veth log noise (one-shot)
|
||||
# unRAID_Essentials/php_fpm_max_children.sh — WebGUI performance tuning (one-shot)
|
||||
# Docker_Essentials/docker_network_connect.sh — ensure networks exist + connect containers (one-shot)
|
||||
# unRAID_Essentials/system_watchdog.sh — system health monitor (continuous loop)
|
||||
# Docker_Essentials/docker_watchdog.sh — container health monitor (continuous loop)
|
||||
# Failover/failover.sh — mutual failover monitor (continuous loop)
|
||||
# ── WHAT IT LAUNCHES ──────────────────────────────────────────────────────────────────────────
|
||||
# Configured in master.conf ARRAY_START_SCRIPTS — no changes to this script ever needed.
|
||||
# Current order (order matters — see below):
|
||||
#
|
||||
# One-shot scripts run and exit naturally — array_start.sh confirms completion.
|
||||
# Continuous scripts run until array stops or SIGTERM received.
|
||||
# This orchestrator exits after launching all scripts — unRAID sees it complete normally.
|
||||
# ONE-SHOT (run and exit naturally):
|
||||
# unRAID_Essentials/inotify_tuning.sh — raise inotify limits before containers start
|
||||
# unRAID_Essentials/docker_syslog_filter.sh — suppress veth log noise before logs fill
|
||||
# unRAID_Essentials/php_fpm_max_children.sh — WebGUI performance tuning
|
||||
# Transcodes/ramdisk_setup.sh — create tmpfs + symlink before Emby starts
|
||||
# Docker_Essentials/docker_network_connect.sh — ensure networks + container connections
|
||||
#
|
||||
# Add or remove scripts: edit ARRAY_START_SCRIPTS in Master.conf.
|
||||
# Order matters — ramdisk first, network before watchdogs, watchdogs before failover.
|
||||
# No changes to this script ever needed.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# CONTINUOUS (run until array stops):
|
||||
# unRAID_Essentials/system_watchdog.sh — system health monitor (last line of defense)
|
||||
# Docker_Essentials/docker_watchdog.sh — container health monitor
|
||||
# Failover/failover.sh — mutual failover monitor
|
||||
#
|
||||
# ── WHY ORDER MATTERS ─────────────────────────────────────────────────────────────────────────
|
||||
# inotify_tuning.sh — must run BEFORE Code-Server and other containers start
|
||||
# containers that start with low inotify limits keep them ✅
|
||||
# docker_syslog_filter — must run BEFORE any container starts creating veth interfaces
|
||||
# ramdisk_setup.sh — must run BEFORE Emby starts transcoding
|
||||
# docker_network_connect — must run BEFORE watchdogs check container states
|
||||
# system_watchdog.sh — before docker_watchdog (system > container priority)
|
||||
# docker_watchdog.sh — before failover (containers must be healthy for failover)
|
||||
# failover.sh — last — needs everything else stable to make decisions
|
||||
#
|
||||
# ── ONE-SHOT vs CONTINUOUS DETECTION ─────────────────────────────────────────────────────────
|
||||
# Script is launched in background with bash script.sh &
|
||||
# After 1 second: if PID still alive → continuous (running in background)
|
||||
# if PID dead + exit 0 → one-shot completed successfully
|
||||
# if PID dead + exit N → failure
|
||||
#
|
||||
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
|
||||
# Root check — all launched scripts require root
|
||||
# acquire_lock — prevents duplicate array start launches
|
||||
# detect_hosts() — MY_ID in notifications
|
||||
# validate_unraid_cmd — notify validated before use
|
||||
# chmod +x auto-fix — non-executable scripts fixed before launch
|
||||
# Full path on failure — shows exact path for debugging
|
||||
# notify on failures — alert if any script fails to launch
|
||||
#
|
||||
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||||
# ARRAY_START_SCRIPTS — ordered list of scripts to launch at array start
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# array_start.sh — normal launch (called by User Scripts at array start)
|
||||
# array_start.sh --dry-run — show what would be launched without launching
|
||||
# array_start.sh --status — show configured scripts and their current state
|
||||
# array_start.sh --log — verbose output per script
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
|
||||
source "$ECOSYSTEM_ROOT/Master.conf"
|
||||
source "$ECOSYSTEM_ROOT/common.sh"
|
||||
source "$ECOSYSTEM_ROOT/load_config.sh"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Array Start — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
parse_args "$@"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Running as root"
|
||||
validate_unraid_cmd \
|
||||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||||
"" "" \
|
||||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||
|
||||
SCRIPT_COUNT=${#ARRAY_START_SCRIPTS[@]}
|
||||
info "Launching $SCRIPT_COUNT script(s)..."
|
||||
acquire_lock
|
||||
|
||||
detect_hosts
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — scripts will not be launched"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY ARRAY START STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_GEAR Scripts: ${#ARRAY_START_SCRIPTS[@]} configured"
|
||||
echo ""
|
||||
|
||||
for relative_path in "${ARRAY_START_SCRIPTS[@]}"; do
|
||||
[[ -z "$relative_path" ]] && continue
|
||||
script_path="$ECOSYSTEM_ROOT/$relative_path"
|
||||
script_name=$(basename "$script_path")
|
||||
|
||||
if [[ ! -f "$script_path" ]]; then
|
||||
echo " $ICON_ERROR $script_name — FILE NOT FOUND"
|
||||
echo " $script_path"
|
||||
continue
|
||||
fi
|
||||
|
||||
[[ ! -x "$script_path" ]] && flag=" (not executable — will auto-fix)" || flag=""
|
||||
|
||||
# Check if currently running
|
||||
if pgrep -f "$script_path" >/dev/null 2>&1; then
|
||||
RUN_PID=$(pgrep -f "$script_path" | head -1)
|
||||
echo " $ICON_RUNNING $script_name — RUNNING (PID $RUN_PID)${flag}"
|
||||
else
|
||||
echo " $ICON_NOT_RUNNING $script_name — not running${flag}"
|
||||
fi
|
||||
done
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Launch Scripts ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Array Start — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
log "Ecosystem root: $ECOSYSTEM_ROOT"
|
||||
log "Launching ${#ARRAY_START_SCRIPTS[@]} script(s)..."
|
||||
echo ""
|
||||
|
||||
START=$(date +%s)
|
||||
LAUNCHED=0
|
||||
FAILED=0
|
||||
FAILED_SCRIPTS=()
|
||||
|
||||
for relative_path in "${ARRAY_START_SCRIPTS[@]}"; do
|
||||
[[ -z "$relative_path" ]] && continue
|
||||
@@ -55,57 +138,80 @@ for relative_path in "${ARRAY_START_SCRIPTS[@]}"; do
|
||||
SCRIPT_PATH="$ECOSYSTEM_ROOT/$relative_path"
|
||||
SCRIPT_NAME=$(basename "$SCRIPT_PATH")
|
||||
|
||||
# File existence check
|
||||
if [[ ! -f "$SCRIPT_PATH" ]]; then
|
||||
error "$SCRIPT_NAME — not found at $SCRIPT_PATH"
|
||||
((FAILED++))
|
||||
error "$SCRIPT_NAME — not found"
|
||||
error " Expected: $SCRIPT_PATH"
|
||||
(( FAILED++ ))
|
||||
FAILED_SCRIPTS+=("$SCRIPT_NAME")
|
||||
continue
|
||||
fi
|
||||
|
||||
# Auto-fix permissions — chmod +x if needed
|
||||
if [[ ! -x "$SCRIPT_PATH" ]]; then
|
||||
error "$SCRIPT_NAME — not executable"
|
||||
((FAILED++))
|
||||
warn "$SCRIPT_NAME — not executable, fixing..."
|
||||
chmod +x "$SCRIPT_PATH" || {
|
||||
error "$SCRIPT_NAME — chmod +x failed"
|
||||
(( FAILED++ ))
|
||||
FAILED_SCRIPTS+=("$SCRIPT_NAME")
|
||||
continue
|
||||
}
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would launch: $SCRIPT_NAME"
|
||||
(( LAUNCHED++ ))
|
||||
continue
|
||||
fi
|
||||
|
||||
info "$ICON_START Launching $SCRIPT_NAME..."
|
||||
log "$ICON_START Launching $SCRIPT_NAME..."
|
||||
bash "$SCRIPT_PATH" &
|
||||
PID=$!
|
||||
|
||||
# Brief pause to let script initialize and catch immediate failures
|
||||
# Brief settle — 1s enough to detect immediate failures
|
||||
sleep 1
|
||||
|
||||
if kill -0 "$PID" 2>/dev/null; then
|
||||
success "$SCRIPT_NAME — running (PID $PID)"
|
||||
((LAUNCHED++))
|
||||
# Still running → continuous script
|
||||
warn "$SCRIPT_NAME — running (PID $PID) ✅"
|
||||
(( LAUNCHED++ ))
|
||||
else
|
||||
# Script exited — check if it was a one-shot (exit 0) or a failure
|
||||
# Exited — check if one-shot success or failure
|
||||
wait "$PID"
|
||||
EXIT_CODE=$?
|
||||
if [[ "$EXIT_CODE" -eq 0 ]]; then
|
||||
success "$SCRIPT_NAME — completed (one-shot)"
|
||||
((LAUNCHED++))
|
||||
log "$SCRIPT_NAME — completed (one-shot) ✅"
|
||||
(( LAUNCHED++ ))
|
||||
else
|
||||
error "$SCRIPT_NAME — exited with code $EXIT_CODE"
|
||||
((FAILED++))
|
||||
error " Path: $SCRIPT_PATH"
|
||||
(( FAILED++ ))
|
||||
FAILED_SCRIPTS+=("$SCRIPT_NAME")
|
||||
fi
|
||||
fi
|
||||
|
||||
done
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Summary ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
END=$(date +%s)
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY ARRAY START SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_SUCCESS Launched: $LAUNCHED"
|
||||
echo "$ICON_ERROR Failed: $FAILED"
|
||||
echo "$ICON_TIME Time: $(date '+%H:%M:%S')"
|
||||
[[ "$FAILED" -gt 0 ]] && echo "$ICON_ERROR Failed: $FAILED — ${FAILED_SCRIPTS[*]}"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
|
||||
if [[ "$FAILED" -gt 0 ]]; then
|
||||
echo "$ICON_WARN Status: $FAILED script(s) failed to launch — check logs"
|
||||
notify "Array start on $(hostname) — $FAILED script(s) failed to launch" "Array Start" "warning"
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no scripts launched"
|
||||
elif [[ "$FAILED" -gt 0 ]]; then
|
||||
warn "Status: $FAILED script(s) failed — ${FAILED_SCRIPTS[*]}"
|
||||
notify "Array start on $(hostname) ($MY_ID) — $FAILED script(s) failed: ${FAILED_SCRIPTS[*]}" \
|
||||
"Array Start" "warning"
|
||||
else
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS All scripts launched"
|
||||
log "$ICON_DONE Status: all $LAUNCHED script(s) launched ✅"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
@@ -1,69 +1,136 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ----------------------------- Critical Sync Maintenance --------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ============================= Critical Sync Maintenance ======================================
|
||||
# ==============================================================================================
|
||||
# Orchestrator for time-sensitive syncs that run every 15 minutes.
|
||||
# Keeps the mirror current between the less frequent daily and weekly windows.
|
||||
# Schedule: */15 * * * * (every 15 minutes)
|
||||
# Schedule: */15 * * * * (every 15 minutes via User Scripts plugin)
|
||||
#
|
||||
# Execution order:
|
||||
# ── EXECUTION ORDER ───────────────────────────────────────────────────────────────────────────
|
||||
# 1. Critical-Data rsync — auth stack, NPM config, certs (containers stopped both sides)
|
||||
# 2. emby-failover rsync — dirty Emby sync (watch states, library — Emby stays running)
|
||||
# 3. partnership --check — read both state files, detect changes, act accordingly
|
||||
# 3. CRITICAL_MAINTENANCE_SCRIPTS — any scripts configured for critical window
|
||||
# 4. partnership --check — read both state files, detect changes, act accordingly
|
||||
#
|
||||
# Why every 15 minutes:
|
||||
# ── WHY EVERY 15 MINUTES ──────────────────────────────────────────────────────────────────────
|
||||
# Auth stack changes (new users, proxy rules, certs) propagate within 15min ✅
|
||||
# Emby watch states stay in sync — mirror users see correct playback position ✅
|
||||
# Partnership state changes detected and acted on quickly ✅
|
||||
# Lock prevents: daily rsync doing Critical-Data mid-critical window ✅
|
||||
#
|
||||
# Rsync gate:
|
||||
# RSYNC_ENABLED=false → skips all syncs (global gate)
|
||||
# CRITICAL_RSYNC_ENABLED=false → skips critical syncs (per-orchestrator)
|
||||
# partnership --check still runs regardless of rsync gate
|
||||
# (state check doesn't need rsync to work)
|
||||
# ── RSYNC GATE ────────────────────────────────────────────────────────────────────────────────
|
||||
# RSYNC_ENABLED=false → skips all syncs (global gate)
|
||||
# CRITICAL_RSYNC_ENABLED=false → skips critical syncs only (per-orchestrator gate)
|
||||
# partnership --check always runs regardless — state check doesn't need rsync
|
||||
#
|
||||
# Lock behavior:
|
||||
# acquire_lock "strict" — if previous 15min run still going, skip this cycle
|
||||
# ── LOCK BEHAVIOUR ────────────────────────────────────────────────────────────────────────────
|
||||
# acquire_lock "strict" — if previous 15min run still going, skip this cycle entirely
|
||||
# Critical-Data taking > 15min is a problem worth knowing about
|
||||
# Lock prevents pile-up ✅
|
||||
# Strict mode prevents pile-up without waiting — log and move on ✅
|
||||
#
|
||||
# Configuration in Master.conf:
|
||||
# CRITICAL_RSYNC_ENABLED — enable/disable rsync section
|
||||
# PARTNERSHIP_ENABLED — enable/disable partnership check
|
||||
# CRITICAL_SYNC_SHARES — shares synced every 15min
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ── SILENT WHEN HEALTHY ───────────────────────────────────────────────────────────────────────
|
||||
# Runs 96 times per day — clean runs must produce zero output ✅
|
||||
# Only failures and notable events produce visible output
|
||||
#
|
||||
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||||
# CRITICAL_RSYNC_ENABLED — enable/disable rsync section
|
||||
# CRITICAL_SYNC_SHARES — shares synced every 15min (HOST*_CRITICAL_SYNC_SHARES)
|
||||
# CRITICAL_MAINTENANCE_SCRIPTS — scripts run in critical window (optional)
|
||||
# PARTNERSHIP_ENABLED — enable/disable partnership check
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# critical_sync_maintenance.sh — normal run
|
||||
# critical_sync_maintenance.sh --dry-run — preview syncs without transferring
|
||||
# critical_sync_maintenance.sh --log — verbose per-share output
|
||||
# critical_sync_maintenance.sh --status — show configuration and exit
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
validate_unraid_cmd \
|
||||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||||
"" "" \
|
||||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||
|
||||
acquire_lock "strict"
|
||||
|
||||
detect_hosts
|
||||
resolve_remote_ip
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY CRITICAL SYNC STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_NET Remote: $REMOTE_ID ($REMOTE_SERVER_NAME)"
|
||||
echo "$ICON_SYNC Rsync enabled: ${RSYNC_ENABLED:-true}"
|
||||
echo "$ICON_SYNC Critical enabled: ${CRITICAL_RSYNC_ENABLED:-false}"
|
||||
echo "$ICON_SHIELD Partnership: ${PARTNERSHIP_ENABLED:-false}"
|
||||
echo ""
|
||||
echo "━━━ Critical Sync Shares ━━━"
|
||||
if [[ ${#CRITICAL_SYNC_SHARES[@]} -eq 0 ]]; then
|
||||
warn " No CRITICAL_SYNC_SHARES configured"
|
||||
else
|
||||
for share in "${CRITICAL_SYNC_SHARES[@]:-}"; do
|
||||
[[ -z "$share" ]] && continue
|
||||
SHARE_PATH="${share%%|*}"
|
||||
SHARE_PROFILE="${share##*|}"
|
||||
SHARE_NAME=$(basename "$SHARE_PATH")
|
||||
[[ "$SHARE_PATH" == "$SHARE_PROFILE" ]] && \
|
||||
echo " $ICON_SYNC $SHARE_NAME — no profile" || \
|
||||
echo " $ICON_SYNC $SHARE_NAME — profile: $SHARE_PROFILE"
|
||||
done
|
||||
fi
|
||||
echo ""
|
||||
echo "━━━ Critical Maintenance Scripts ━━━"
|
||||
if [[ ${#CRITICAL_MAINTENANCE_SCRIPTS[@]} -eq 0 ]]; then
|
||||
echo " None configured"
|
||||
else
|
||||
for entry in "${CRITICAL_MAINTENANCE_SCRIPTS[@]:-}"; do
|
||||
[[ -z "$entry" || "$entry" == \#* ]] && continue
|
||||
echo " $ICON_GEAR $(basename "${entry%% *}")"
|
||||
done
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Critical Shares Sync ━━━
|
||||
# ==============================================================================================
|
||||
START=$(date +%s)
|
||||
RSYNC_OK=false
|
||||
PASS=()
|
||||
FAIL=()
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SYNC Critical Shares Sync ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Critical Sync — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
|
||||
if ! check_rsync_enabled "CRITICAL"; then
|
||||
warn "Critical rsync disabled — skipping sync, running partnership check only"
|
||||
log "Critical rsync disabled — skipping sync, running partnership check only"
|
||||
elif [[ ${#CRITICAL_SYNC_SHARES[@]} -eq 0 ]]; then
|
||||
warn "CRITICAL_RSYNC_ENABLED=true but CRITICAL_SYNC_SHARES is empty for $MY_ID"
|
||||
warn "Check HOST*_CRITICAL_SYNC_SHARES in master_host*.conf"
|
||||
else
|
||||
log "Critical sync — $MY_ID → $REMOTE_ID — $(date '+%H:%M:%S')"
|
||||
|
||||
# Build dry-run flag to pass through
|
||||
RSYNC_DRY=""
|
||||
[[ "$DRY_RUN" == true ]] && RSYNC_DRY="--dry-run"
|
||||
|
||||
for share in "${CRITICAL_SYNC_SHARES[@]:-}"; do
|
||||
[[ -z "$share" ]] && continue
|
||||
|
||||
@@ -72,101 +139,94 @@ else
|
||||
SHARE_PROFILE="${share##*|}"
|
||||
SHARE_NAME=$(basename "$SHARE_PATH")
|
||||
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC $SHARE_NAME ━━━"
|
||||
|
||||
SHARE_START=$(date +%s)
|
||||
|
||||
if [[ "$SHARE_PATH" == "$SHARE_PROFILE" ]]; then
|
||||
# No profile specified
|
||||
bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$SHARE_PATH"
|
||||
bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$SHARE_PATH" $RSYNC_DRY
|
||||
else
|
||||
bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$SHARE_PATH" --profile="$SHARE_PROFILE"
|
||||
bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$SHARE_PATH" \
|
||||
--profile="$SHARE_PROFILE" $RSYNC_DRY
|
||||
fi
|
||||
|
||||
RSYNC_EXIT=$?
|
||||
SHARE_END=$(date +%s)
|
||||
SHARE_DUR=$(format_duration $(( SHARE_END - SHARE_START )))
|
||||
SHARE_DUR=$(format_duration $(( $(date +%s) - SHARE_START )))
|
||||
|
||||
if [[ "$RSYNC_EXIT" -eq 0 ]]; then
|
||||
PASS+=("$SHARE_NAME")
|
||||
success "$SHARE_NAME — done in $SHARE_DUR ✅"
|
||||
log "$SHARE_NAME — done in $SHARE_DUR ✅"
|
||||
RSYNC_OK=true
|
||||
else
|
||||
FAIL+=("$SHARE_NAME")
|
||||
error "$SHARE_NAME — failed after $SHARE_DUR"
|
||||
error "$SHARE_NAME — failed after $SHARE_DUR (exit $RSYNC_EXIT)"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_CLEAN Critical Maintenance Scripts ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_CLEAN Critical Maintenance — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
|
||||
if [[ ${#CRITICAL_MAINTENANCE_SCRIPTS[@]} -eq 0 ]]; then
|
||||
log "No CRITICAL_MAINTENANCE_SCRIPTS defined — skipping"
|
||||
else
|
||||
# ==============================================================================================
|
||||
# ━━━ Critical Maintenance Scripts ━━━
|
||||
# ==============================================================================================
|
||||
if [[ ${#CRITICAL_MAINTENANCE_SCRIPTS[@]} -gt 0 ]]; then
|
||||
for script_entry in "${CRITICAL_MAINTENANCE_SCRIPTS[@]}"; do
|
||||
[[ -z "$script_entry" ]] && continue
|
||||
# Strip leading comment lines
|
||||
[[ "$script_entry" == \#* ]] && continue
|
||||
[[ -z "$script_entry" || "$script_entry" == \#* ]] && continue
|
||||
|
||||
SCRIPT_PATH="$SCRIPT_DIR/../${script_entry%% *}"
|
||||
SCRIPT_ARGS="${script_entry#* }"
|
||||
[[ "$SCRIPT_ARGS" == "$script_entry" ]] && SCRIPT_ARGS=""
|
||||
[[ "$DRY_RUN" == true ]] && SCRIPT_ARGS="$SCRIPT_ARGS --dry-run"
|
||||
|
||||
SCRIPT_NAME=$(basename "$SCRIPT_PATH")
|
||||
echo " → $SCRIPT_NAME"
|
||||
|
||||
if [[ ! -f "$SCRIPT_PATH" ]]; then
|
||||
warn "$SCRIPT_NAME not found at $SCRIPT_PATH — skipping"
|
||||
continue
|
||||
fi
|
||||
|
||||
log "Running: $SCRIPT_NAME"
|
||||
bash "$SCRIPT_PATH" $SCRIPT_ARGS
|
||||
EXIT_CODE=$?
|
||||
if [[ "$EXIT_CODE" -ne 0 ]]; then
|
||||
[[ "$EXIT_CODE" -ne 0 ]] && \
|
||||
warn "$SCRIPT_NAME exited with code $EXIT_CODE"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SHIELD Partnership Check ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_SHIELD Partnership Check ━━━"
|
||||
# ==============================================================================================
|
||||
# ━━━ Partnership Check ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "${PARTNERSHIP_ENABLED:-false}" == true ]]; then
|
||||
PARTNER_DRY=""
|
||||
[[ "$DRY_RUN" == true ]] && PARTNER_DRY="--dry-run"
|
||||
|
||||
if [[ "${PARTNERSHIP_ENABLED:-false}" == false ]]; then
|
||||
log "Partnership disabled — skipping check"
|
||||
else
|
||||
# Pass rsync outcome to --check so it can update last_seen_remote
|
||||
if [[ "$RSYNC_OK" == true ]]; then
|
||||
bash "$SCRIPT_DIR/partnership_manage.sh" --check --remote-seen
|
||||
bash "$SCRIPT_DIR/partnership_manage.sh" \
|
||||
--check --remote-seen $PARTNER_DRY
|
||||
else
|
||||
bash "$SCRIPT_DIR/partnership_manage.sh" --check --remote-unseen
|
||||
bash "$SCRIPT_DIR/partnership_manage.sh" \
|
||||
--check --remote-unseen $PARTNER_DRY
|
||||
fi
|
||||
else
|
||||
log "Partnership disabled — skipping check"
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Summary ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
END=$(date +%s)
|
||||
DURATION=$(format_duration $(( END - START )))
|
||||
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY CRITICAL SYNC SUMMARY ━━━━━"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
|
||||
if [[ ${#PASS[@]} -gt 0 ]]; then
|
||||
echo "$ICON_SUCCESS Synced: ${PASS[*]}"
|
||||
fi
|
||||
# Silent when healthy — only show summary if there were failures or notable events
|
||||
if [[ ${#FAIL[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY CRITICAL SYNC SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_TIME Duration: $DURATION"
|
||||
[[ ${#PASS[@]} -gt 0 ]] && log "Synced: ${PASS[*]}"
|
||||
echo "$ICON_ERROR Failed: ${FAIL[*]}"
|
||||
notify "Critical sync failed on $(hostname) — ${FAIL[*]}" "Critical Sync" "warning"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
notify "Critical sync failed on $(hostname) ($MY_ID) — ${FAIL[*]}" \
|
||||
"Critical Sync" "warning"
|
||||
exit 1
|
||||
else
|
||||
log "Critical sync complete — $MY_ID — ${DURATION} — ${#PASS[@]} share(s)"
|
||||
fi
|
||||
if [[ ${#PASS[@]} -eq 0 ]] && [[ ${#FAIL[@]} -eq 0 ]]; then
|
||||
echo "$ICON_SKIP Rsync: disabled"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
exit 0
|
||||
@@ -1,81 +1,139 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- Daily Sync Maintenance ------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ============================= Daily Sync Maintenance =========================================
|
||||
# ==============================================================================================
|
||||
# Daily orchestrator — runs the full daily maintenance window in the correct order.
|
||||
# Schedule: 0 1 * * * (1am daily)
|
||||
# Schedule: 0 1 * * * (1am daily via User Scripts plugin)
|
||||
#
|
||||
# Execution order:
|
||||
# 1. git_pull_execute.sh — pull latest scripts first, always
|
||||
# 2. Media share sync — HOST*_DAILY_SYNC_SHARES pushed to remote
|
||||
# 3. HOST*_PERSONAL_SHARES — personal encrypted shares after media
|
||||
# 4. media_shares_permissions.sh — fix ownership before arr cleanup
|
||||
# 5. media_cleaner.sh anime — remove junk from anime shares
|
||||
# 6. media_cleaner.sh media — remove junk from media shares
|
||||
# 7. lidarr_cleanup.sh — remove orphaned music files
|
||||
# 8. sonarr_cleanup.sh — remove orphaned TV files
|
||||
# 9. radarr_cleanup.sh — remove orphaned movie files
|
||||
# 10. docker_daily_restart.sh — restart containers that need daily restart
|
||||
# ── EXECUTION ORDER ───────────────────────────────────────────────────────────────────────────
|
||||
# Pre-sync:
|
||||
# git_pull_execute.sh — pull latest scripts first, always
|
||||
#
|
||||
# What triggers weekly_health_digest.sh:
|
||||
# NOT this script — weekly_health_digest.sh runs on its own schedule (Saturday)
|
||||
# This script writes no stats — it just syncs and maintains
|
||||
# Rsync window (DAILY_SYNC_SHARES per host):
|
||||
# HOST*_DAILY_SYNC_SHARES — media shares pushed to mirror
|
||||
# HOST*_PERSONAL_SHARES — encrypted personal shares
|
||||
#
|
||||
# Configuration in Master.conf:
|
||||
# DAILY_MAINTENANCE_SCRIPTS — pre/post-sync scripts (git pull, docker restart)
|
||||
# DAILY_MAINTENANCE_SCRIPTS — media maintenance jobs run after sync
|
||||
# HOST1_DAILY_SYNC_SHARES — shares HOST1 pushes to HOST2
|
||||
# HOST2_DAILY_SYNC_SHARES — shares HOST2 pushes to HOST1
|
||||
# HOST1/2_PERSONAL_SHARES — encrypted personal shares
|
||||
# Post-sync maintenance (DAILY_MAINTENANCE_SCRIPTS):
|
||||
# media_shares_permissions.sh — fix ownership before arr cleanup
|
||||
# media_cleaner.sh anime — remove junk from anime shares
|
||||
# media_cleaner.sh media — remove junk from media shares
|
||||
# lidarr_cleanup.sh — remove orphaned music files
|
||||
# sonarr_cleanup.sh — remove orphaned TV files
|
||||
# radarr_cleanup.sh — remove orphaned movie files
|
||||
# docker_daily_restart.sh — restart containers needing daily restart
|
||||
#
|
||||
# ── WHY ORDER MATTERS ─────────────────────────────────────────────────────────────────────────
|
||||
# git pull first — maintenance runs on latest code, not yesterday's
|
||||
# Rsync before cleanup — cleanup sees the post-sync state
|
||||
# Permissions before arr cleanup — arrs need correct ownership to delete/rename
|
||||
# Arr cleanup after permissions — clean ownership = successful orphan deletion
|
||||
# Docker restart last — containers already processed by cleanup
|
||||
#
|
||||
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
|
||||
# Bidirectional — same script runs on both servers, correct direction automatic.
|
||||
# Per-share rsync handled by rsync.sh — this script tracks pass/fail and total time.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# detect_hosts() aliases DAILY_SYNC_SHARES and PERSONAL_SHARES from HOST*_ vars.
|
||||
# No manual HOST1/HOST2 comparisons — MY_ID routes correctly on any server.
|
||||
#
|
||||
# ── DRIVE TEMP HANDLING ───────────────────────────────────────────────────────────────────────
|
||||
# rsync.sh returns exit codes for temperature issues:
|
||||
# exit 1 = temp WARN — skip this share, continue to next
|
||||
# exit 2 = temp CRITICAL — abort ALL remaining syncs in this window
|
||||
# All other failures — skip share, continue to next
|
||||
#
|
||||
# ── SILENT WHEN HEALTHY ───────────────────────────────────────────────────────────────────────
|
||||
# Runs daily at 1am — clean run should produce minimal output.
|
||||
# Each job reports log() on success (silent), warn()/error() on failure (visible).
|
||||
# Summary always shown — gives window timing and share/job counts.
|
||||
# Notify only on failure — successful daily maintenance doesn't need notification.
|
||||
#
|
||||
# ── CONFIGURATION (master.conf + master_host*.conf) ───────────────────────────────────────────
|
||||
# HOST*_DAILY_SYNC_SHARES — shares pushed to mirror each day
|
||||
# HOST*_PERSONAL_SHARES — encrypted personal shares
|
||||
# DAILY_MAINTENANCE_SCRIPTS — maintenance jobs (permissions, cleanup, restart)
|
||||
# DAILY_RSYNC_ENABLED — enable/disable rsync section
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# daily_sync_maintenance.sh — normal run
|
||||
# daily_sync_maintenance.sh --dry-run — preview without syncing or changing
|
||||
# daily_sync_maintenance.sh --log — verbose per-share/per-job output
|
||||
# daily_sync_maintenance.sh --status — show configured shares and jobs
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
RSYNC_SCRIPT="$SCRIPT_DIR/../Rsync/rsync.sh"
|
||||
SCRIPTS_ROOT="$SCRIPT_DIR/.."
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
parse_args "$@"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
validate_unraid_cmd \
|
||||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||||
"" "" \
|
||||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||
|
||||
detect_hosts
|
||||
resolve_remote_ip
|
||||
|
||||
acquire_lock
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SHIELD Pre-flight Checks ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_SHIELD Pre-flight Checks ━━━"
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
|
||||
check_connectivity
|
||||
check_remote_rootfs
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY DAILY SYNC STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_NET Remote: $REMOTE_ID ($REMOTE_SERVER_NAME)"
|
||||
echo "$ICON_SYNC Rsync enabled: ${RSYNC_ENABLED:-true}"
|
||||
echo "$ICON_SYNC Daily enabled: ${DAILY_RSYNC_ENABLED:-false}"
|
||||
echo ""
|
||||
echo "━━━ Daily Sync Shares ━━━"
|
||||
for share in "${DAILY_SYNC_SHARES[@]:-}"; do
|
||||
[[ -n "$share" ]] && echo " $ICON_SYNC $share"
|
||||
done
|
||||
for share in "${PERSONAL_SHARES[@]:-}"; do
|
||||
[[ -n "$share" ]] && echo " $ICON_SYNC $share (personal)"
|
||||
done
|
||||
echo ""
|
||||
echo "━━━ Daily Maintenance Scripts ━━━"
|
||||
for entry in "${DAILY_MAINTENANCE_SCRIPTS[@]:-}"; do
|
||||
[[ -z "$entry" ]] && continue
|
||||
echo " $ICON_GEAR $(basename "${entry%% *}") ${entry#* }"
|
||||
done
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
WINDOW_START=$(date +%s)
|
||||
JOB_PASS=()
|
||||
JOB_FAIL=()
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GIT Pre-sync Jobs ━━━
|
||||
# git_pull_execute.sh runs first — pulls latest scripts before anything else runs
|
||||
# Identified by script name — all other DAILY_MAINTENANCE_SCRIPTS run after sync
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Pre-sync Jobs ━━━"
|
||||
# ==============================================================================================
|
||||
# ── BUILD SHARE LIST via detect_hosts aliases ─────────────────────────────────────────────────
|
||||
# detect_hosts() sets DAILY_SYNC_SHARES and PERSONAL_SHARES from HOST*_ vars
|
||||
# No manual HOST1/HOST2 comparison needed — aliased automatically per server
|
||||
# ==============================================================================================
|
||||
ALL_SHARES=()
|
||||
for share in "${DAILY_SYNC_SHARES[@]:-}"; do
|
||||
[[ -n "$share" ]] && ALL_SHARES+=("$share")
|
||||
done
|
||||
for share in "${PERSONAL_SHARES[@]:-}"; do
|
||||
[[ -n "$share" ]] && ALL_SHARES+=("$share")
|
||||
done
|
||||
SHARE_COUNT=${#ALL_SHARES[@]}
|
||||
|
||||
# ── Split maintenance scripts: git pull runs pre-sync, rest run post-sync ─────────────────────
|
||||
PRE_SYNC_SCRIPTS=()
|
||||
POST_SYNC_SCRIPTS=()
|
||||
|
||||
for script_entry in "${DAILY_MAINTENANCE_SCRIPTS[@]}"; do
|
||||
for script_entry in "${DAILY_MAINTENANCE_SCRIPTS[@]:-}"; do
|
||||
[[ -z "$script_entry" ]] && continue
|
||||
script_name=$(basename "${script_entry%% *}")
|
||||
if [[ "$script_name" == "git_pull_execute.sh" ]]; then
|
||||
@@ -85,236 +143,181 @@ for script_entry in "${DAILY_MAINTENANCE_SCRIPTS[@]}"; do
|
||||
fi
|
||||
done
|
||||
|
||||
for script_entry in "${PRE_SYNC_SCRIPTS[@]}"; do
|
||||
[[ -z "$script_entry" ]] && continue
|
||||
script_args=($script_entry)
|
||||
script_path="$SCRIPTS_ROOT/${script_args[0]}"
|
||||
script_name=$(basename "${script_args[0]}")
|
||||
extra_args=("${script_args[@]:1}")
|
||||
# Helper — run a maintenance script, track pass/fail
|
||||
run_job() {
|
||||
local script_entry="$1"
|
||||
local extra_dry=""
|
||||
[[ "$DRY_RUN" == true ]] && extra_dry="--dry-run"
|
||||
|
||||
echo ""
|
||||
info "$ICON_START Running: $script_name"
|
||||
read -r -a script_args <<< "$script_entry"
|
||||
local script_path="$SCRIPTS_ROOT/${script_args[0]}"
|
||||
local script_name
|
||||
script_name=$(basename "${script_args[0]}")
|
||||
local extra_args=("${script_args[@]:1}")
|
||||
|
||||
if [[ ! -f "$script_path" ]]; then
|
||||
error "$script_name — not found at $script_path"
|
||||
JOB_FAIL+=("$script_name")
|
||||
continue
|
||||
return 1
|
||||
fi
|
||||
|
||||
if bash "$script_path" "${extra_args[@]}"; then
|
||||
success "$script_name — done"
|
||||
JOB_PASS+=("$script_name")
|
||||
log "Running: $script_name ${extra_args[*]}"
|
||||
# shellcheck disable=SC2086
|
||||
if bash "$script_path" "${extra_args[@]}" $extra_dry; then
|
||||
log "$script_name — done ✅"
|
||||
JOB_PASS+=("$script_name ${extra_args[*]}")
|
||||
else
|
||||
error "$script_name — failed"
|
||||
JOB_FAIL+=("$script_name")
|
||||
error "$script_name — failed (exit $?)"
|
||||
JOB_FAIL+=("$script_name ${extra_args[*]}")
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Build share list — host-specific truth shares + personal shares
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
WINDOW_START=$(date +%s)
|
||||
JOB_PASS=()
|
||||
JOB_FAIL=()
|
||||
PASS=()
|
||||
FAIL=()
|
||||
SHARE_TIMES=()
|
||||
TOTAL_START=$(date +%s)
|
||||
|
||||
ALL_SHARES=()
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Daily Maintenance — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
|
||||
if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then
|
||||
for share in "${HOST1_DAILY_SYNC_SHARES[@]}"; do
|
||||
[[ -n "$share" ]] && ALL_SHARES+=("$share")
|
||||
done
|
||||
for share in "${HOST1_PERSONAL_SHARES[@]}"; do
|
||||
[[ -n "$share" ]] && ALL_SHARES+=("$share")
|
||||
done
|
||||
elif [[ "$LOCAL_SERVER_NAME" == "$HOST2" ]]; then
|
||||
for share in "${HOST2_DAILY_SYNC_SHARES[@]}"; do
|
||||
[[ -n "$share" ]] && ALL_SHARES+=("$share")
|
||||
done
|
||||
for share in "${HOST2_PERSONAL_SHARES[@]}"; do
|
||||
[[ -n "$share" ]] && ALL_SHARES+=("$share")
|
||||
# ==============================================================================================
|
||||
# ━━━ Pre-sync — git pull ━━━
|
||||
# ==============================================================================================
|
||||
if [[ ${#PRE_SYNC_SCRIPTS[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_GIT Pre-sync ━━━"
|
||||
for script_entry in "${PRE_SYNC_SCRIPTS[@]}"; do
|
||||
run_job "$script_entry"
|
||||
done
|
||||
fi
|
||||
|
||||
SHARE_COUNT=${#ALL_SHARES[@]}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SYNC Media Share Sync ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Media Share Sync — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo "$ICON_SUMMARY Shares: $SHARE_COUNT"
|
||||
# ==============================================================================================
|
||||
# ━━━ Media Share Sync ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Media Share Sync — $SHARE_COUNT share(s) ━━━"
|
||||
|
||||
TOTAL_START=$(date +%s)
|
||||
SHARE_INDEX=0
|
||||
ABORT_ALL_SYNCS=false
|
||||
|
||||
# Tier 1 + Tier 2 rsync gate check
|
||||
if ! check_rsync_enabled "DAILY"; then
|
||||
warn "Rsync disabled — skipping all $SHARE_COUNT share syncs"
|
||||
warn "Proceeding to media management jobs..."
|
||||
warn "Daily rsync disabled — skipping all $SHARE_COUNT share syncs"
|
||||
warn "Proceeding to maintenance jobs..."
|
||||
elif [[ "$SHARE_COUNT" -eq 0 ]]; then
|
||||
warn "No shares configured for $MY_ID — check HOST*_DAILY_SYNC_SHARES in master_host*.conf"
|
||||
else
|
||||
# Pre-flight — connectivity then remote rootfs
|
||||
check_connectivity
|
||||
check_remote_rootfs
|
||||
|
||||
RSYNC_DRY=""
|
||||
[[ "$DRY_RUN" == true ]] && RSYNC_DRY="--dry-run"
|
||||
|
||||
for SHARE in "${ALL_SHARES[@]}"; do
|
||||
SHARE_INDEX=$((SHARE_INDEX + 1))
|
||||
SHARE_NAME=$(basename "$SHARE")
|
||||
SHARE_START=$(date +%s)
|
||||
|
||||
echo "━━━ $ICON_SYNC Share $SHARE_INDEX of $SHARE_COUNT: $SHARE_NAME ━━━"
|
||||
|
||||
if [[ "$ABORT_ALL_SYNCS" == true ]]; then
|
||||
warn "$SHARE_NAME — skipped (drive temps CRITICAL earlier in window)"
|
||||
FAIL+=("$SHARE_NAME:temp-critical")
|
||||
echo ""
|
||||
continue
|
||||
fi
|
||||
|
||||
bash "$RSYNC_SCRIPT" "$SHARE"
|
||||
RSYNC_EXIT=$?
|
||||
|
||||
SHARE_END=$(date +%s)
|
||||
SHARE_TIMES+=("$SHARE_NAME:$((SHARE_END - SHARE_START))")
|
||||
|
||||
if [[ "$RSYNC_EXIT" -eq 0 ]]; then
|
||||
PASS+=("$SHARE_NAME")
|
||||
echo "$ICON_DONE $SHARE_NAME complete"
|
||||
elif [[ "$RSYNC_EXIT" -eq 1 ]]; then
|
||||
# Temp warning — skip this profile, continue to next
|
||||
FAIL+=("$SHARE_NAME:temp-warn")
|
||||
warn "$SHARE_NAME skipped — drive temps too high"
|
||||
elif [[ "$RSYNC_EXIT" -eq 2 ]]; then
|
||||
# Temp critical — abort all remaining syncs
|
||||
FAIL+=("$SHARE_NAME:temp-critical")
|
||||
ABORT_ALL_SYNCS=true
|
||||
error "$SHARE_NAME aborted — drive temps CRITICAL, stopping all remaining syncs"
|
||||
notify "Daily sync aborted on $(hostname) — drive temps CRITICAL during $SHARE_NAME sync" "Daily Sync" "warning"
|
||||
else
|
||||
FAIL+=("$SHARE_NAME")
|
||||
error "$SHARE_NAME failed — continuing to next share"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
done
|
||||
fi # end check_rsync_enabled "DAILY"
|
||||
|
||||
TOTAL_END=$(date +%s)
|
||||
TOTAL_DURATION=$((TOTAL_END - TOTAL_START))
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_CLEAN Post-sync Media Jobs ━━━
|
||||
# Reads DAILY_MAINTENANCE_SCRIPTS from Master.conf — permissions, cleaners, arr cleanup
|
||||
# Runs after sync completes — correct ownership available, clean folders guaranteed
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_CLEAN Post-sync Media Jobs ━━━"
|
||||
|
||||
if [[ ${#DAILY_MAINTENANCE_SCRIPTS[@]} -gt 0 ]]; then
|
||||
for script_entry in "${DAILY_MAINTENANCE_SCRIPTS[@]}"; do
|
||||
[[ -z "$script_entry" ]] && continue
|
||||
script_args=($script_entry)
|
||||
script_path="$SCRIPTS_ROOT/${script_args[0]}"
|
||||
script_name=$(basename "${script_args[0]}")
|
||||
extra_args=("${script_args[@]:1}")
|
||||
(( SHARE_INDEX++ ))
|
||||
SHARE_NAME=$(basename "$SHARE")
|
||||
SHARE_START=$(date +%s)
|
||||
|
||||
echo ""
|
||||
info "$ICON_START Running: $script_name ${extra_args[*]}"
|
||||
echo "━━━ $ICON_SYNC Share $SHARE_INDEX/$SHARE_COUNT: $SHARE_NAME ━━━"
|
||||
|
||||
if [[ ! -f "$script_path" ]]; then
|
||||
error "$script_name — not found at $script_path"
|
||||
JOB_FAIL+=("$script_name ${extra_args[*]}")
|
||||
if [[ "$ABORT_ALL_SYNCS" == true ]]; then
|
||||
warn "$SHARE_NAME — skipped (drive temps CRITICAL earlier in window)"
|
||||
FAIL+=("$SHARE_NAME:temp-critical")
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
if bash "$script_path" "${extra_args[@]}" --dry-run; then
|
||||
success "$script_name — done (dry run)"
|
||||
JOB_PASS+=("$script_name ${extra_args[*]}")
|
||||
else
|
||||
error "$script_name — failed"
|
||||
JOB_FAIL+=("$script_name ${extra_args[*]}")
|
||||
fi
|
||||
else
|
||||
if bash "$script_path" "${extra_args[@]}"; then
|
||||
success "$script_name — done"
|
||||
JOB_PASS+=("$script_name ${extra_args[*]}")
|
||||
else
|
||||
error "$script_name — failed"
|
||||
JOB_FAIL+=("$script_name ${extra_args[*]}")
|
||||
fi
|
||||
fi
|
||||
bash "$RSYNC_SCRIPT" "$SHARE" $RSYNC_DRY
|
||||
RSYNC_EXIT=$?
|
||||
|
||||
SHARE_TIMES+=("$SHARE_NAME:$(( $(date +%s) - SHARE_START ))")
|
||||
|
||||
case "$RSYNC_EXIT" in
|
||||
0)
|
||||
PASS+=("$SHARE_NAME")
|
||||
log "$SHARE_NAME — done ✅"
|
||||
;;
|
||||
1)
|
||||
FAIL+=("$SHARE_NAME:temp-warn")
|
||||
warn "$SHARE_NAME skipped — drive temps too high"
|
||||
;;
|
||||
2)
|
||||
FAIL+=("$SHARE_NAME:temp-critical")
|
||||
ABORT_ALL_SYNCS=true
|
||||
error "$SHARE_NAME aborted — drive temps CRITICAL, stopping all remaining syncs"
|
||||
notify "Daily sync aborted on $(hostname) ($MY_ID) — drive temps CRITICAL during $SHARE_NAME" \
|
||||
"Daily Sync" "warning"
|
||||
;;
|
||||
*)
|
||||
FAIL+=("$SHARE_NAME")
|
||||
error "$SHARE_NAME failed (exit $RSYNC_EXIT) — continuing to next share"
|
||||
;;
|
||||
esac
|
||||
|
||||
done
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Post-sync System Jobs ━━━
|
||||
# Reads remaining DAILY_MAINTENANCE_SCRIPTS — docker restart etc.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Post-sync Jobs ━━━"
|
||||
|
||||
for script_entry in "${POST_SYNC_SCRIPTS[@]}"; do
|
||||
[[ -z "$script_entry" ]] && continue
|
||||
script_args=($script_entry)
|
||||
script_path="$SCRIPTS_ROOT/${script_args[0]}"
|
||||
script_name=$(basename "${script_args[0]}")
|
||||
extra_args=("${script_args[@]:1}")
|
||||
TOTAL_END=$(date +%s)
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Post-sync Maintenance Jobs ━━━
|
||||
# ==============================================================================================
|
||||
if [[ ${#POST_SYNC_SCRIPTS[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
info "$ICON_START Running: $script_name"
|
||||
|
||||
if [[ ! -f "$script_path" ]]; then
|
||||
error "$script_name — not found at $script_path"
|
||||
JOB_FAIL+=("$script_name")
|
||||
continue
|
||||
fi
|
||||
|
||||
if bash "$script_path" "${extra_args[@]}"; then
|
||||
success "$script_name — done"
|
||||
JOB_PASS+=("$script_name")
|
||||
else
|
||||
error "$script_name — failed"
|
||||
JOB_FAIL+=("$script_name")
|
||||
fi
|
||||
done
|
||||
echo "━━━ $ICON_CLEAN Post-sync Maintenance ━━━"
|
||||
for script_entry in "${POST_SYNC_SCRIPTS[@]}"; do
|
||||
echo ""
|
||||
run_job "$script_entry"
|
||||
done
|
||||
fi
|
||||
|
||||
WINDOW_END=$(date +%s)
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Summary ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY DAILY SYNC MAINTENANCE SUMMARY ━━━━━"
|
||||
echo "$ICON_TIME Window: $(date -d @$WINDOW_START '+%Y-%m-%d %H:%M:%S') → $(date -d @$WINDOW_END '+%H:%M:%S')"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( WINDOW_END - WINDOW_START )))"
|
||||
echo "━━━━━ $ICON_SUMMARY DAILY MAINTENANCE SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_TIME Window: $(date -d @"$WINDOW_START" '+%Y-%m-%d %H:%M:%S') → $(date -d @"$WINDOW_END" '+%H:%M:%S')"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( WINDOW_END - WINDOW_START )))"
|
||||
echo ""
|
||||
|
||||
echo "$ICON_SYNC Media shares:"
|
||||
echo "$ICON_SYNC Shares ($SHARE_COUNT):"
|
||||
for entry in "${SHARE_TIMES[@]}"; do
|
||||
SHARE_NAME="${entry%%:*}"
|
||||
DURATION="${entry##*:}"
|
||||
if printf '%s\n' "${FAIL[@]}" | grep -qx "$SHARE_NAME"; then
|
||||
echo " $ICON_ERROR $SHARE_NAME — $(format_duration $DURATION)"
|
||||
sname="${entry%%:*}"
|
||||
sdur="${entry##*:}"
|
||||
if printf '%s\n' "${FAIL[@]}" | grep -q "^${sname}"; then
|
||||
echo " $ICON_ERROR $sname — $(format_duration "$sdur")"
|
||||
else
|
||||
echo " $ICON_DONE $SHARE_NAME — $(format_duration $DURATION)"
|
||||
echo " $ICON_DONE $sname — $(format_duration "$sdur")"
|
||||
fi
|
||||
done
|
||||
echo " Passed: ${#PASS[@]}/$SHARE_COUNT Failed: ${#FAIL[@]}/$SHARE_COUNT"
|
||||
[[ "$SHARE_COUNT" -eq 0 || "${DAILY_RSYNC_ENABLED:-false}" == "false" ]] && \
|
||||
echo " (rsync disabled)"
|
||||
echo " Passed: ${#PASS[@]}/$SHARE_COUNT Failed: ${#FAIL[@]}/$SHARE_COUNT"
|
||||
echo ""
|
||||
|
||||
if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then
|
||||
echo "$ICON_GEAR Jobs (media + system):"
|
||||
for job in "${JOB_PASS[@]}"; do echo " $ICON_DONE $job"; done
|
||||
for job in "${JOB_FAIL[@]}"; do echo " $ICON_ERROR $job"; done
|
||||
echo "$ICON_GEAR Jobs:"
|
||||
for job in "${JOB_PASS[@]}"; do echo " $ICON_DONE $job"; done
|
||||
for job in "${JOB_FAIL[@]}"; do echo " $ICON_ERROR $job"; done
|
||||
echo ""
|
||||
fi
|
||||
|
||||
TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} ))
|
||||
|
||||
if [[ "$TOTAL_FAIL" -gt 0 ]]; then
|
||||
echo "$ICON_WARN Status: $TOTAL_FAIL failure(s) — check logs"
|
||||
notify "Daily sync maintenance completed with failures on $(hostname) — shares: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" "Daily Maintenance" "warning"
|
||||
warn "Status: $TOTAL_FAIL failure(s)"
|
||||
notify "Daily maintenance completed with failures on $(hostname) ($MY_ID) — shares: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \
|
||||
"Daily Maintenance" "warning"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 1
|
||||
else
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS ALL COMPLETE"
|
||||
notify "Daily sync maintenance complete on $(hostname) — ${#PASS[@]} shares synced, ${#JOB_PASS[@]} jobs run in $(format_duration $(( WINDOW_END - WINDOW_START )))" "Daily Maintenance" "normal"
|
||||
log "$ICON_DONE Status: all complete — ${#PASS[@]} share(s) synced, ${#JOB_PASS[@]} job(s) run"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,132 +1,127 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- Transcode Management ---------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Runs transcode cleanup then transcode manager in the correct order every cycle.
|
||||
# Cleanup runs first — clears stale files so manager sees accurate usage.
|
||||
# Manager runs after — threshold decisions based on real current usage post-cleanup.
|
||||
#
|
||||
# Running cleanup before manager prevents unnecessary SSD flips caused by stale
|
||||
# segment files from ended sessions inflating the ramdisk usage reading.
|
||||
#
|
||||
# Also tracks daily transcode statistics to a bounded log for weekly_health_digest.sh:
|
||||
# Peak ramdisk usage per day
|
||||
# Total flip count per day
|
||||
# Ramdisk vs SSD session ratio
|
||||
# Files cleaned per day
|
||||
#
|
||||
# Scheduled as: */3 * * * * (every 3 minutes)
|
||||
# ==============================================================================================
|
||||
# ============================= Transcode Management ===========================================
|
||||
# ==============================================================================================
|
||||
# Orchestrator — runs transcode_cleanup.sh then transcode_manager.sh in the correct order.
|
||||
# Replace individual transcode_manager and transcode_cleanup cron entries with this.
|
||||
# Schedule: */3 * * * * (every 3 minutes via User Scripts plugin)
|
||||
#
|
||||
# All configuration in Master.conf under Transcode Manager section.
|
||||
# Supports --dry-run — passes through to both child scripts.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ── WHY CLEANUP BEFORE MANAGER ────────────────────────────────────────────────────────────────
|
||||
# Cleanup runs first — removes stale segment files from ended sessions.
|
||||
# Manager runs after — threshold decisions based on real current usage post-cleanup.
|
||||
# Without this order, stale files inflate the ramdisk usage reading and trigger
|
||||
# unnecessary SSD flips even when active sessions would fit on the ramdisk.
|
||||
#
|
||||
# ── WHAT EACH SCRIPT DOES ─────────────────────────────────────────────────────────────────────
|
||||
# transcode_cleanup.sh — removes aged segment files not open by any process
|
||||
# uses lsof for O(1) per-file active check (never per-file lsof)
|
||||
# also triggers flip-back to ramdisk after cleanup if recovered ✅
|
||||
#
|
||||
# transcode_manager.sh — checks ramdisk usage against thresholds
|
||||
# flips symlink between ramdisk and SSD as needed
|
||||
# writes one entry to TRANSCODE_DAILY_LOG after each run
|
||||
# shows active Emby sessions with play method
|
||||
#
|
||||
# ── DAILY LOG ─────────────────────────────────────────────────────────────────────────────────
|
||||
# transcode_manager.sh writes to TRANSCODE_DAILY_LOG after each run:
|
||||
# Format: DATE|RAMDISK_USED_GB|FLIP_COUNT|RAM_SESSIONS|SSD_SESSIONS
|
||||
# This orchestrator does NOT write its own log — manager handles it ✅
|
||||
# Log trimmed to TRANSCODE_LOG_RETENTION days by manager on each write.
|
||||
# Read by sunday_morning_coffee_report.sh and weekly_health_digest.sh.
|
||||
#
|
||||
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
|
||||
# detect_hosts() sets MY_ID and aliases RAMDISK_PATH, TRANSCODE_SSD, RAMDISK_WARN_GB etc.
|
||||
# Each server manages its own transcode location independently.
|
||||
#
|
||||
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
|
||||
# Root check — mount and docker operations require root
|
||||
# acquire_lock — prevents concurrent 3-minute cycles overlapping
|
||||
# detect_hosts() — correct paths per host
|
||||
# --dry-run — passed through to both child scripts
|
||||
# Exit code — worst exit code of both scripts returned
|
||||
#
|
||||
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||||
# TRANSCODE_DAILY_LOG — daily stats log (written by transcode_manager.sh)
|
||||
# TRANSCODE_LOG_RETENTION — days to keep (trimmed by manager)
|
||||
# TRANSCODE_STATE_FILE — current state (ramdisk_used, flip_count etc.)
|
||||
# All TRANSCODE_* threshold vars — see master.conf Transcode Manager section
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# transcode_management.sh — normal run (every 3 minutes via cron)
|
||||
# transcode_management.sh --dry-run — preview without changes (passed to children)
|
||||
# transcode_management.sh --status — show configuration and current state
|
||||
# transcode_management.sh --log — verbose output from both child scripts
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# State and log files
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
TRANSCODE_DAILY_LOG="/boot/config/transcode_daily.db"
|
||||
TRANSCODE_STATE_FILE="/tmp/transcode_state.db"
|
||||
CLEANUP_SCRIPT="$SCRIPT_DIR/../Transcodes/transcode_cleanup.sh"
|
||||
MANAGER_SCRIPT="$SCRIPT_DIR/../Transcodes/transcode_manager.sh"
|
||||
|
||||
# Bounded log — keeps last 90 days
|
||||
TRANSCODE_LOG_RETENTION=90
|
||||
|
||||
touch "$TRANSCODE_DAILY_LOG" 2>/dev/null
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — passing to child scripts"
|
||||
acquire_lock "wait"
|
||||
|
||||
acquire_lock
|
||||
# detect_hosts() sets MY_ID and aliases all HOST*_TRANSCODE_* vars
|
||||
detect_hosts
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# HELPERS
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — passing through to child scripts"
|
||||
|
||||
# Read a value from transcode state file
|
||||
state_get() {
|
||||
grep "^${1}=" "$TRANSCODE_STATE_FILE" 2>/dev/null | cut -d= -f2
|
||||
}
|
||||
|
||||
# Get today's date key
|
||||
today() {
|
||||
date '+%Y-%m-%d'
|
||||
}
|
||||
|
||||
# Update daily log entry for today
|
||||
# Format: YYYY-MM-DD|peak_gb|flip_count|ram_sessions|ssd_sessions|files_cleaned
|
||||
update_daily_log() {
|
||||
local peak_gb="$1"
|
||||
local flips="$2"
|
||||
local ram_sessions="$3"
|
||||
local ssd_sessions="$4"
|
||||
local files_cleaned="$5"
|
||||
local today_key
|
||||
today_key=$(today)
|
||||
|
||||
local existing
|
||||
existing=$(grep "^${today_key}|" "$TRANSCODE_DAILY_LOG" 2>/dev/null)
|
||||
|
||||
if [[ -z "$existing" ]]; then
|
||||
# New entry for today
|
||||
echo "${today_key}|${peak_gb}|${flips}|${ram_sessions}|${ssd_sessions}|${files_cleaned}" \
|
||||
>> "$TRANSCODE_DAILY_LOG"
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY TRANSCODE MANAGEMENT STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_RAM Ramdisk: $RAMDISK_PATH ($RAMDISK_SIZE)"
|
||||
echo "$ICON_DISK SSD fallback: $TRANSCODE_SSD"
|
||||
echo "$ICON_LINK Symlink: $TRANSCODE_LINK"
|
||||
echo "$ICON_TIME Schedule: every 3 minutes"
|
||||
echo ""
|
||||
echo "━━━ Child Scripts ━━━"
|
||||
[[ -f "$CLEANUP_SCRIPT" ]] && \
|
||||
echo " $ICON_SUCCESS transcode_cleanup.sh — found" || \
|
||||
echo " $ICON_ERROR transcode_cleanup.sh — NOT FOUND at $CLEANUP_SCRIPT"
|
||||
[[ -f "$MANAGER_SCRIPT" ]] && \
|
||||
echo " $ICON_SUCCESS transcode_manager.sh — found" || \
|
||||
echo " $ICON_ERROR transcode_manager.sh — NOT FOUND at $MANAGER_SCRIPT"
|
||||
echo ""
|
||||
echo "━━━ Daily Log ━━━"
|
||||
if [[ -f "${TRANSCODE_DAILY_LOG:-}" ]] && [[ -s "$TRANSCODE_DAILY_LOG" ]]; then
|
||||
ENTRY_COUNT=$(wc -l < "$TRANSCODE_DAILY_LOG")
|
||||
OLDEST=$(awk -F'|' 'NR==1{print $1}' "$TRANSCODE_DAILY_LOG")
|
||||
NEWEST=$(awk -F'|' 'END{print $1}' "$TRANSCODE_DAILY_LOG")
|
||||
echo " $ICON_SUCCESS $TRANSCODE_DAILY_LOG ($ENTRY_COUNT entries, $OLDEST → $NEWEST)"
|
||||
else
|
||||
# Update existing — keep highest peak, accumulate flips, sessions, files
|
||||
local old_peak old_flips old_ram old_ssd old_files
|
||||
old_peak=$(echo "$existing" | cut -d'|' -f2)
|
||||
old_flips=$(echo "$existing" | cut -d'|' -f3)
|
||||
old_ram=$(echo "$existing" | cut -d'|' -f4)
|
||||
old_ssd=$(echo "$existing" | cut -d'|' -f5)
|
||||
old_files=$(echo "$existing" | cut -d'|' -f6)
|
||||
|
||||
# Peak — keep highest
|
||||
local new_peak
|
||||
new_peak=$(awk "BEGIN {print ($peak_gb > $old_peak) ? $peak_gb : $old_peak}")
|
||||
|
||||
# Accumulate
|
||||
local new_flips=$(( old_flips + flips ))
|
||||
local new_ram=$(( old_ram + ram_sessions ))
|
||||
local new_ssd=$(( old_ssd + ssd_sessions ))
|
||||
local new_files=$(( old_files + files_cleaned ))
|
||||
|
||||
# Replace line
|
||||
sed -i "s|^${today_key}|.*|${today_key}|${new_peak}|${new_flips}|${new_ram}|${new_ssd}|${new_files}|" \
|
||||
"$TRANSCODE_DAILY_LOG" 2>/dev/null || {
|
||||
# sed replacement failed — remove and re-add
|
||||
sed -i "/^${today_key}|/d" "$TRANSCODE_DAILY_LOG"
|
||||
echo "${today_key}|${new_peak}|${new_flips}|${new_ram}|${new_ssd}|${new_files}" \
|
||||
>> "$TRANSCODE_DAILY_LOG"
|
||||
}
|
||||
echo " $ICON_SKIP $TRANSCODE_DAILY_LOG — no data yet"
|
||||
fi
|
||||
echo ""
|
||||
echo "━━━ Current State ━━━"
|
||||
if [[ -f "${TRANSCODE_STATE_FILE:-/tmp/transcode_state.db}" ]]; then
|
||||
while IFS='=' read -r key val; do
|
||||
[[ -n "$key" ]] && echo " $key = $val"
|
||||
done < "${TRANSCODE_STATE_FILE:-/tmp/transcode_state.db}"
|
||||
else
|
||||
echo " State file not found (ramdisk_setup.sh creates it at array start)"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Purge old entries beyond retention
|
||||
local cutoff
|
||||
cutoff=$(date -d "${TRANSCODE_LOG_RETENTION} days ago" '+%Y-%m-%d')
|
||||
awk -F'|' -v cutoff="$cutoff" '$1 >= cutoff' \
|
||||
"$TRANSCODE_DAILY_LOG" > "${TRANSCODE_DAILY_LOG}.tmp" && \
|
||||
mv "${TRANSCODE_DAILY_LOG}.tmp" "$TRANSCODE_DAILY_LOG"
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SYNC Run Cleanup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
CLEANUP_SCRIPT="$SCRIPT_DIR/../Transcodes/transcode_cleanup.sh"
|
||||
MANAGER_SCRIPT="$SCRIPT_DIR/../Transcodes/transcode_manager.sh"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Validate Child Scripts ━━━
|
||||
# ==============================================================================================
|
||||
if [[ ! -f "$CLEANUP_SCRIPT" ]]; then
|
||||
error "transcode_cleanup.sh not found: $CLEANUP_SCRIPT"
|
||||
exit 1
|
||||
@@ -137,50 +132,26 @@ if [[ ! -f "$MANAGER_SCRIPT" ]]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Capture cleanup output for file count
|
||||
CLEANUP_OUTPUT=$(bash "$CLEANUP_SCRIPT" $([[ "$DRY_RUN" == true ]] && echo "--dry-run") 2>&1)
|
||||
# ==============================================================================================
|
||||
# ━━━ Run Cleanup ━━━
|
||||
# ==============================================================================================
|
||||
DRY_FLAG=""
|
||||
[[ "$DRY_RUN" == true ]] && DRY_FLAG="--dry-run"
|
||||
|
||||
bash "$CLEANUP_SCRIPT" $DRY_FLAG
|
||||
CLEANUP_EXIT=$?
|
||||
echo "$CLEANUP_OUTPUT"
|
||||
|
||||
# Extract files cleaned from cleanup output
|
||||
FILES_CLEANED=$(echo "$CLEANUP_OUTPUT" | grep -oE "Removed [0-9]+ file" | grep -oE "[0-9]+" | head -1)
|
||||
FILES_CLEANED="${FILES_CLEANED:-0}"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_WATCHDOG Run Manager ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
MANAGER_OUTPUT=$(bash "$MANAGER_SCRIPT" $([[ "$DRY_RUN" == true ]] && echo "--dry-run") 2>&1)
|
||||
# ==============================================================================================
|
||||
# ━━━ Run Manager ━━━
|
||||
# ==============================================================================================
|
||||
# transcode_manager.sh writes to TRANSCODE_DAILY_LOG after each run
|
||||
# No --no-log flag here — manager owns the log write for this cycle ✅
|
||||
bash "$MANAGER_SCRIPT" $DRY_FLAG
|
||||
MANAGER_EXIT=$?
|
||||
echo "$MANAGER_OUTPUT"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Collect stats for daily log
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Ramdisk usage from state file
|
||||
RAMDISK_USED_GB=$(state_get "ramdisk_used_gb" 2>/dev/null || echo "0")
|
||||
[[ -z "$RAMDISK_USED_GB" || "$RAMDISK_USED_GB" == "0" ]] && \
|
||||
RAMDISK_USED_GB=$(df "$RAMDISK_PATH" --output=used 2>/dev/null | tail -1 | \
|
||||
awk '{printf "%.2f", $1/1048576}' || echo "0")
|
||||
|
||||
# Flip count from state file
|
||||
FLIP_COUNT=$(state_get "flip_count_hour" 2>/dev/null || echo "0")
|
||||
FLIP_COUNT="${FLIP_COUNT:-0}"
|
||||
|
||||
# Session counts from manager output
|
||||
RAM_SESSIONS=$(echo "$MANAGER_OUTPUT" | grep -oE "ramdisk \([0-9]+\)" | grep -oE "[0-9]+" | head -1)
|
||||
SSD_SESSIONS=$(echo "$MANAGER_OUTPUT" | grep -oE "SSD \([0-9]+\)" | grep -oE "[0-9]+" | head -1)
|
||||
RAM_SESSIONS="${RAM_SESSIONS:-0}"
|
||||
SSD_SESSIONS="${SSD_SESSIONS:-0}"
|
||||
|
||||
# Update daily log
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
update_daily_log "$RAMDISK_USED_GB" "$FLIP_COUNT" "$RAM_SESSIONS" "$SSD_SESSIONS" "$FILES_CLEANED"
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Exit with worst exit code
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
if [[ "$CLEANUP_EXIT" -ne 0 || "$MANAGER_EXIT" -ne 0 ]]; then
|
||||
exit 1
|
||||
fi
|
||||
# ==============================================================================================
|
||||
# ━━━ Exit ━━━
|
||||
# ==============================================================================================
|
||||
# Return worst exit code — caller knows if either script failed
|
||||
[[ "$CLEANUP_EXIT" -ne 0 || "$MANAGER_EXIT" -ne 0 ]] && exit 1
|
||||
exit 0
|
||||
@@ -1,99 +1,193 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ----------------------------- Weekly Sync Maintenance ----------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ============================= Weekly Sync Maintenance ========================================
|
||||
# ==============================================================================================
|
||||
# Weekly maintenance window orchestrator — clean sync, container updates, weekly restarts.
|
||||
# Schedule: 30 2 * * 0 (Sunday 2:30am — fits before 3am network reboot)
|
||||
# Schedule: 30 2 * * 0 (Sunday 2:30am — before Sunday 7am coffee report)
|
||||
#
|
||||
# Execution order:
|
||||
# 1. Stop local containers — Emby + auth stack stopped locally
|
||||
# 2. Stop remote containers — Emby + auth stack stopped remotely via SSH
|
||||
# 3. Pull updates locally — if WEEKLY_SYNC_UPDATES=true
|
||||
# 4. Pull updates remotely — if WEEKLY_SYNC_UPDATES_REMOTE=true
|
||||
# 5. rsync Emby — full clean mirror, both instances stopped
|
||||
# 6. rsync Critical-Data — auth stack clean sync, databases flushed
|
||||
# 7. Start remote containers — correct order, delayed start respected
|
||||
# 8. Start local containers — correct order, delayed start respected
|
||||
# 9. docker_weekly_restart.sh — weekly container restarts
|
||||
# ── EXECUTION ORDER ───────────────────────────────────────────────────────────────────────────
|
||||
# 1. Stop local containers — Emby + auth stack stopped locally
|
||||
# 2. Stop remote containers — Emby + auth stack stopped remotely via SSH
|
||||
# 3. Pull updates locally — if WEEKLY_SYNC_UPDATES=true (zero extra downtime)
|
||||
# 4. Pull updates remotely — if WEEKLY_SYNC_UPDATES_REMOTE=true
|
||||
# 5. rsync WEEKLY_SYNC_SHARES — full clean mirror, containers stopped both sides
|
||||
# 6. Start remote containers — correct order, delayed start respected
|
||||
# 7. Start local containers — correct order, delayed start respected
|
||||
# 8. WEEKLY_MAINTENANCE_SCRIPTS — weekly restarts etc. (docker_weekly_restart.sh)
|
||||
#
|
||||
# Synced shares (WEEKLY_SYNC_SHARES in Master.conf):
|
||||
# /mnt/user/Media_Server/Emby — emby profile — full mirror, cache resets weekly
|
||||
# /mnt/user/appdata-Failover/Critical-Data — critical-data — auth stack clean state
|
||||
# ── WHY WEEKLY NOT NIGHTLY FOR EMBY ──────────────────────────────────────────────────────────
|
||||
# Emby builds a warm image cache on HOST2 throughout the week.
|
||||
# Syncing nightly resets cache — cold loads every morning for users.
|
||||
# Weekly sync: cache stays warm 6 days, resets Sunday night while users sleep.
|
||||
# emby-failover dirty sync covers watch states + library every 15min between weekly syncs.
|
||||
#
|
||||
# Why weekly instead of nightly for Emby:
|
||||
# Emby builds a warm image cache on HOST2 throughout the week
|
||||
# Syncing nightly resets cache — cold loads every morning for users
|
||||
# Weekly sync: cache stays warm 6 days, resets Sunday night while users sleep
|
||||
# emby-failover dirty sync covers watch states + library every 30-60min between syncs
|
||||
# ── CONTAINER UPDATES ─────────────────────────────────────────────────────────────────────────
|
||||
# Containers already stopped for sync — updates pull at zero extra downtime.
|
||||
# Both servers start on identical image versions after the window completes.
|
||||
# Toggle: WEEKLY_SYNC_UPDATES / WEEKLY_SYNC_UPDATES_REMOTE in master.conf
|
||||
#
|
||||
# Container updates during the window:
|
||||
# Containers already stopped for sync — updates pull at zero extra downtime
|
||||
# Both servers start on identical image versions after the window completes
|
||||
# Toggle: WEEKLY_SYNC_UPDATES / WEEKLY_SYNC_UPDATES_REMOTE in Master.conf
|
||||
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
|
||||
# detect_hosts() sets MY_ID — used in banner, summary, and notifications.
|
||||
# WEEKLY_SYNC_SHARES and WEEKLY_MAINTENANCE_SCRIPTS configured in master.conf.
|
||||
# Same script runs correctly on both servers.
|
||||
#
|
||||
# What triggers weekly_health_digest.sh:
|
||||
# NOT this script — weekly_health_digest.sh runs on its own Saturday schedule
|
||||
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
|
||||
# Root check — stop/start containers, rsync require root
|
||||
# acquire_lock — prevents concurrent weekly windows
|
||||
# check_connectivity — verifies remote before any remote operations
|
||||
# check_remote_rootfs — aborts if remote rootfs nearly full
|
||||
# DOCKER_TIMEOUT — all docker calls protected
|
||||
# SSH_TIMEOUT — all SSH calls protected
|
||||
# validate_unraid_cmd — notify validated before use
|
||||
# Silent on success — runs weekly, only failures warrant notification
|
||||
#
|
||||
# Configuration in Master.conf:
|
||||
# WEEKLY_SYNC_SHARES — shares synced during the maintenance window
|
||||
# WEEKLY_MAINTENANCE_SCRIPTS — scripts run after sync (docker_weekly_restart)
|
||||
# WEEKLY_SYNC_UPDATES — toggle container updates on/off
|
||||
# WEEKLY_SYNC_UPDATES_REMOTE — toggle remote container updates on/off
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# All configuration in Master.conf.
|
||||
# Supports --dry-run to walk through without stopping containers, syncing, or updating.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||||
# WEEKLY_SYNC_SHARES — shares synced during window
|
||||
# WEEKLY_MAINTENANCE_SCRIPTS — scripts run after sync
|
||||
# WEEKLY_SYNC_UPDATES — toggle local container updates
|
||||
# WEEKLY_SYNC_UPDATES_REMOTE — toggle remote container updates
|
||||
# WEEKLY_RSYNC_ENABLED — enable/disable rsync section
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# weekly_sync_maintenance.sh — normal run
|
||||
# weekly_sync_maintenance.sh --dry-run — preview without stopping containers or syncing
|
||||
# weekly_sync_maintenance.sh --log — verbose per-share/per-job output
|
||||
# weekly_sync_maintenance.sh --status — show configuration and exit
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
RSYNC_SCRIPT="$SCRIPT_DIR/../Rsync/rsync.sh"
|
||||
SCRIPTS_ROOT="$SCRIPT_DIR/.."
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
DOCKER_TIMEOUT=30 # container stop/start needs longer than normal
|
||||
SSH_TIMEOUT=30 # remote pulls can be slow
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Running as root"
|
||||
validate_unraid_cmd \
|
||||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||||
"" "" \
|
||||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||
|
||||
acquire_lock
|
||||
|
||||
detect_hosts
|
||||
resolve_remote_ip
|
||||
|
||||
# Load container list from emby profile — used for update pulls
|
||||
read -r -a MAINTENANCE_CONTAINERS <<< "${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-}"
|
||||
WINDOW_START=$(date +%s)
|
||||
PASS=()
|
||||
FAIL=()
|
||||
JOB_PASS=()
|
||||
JOB_FAIL=()
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SHIELD Pre-flight Checks ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Load container lists from profile config
|
||||
read -r -a MAINTENANCE_CONTAINERS <<< \
|
||||
"${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-}"
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be stopped, no sync, no updates"
|
||||
|
||||
# ── Helper — run a post-sync maintenance script ────────────────────────────────────────────────
|
||||
run_job() {
|
||||
local script_entry="$1"
|
||||
local extra_dry=""
|
||||
[[ "$DRY_RUN" == true ]] && extra_dry="--dry-run"
|
||||
|
||||
read -r -a script_args <<< "$script_entry"
|
||||
local script_path="$SCRIPTS_ROOT/${script_args[0]}"
|
||||
local script_name
|
||||
script_name=$(basename "${script_args[0]}")
|
||||
local extra_args=("${script_args[@]:1}")
|
||||
|
||||
if [[ ! -f "$script_path" ]]; then
|
||||
error "$script_name — not found at $script_path"
|
||||
JOB_FAIL+=("$script_name")
|
||||
return 1
|
||||
fi
|
||||
|
||||
log "Running: $script_name ${extra_args[*]}"
|
||||
# shellcheck disable=SC2086
|
||||
if bash "$script_path" "${extra_args[@]}" $extra_dry; then
|
||||
log "$script_name — done ✅"
|
||||
JOB_PASS+=("$script_name ${extra_args[*]}")
|
||||
else
|
||||
error "$script_name — failed (exit $?)"
|
||||
JOB_FAIL+=("$script_name ${extra_args[*]}")
|
||||
fi
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY WEEKLY SYNC STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_NET Remote: $REMOTE_ID ($REMOTE_SERVER_NAME)"
|
||||
echo "$ICON_SYNC Rsync enabled: ${RSYNC_ENABLED:-true}"
|
||||
echo "$ICON_SYNC Weekly enabled: ${WEEKLY_RSYNC_ENABLED:-false}"
|
||||
echo "$ICON_GEAR Local updates: ${WEEKLY_SYNC_UPDATES:-false}"
|
||||
echo "$ICON_GEAR Remote updates: ${WEEKLY_SYNC_UPDATES_REMOTE:-false}"
|
||||
echo ""
|
||||
echo "━━━ Weekly Sync Shares ━━━"
|
||||
if [[ ${#WEEKLY_SYNC_SHARES[@]} -eq 0 ]]; then
|
||||
warn " No WEEKLY_SYNC_SHARES configured"
|
||||
else
|
||||
for share in "${WEEKLY_SYNC_SHARES[@]:-}"; do
|
||||
[[ -n "$share" ]] && echo " $ICON_SYNC $(basename "$share") ($share)"
|
||||
done
|
||||
fi
|
||||
echo ""
|
||||
echo "━━━ Weekly Maintenance Scripts ━━━"
|
||||
if [[ ${#WEEKLY_MAINTENANCE_SCRIPTS[@]} -eq 0 ]]; then
|
||||
echo " None configured"
|
||||
else
|
||||
for entry in "${WEEKLY_MAINTENANCE_SCRIPTS[@]:-}"; do
|
||||
[[ -n "$entry" ]] && echo " $ICON_GEAR $(basename "${entry%% *}") ${entry#* }"
|
||||
done
|
||||
fi
|
||||
echo ""
|
||||
echo "━━━ Containers (from profile config) ━━━"
|
||||
for c in "${MAINTENANCE_CONTAINERS[@]:-}"; do
|
||||
[[ -n "$c" ]] && echo " $ICON_CONTAINERS $c"
|
||||
done
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Pre-flight Checks ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_SHIELD Pre-flight Checks ━━━"
|
||||
echo "━━━ $ICON_GEAR Weekly Sync Maintenance — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo ""
|
||||
echo "━━━ $ICON_SHIELD Pre-flight ━━━"
|
||||
|
||||
check_connectivity
|
||||
check_remote_rootfs
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_STOP $ICON_CONTAINERS Stop Containers ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Stop Containers ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_STOP $ICON_CONTAINERS Stop Containers ━━━"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — containers will not be stopped"
|
||||
else
|
||||
# Load critical-data + emby container list for stops
|
||||
read -r -a CRITICAL_CONTAINER_NAMES <<< "${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-}"
|
||||
# Load container lists for stop functions
|
||||
read -r -a CRITICAL_CONTAINER_NAMES <<< \
|
||||
"${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-}"
|
||||
read -r -a DELAYED_CONTAINERS <<< "${PROFILE_DELAYED_CONTAINERS[critical-data]:-}"
|
||||
CONTAINER_DELAY="${PROFILE_CONTAINER_DELAY[critical-data]:-15}"
|
||||
|
||||
@@ -101,123 +195,118 @@ else
|
||||
stop_containers
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Container Updates ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Container Updates ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Container Updates ━━━"
|
||||
|
||||
if [[ "$WEEKLY_SYNC_UPDATES" == true ]]; then
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would pull updates for local containers"
|
||||
for c in "${MAINTENANCE_CONTAINERS[@]}"; do
|
||||
[[ -z "$c" ]] && continue
|
||||
warn "DRY RUN — would docker pull: $c"
|
||||
for c in "${MAINTENANCE_CONTAINERS[@]:-}"; do
|
||||
[[ -n "$c" ]] && warn "DRY RUN — would pull: $c"
|
||||
done
|
||||
else
|
||||
info "Pulling local container updates..."
|
||||
for c in "${MAINTENANCE_CONTAINERS[@]}"; do
|
||||
log "Pulling local container updates..."
|
||||
for c in "${MAINTENANCE_CONTAINERS[@]:-}"; do
|
||||
[[ -z "$c" ]] && continue
|
||||
# Get image name from running or stopped container
|
||||
IMAGE=$(docker inspect "$c" --format '{{.Config.Image}}' 2>/dev/null)
|
||||
IMAGE=$(timeout "$DOCKER_TIMEOUT" docker inspect \
|
||||
"$c" --format '{{.Config.Image}}' 2>/dev/null)
|
||||
if [[ -z "$IMAGE" ]]; then
|
||||
log "$c — not found locally, skipping update"
|
||||
continue
|
||||
fi
|
||||
info "Pulling $IMAGE for $c..."
|
||||
log "Pulling $IMAGE for $c..."
|
||||
if docker pull "$IMAGE" >/dev/null 2>&1; then
|
||||
success "$c — image updated"
|
||||
log "$c — image updated ✅"
|
||||
else
|
||||
warn "$c — pull failed, will start on existing image"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
else
|
||||
info "WEEKLY_SYNC_UPDATES=false — skipping local updates"
|
||||
log "WEEKLY_SYNC_UPDATES=false — skipping local updates"
|
||||
fi
|
||||
|
||||
if [[ "$WEEKLY_SYNC_UPDATES_REMOTE" == true ]]; then
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would pull updates on $REMOTE_SERVER_NAME"
|
||||
else
|
||||
info "Pulling remote container updates on $REMOTE_SERVER_NAME..."
|
||||
for c in "${MAINTENANCE_CONTAINERS[@]}"; do
|
||||
log "Pulling remote container updates on $REMOTE_SERVER_NAME..."
|
||||
for c in "${MAINTENANCE_CONTAINERS[@]:-}"; do
|
||||
[[ -z "$c" ]] && continue
|
||||
IMAGE=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||||
IMAGE=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" \
|
||||
root@"$REMOTE_SERVER" \
|
||||
"docker inspect $c --format '{{.Config.Image}}' 2>/dev/null" 2>/dev/null)
|
||||
if [[ -z "$IMAGE" ]]; then
|
||||
log "$c — not found on remote, skipping update"
|
||||
continue
|
||||
fi
|
||||
info "Pulling $IMAGE for $c on $REMOTE_SERVER_NAME..."
|
||||
if ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" "docker pull $IMAGE" >/dev/null 2>&1; then
|
||||
success "$c — remote image updated"
|
||||
log "Pulling $IMAGE for $c on $REMOTE_SERVER_NAME..."
|
||||
if timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" \
|
||||
root@"$REMOTE_SERVER" \
|
||||
"docker pull $IMAGE" >/dev/null 2>&1; then
|
||||
log "$c — remote image updated ✅"
|
||||
else
|
||||
warn "$c — remote pull failed, will start on existing image"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
else
|
||||
info "WEEKLY_SYNC_UPDATES_REMOTE=false — skipping remote updates"
|
||||
log "WEEKLY_SYNC_UPDATES_REMOTE=false — skipping remote updates"
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SYNC Critical Shares Sync ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
PASS=()
|
||||
FAIL=()
|
||||
TOTAL_START=$(date +%s)
|
||||
|
||||
SYNC_JOBS=("${WEEKLY_SYNC_SHARES[@]}")
|
||||
SHARE_COUNT=${#SYNC_JOBS[@]}
|
||||
# ==============================================================================================
|
||||
# ━━━ Critical Shares Sync ━━━
|
||||
# ==============================================================================================
|
||||
SHARE_COUNT=${#WEEKLY_SYNC_SHARES[@]}
|
||||
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Critical Shares Sync — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo "$ICON_SUMMARY Jobs: $SHARE_COUNT"
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Critical Shares Sync — $SHARE_COUNT share(s) ━━━"
|
||||
|
||||
SYNC_START=$(date +%s)
|
||||
JOB_NUM=0
|
||||
|
||||
# Tier 1 + Tier 2 rsync gate check
|
||||
if ! check_rsync_enabled "WEEKLY"; then
|
||||
warn "Rsync disabled — skipping all $SHARE_COUNT weekly sync jobs"
|
||||
warn "Proceeding to container updates and maintenance scripts..."
|
||||
warn "Weekly rsync disabled — skipping all $SHARE_COUNT sync job(s)"
|
||||
warn "Proceeding to container start and maintenance scripts..."
|
||||
elif [[ "$SHARE_COUNT" -eq 0 ]]; then
|
||||
warn "No WEEKLY_SYNC_SHARES configured — skipping sync"
|
||||
warn "Check WEEKLY_SYNC_SHARES in master.conf"
|
||||
else
|
||||
for JOB in "${SYNC_JOBS[@]}"; do
|
||||
((JOB_NUM++))
|
||||
RSYNC_DRY=""
|
||||
[[ "$DRY_RUN" == true ]] && RSYNC_DRY="--dry-run"
|
||||
|
||||
for JOB in "${WEEKLY_SYNC_SHARES[@]}"; do
|
||||
(( JOB_NUM++ ))
|
||||
JOB_NAME=$(basename "$JOB")
|
||||
|
||||
echo ""
|
||||
echo "━━━ [$JOB_NUM/$SHARE_COUNT] $JOB_NAME ━━━"
|
||||
|
||||
JOB_START=$(date +%s)
|
||||
JOB_START=$(date +%s)
|
||||
bash "$RSYNC_SCRIPT" "$JOB" $RSYNC_DRY
|
||||
EXIT_CODE=$?
|
||||
JOB_DUR=$(format_duration $(( $(date +%s) - JOB_START )))
|
||||
|
||||
# Containers already stopped — rsync profile won't try to stop them again
|
||||
# Pass --no-container-stop flag would be ideal but profiles handle this naturally
|
||||
# since containers are already stopped, stop_containers finds nothing running
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
bash "$RSYNC_SCRIPT" "$JOB" --dry-run
|
||||
else
|
||||
bash "$RSYNC_SCRIPT" "$JOB"
|
||||
fi
|
||||
if [[ "$EXIT_CODE" -eq 0 ]]; then
|
||||
PASS+=("$JOB_NAME")
|
||||
log "$JOB_NAME — done in $JOB_DUR ✅"
|
||||
else
|
||||
FAIL+=("$JOB_NAME")
|
||||
error "$JOB_NAME — failed after $JOB_DUR (exit $EXIT_CODE)"
|
||||
fi
|
||||
echo ""
|
||||
done
|
||||
fi
|
||||
|
||||
EXIT_CODE=$?
|
||||
JOB_END=$(date +%s)
|
||||
JOB_DURATION=$(format_duration $(( JOB_END - JOB_START )))
|
||||
SYNC_END=$(date +%s)
|
||||
|
||||
if [[ "$EXIT_CODE" -eq 0 ]]; then
|
||||
PASS+=("$JOB_NAME")
|
||||
success "$JOB_NAME — $ICON_SUCCESS done in $JOB_DURATION"
|
||||
else
|
||||
FAIL+=("$JOB_NAME")
|
||||
error "$JOB_NAME — $ICON_ERROR failed after $JOB_DURATION"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
done
|
||||
fi # end check_rsync_enabled "WEEKLY"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_START $ICON_CONTAINERS Start Containers ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Start Containers ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_START $ICON_CONTAINERS Start Containers ━━━"
|
||||
|
||||
@@ -228,88 +317,57 @@ else
|
||||
start_local_containers
|
||||
fi
|
||||
|
||||
TOTAL_END=$(date +%s)
|
||||
TOTAL_DURATION=$(format_duration $(( TOTAL_END - TOTAL_START )))
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Post-sync Jobs ━━━
|
||||
# docker_weekly_restart.sh and any other WEEKLY_MAINTENANCE_SCRIPTS run after sync
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
JOB_PASS=()
|
||||
JOB_FAIL=()
|
||||
SCRIPTS_ROOT="$SCRIPT_DIR/.."
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Post-sync Jobs ━━━
|
||||
# ==============================================================================================
|
||||
if [[ ${#WEEKLY_MAINTENANCE_SCRIPTS[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Post-sync Jobs ━━━"
|
||||
|
||||
for script_entry in "${WEEKLY_MAINTENANCE_SCRIPTS[@]}"; do
|
||||
for script_entry in "${WEEKLY_MAINTENANCE_SCRIPTS[@]:-}"; do
|
||||
[[ -z "$script_entry" ]] && continue
|
||||
script_args=($script_entry)
|
||||
script_path="$SCRIPTS_ROOT/${script_args[0]}"
|
||||
script_name=$(basename "${script_args[0]}")
|
||||
extra_args=("${script_args[@]:1}")
|
||||
|
||||
echo ""
|
||||
info "$ICON_START Running: $script_name"
|
||||
|
||||
if [[ ! -f "$script_path" ]]; then
|
||||
error "$script_name — not found at $script_path"
|
||||
JOB_FAIL+=("$script_name")
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would run: $script_name"
|
||||
JOB_PASS+=("$script_name (dry run)")
|
||||
elif bash "$script_path" "${extra_args[@]}"; then
|
||||
success "$script_name — done"
|
||||
JOB_PASS+=("$script_name")
|
||||
else
|
||||
error "$script_name — failed"
|
||||
JOB_FAIL+=("$script_name")
|
||||
fi
|
||||
run_job "$script_entry"
|
||||
done
|
||||
fi
|
||||
|
||||
WINDOW_END=$(date +%s)
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Summary ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY WEEKLY SYNC MAINTENANCE SUMMARY ━━━━━"
|
||||
echo "$ICON_TIME Duration: $TOTAL_DURATION"
|
||||
echo "$ICON_GEAR Updates: local=$WEEKLY_SYNC_UPDATES remote=$WEEKLY_SYNC_UPDATES_REMOTE"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_TIME Window: $(date -d @"$WINDOW_START" '+%Y-%m-%d %H:%M:%S') → $(date -d @"$WINDOW_END" '+%H:%M:%S')"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( WINDOW_END - WINDOW_START )))"
|
||||
echo "$ICON_GEAR Updates: local=${WEEKLY_SYNC_UPDATES:-false} remote=${WEEKLY_SYNC_UPDATES_REMOTE:-false}"
|
||||
echo ""
|
||||
|
||||
echo "$ICON_SYNC Sync jobs:"
|
||||
if [[ ${#PASS[@]} -gt 0 ]]; then
|
||||
for job in "${PASS[@]}"; do echo " $ICON_SUCCESS $job"; done
|
||||
fi
|
||||
if [[ ${#FAIL[@]} -gt 0 ]]; then
|
||||
for job in "${FAIL[@]}"; do echo " $ICON_ERROR $job"; done
|
||||
fi
|
||||
echo " Passed: ${#PASS[@]} Failed: ${#FAIL[@]}"
|
||||
echo "$ICON_SYNC Sync jobs ($SHARE_COUNT):"
|
||||
for job in "${PASS[@]:-}"; do echo " $ICON_DONE $job"; done
|
||||
for job in "${FAIL[@]:-}"; do echo " $ICON_ERROR $job"; done
|
||||
echo " Passed: ${#PASS[@]} Failed: ${#FAIL[@]}"
|
||||
|
||||
if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
echo "$ICON_GEAR Post-sync jobs:"
|
||||
for job in "${JOB_PASS[@]}"; do echo " $ICON_SUCCESS $job"; done
|
||||
for job in "${JOB_FAIL[@]}"; do echo " $ICON_ERROR $job"; done
|
||||
for job in "${JOB_PASS[@]:-}"; do echo " $ICON_DONE $job"; done
|
||||
for job in "${JOB_FAIL[@]:-}"; do echo " $ICON_ERROR $job"; done
|
||||
fi
|
||||
|
||||
echo ""
|
||||
|
||||
TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} ))
|
||||
|
||||
echo ""
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "$ICON_WARN Status: DRY RUN — no changes made"
|
||||
warn "DRY RUN — no changes made"
|
||||
elif [[ "$TOTAL_FAIL" -eq 0 ]]; then
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS ALL COMPLETE"
|
||||
notify "Weekly sync maintenance complete on $(hostname) — synced + updated (local=$WEEKLY_SYNC_UPDATES remote=$WEEKLY_SYNC_UPDATES_REMOTE)" "Weekly Maintenance" "normal"
|
||||
log "$ICON_DONE Status: all complete ✅ — ${#PASS[@]} share(s) synced, ${#JOB_PASS[@]} job(s) run"
|
||||
else
|
||||
echo "$ICON_ERROR Status: $TOTAL_FAIL failure(s) — check logs"
|
||||
notify "Weekly sync maintenance failed on $(hostname) — sync: ${#FAIL[@]} failed, jobs: ${#JOB_FAIL[@]} failed" "Weekly Maintenance" "warning"
|
||||
warn "Status: $TOTAL_FAIL failure(s)"
|
||||
notify "Weekly maintenance failed on $(hostname) ($MY_ID) — sync: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \
|
||||
"Weekly Maintenance" "warning"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ "$TOTAL_FAIL" -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
Reference in New Issue
Block a user