Gmer4Lfe
2a062e5140
Every orchestrator invoked its children differently — four near-duplicate run_job() copies, a differently-shaped run_watchdog(), or plain inline bash calls, each with its own take on path resolution, pass/fail naming, and dry-run threading. Extracted one shared run_orch_child() into common.sh so there's a single place to fix or extend this behavior going forward. Along the way: watchdog_orchestrator.sh and monthly_maintenance.sh were checking $VERBOSE, a variable nothing in the codebase ever assigns, so --log silently did nothing beyond basic logging on those two. Fixed to $ENABLE_LOGGING. watchdog_orchestrator.sh and array_started.sh had no trailing exit, so their exit codes reflected whatever the last command happened to return rather than actual success/failure. transcode_management.sh had no failure notification and no summary at all. Also made transcode_management.sh's two-script pipeline config-driven (TRANSCODE_MANAGEMENT_SCRIPTS in master.conf) instead of hardcoded, for room to extend it later without editing the orchestrator itself.
408 lines
17 KiB
Bash
Executable File
408 lines
17 KiB
Bash
Executable File
#!/bin/bash
|
|
# ==============================================================================================
|
|
# ============================= Weekly Sync Maintenance ========================================
|
|
# ==============================================================================================
|
|
#
|
|
# PURPOSE
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Weekly maintenance window orchestrator — clean Emby sync, container updates,
|
|
# and weekly restarts. Schedule: 30 2 * * 0 (Sunday 2:30am)
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL MODEL
|
|
# ==============================================================================================
|
|
#
|
|
# 1. Stop local containers — Emby + auth stack stopped locally
|
|
# 2. Stop remote containers — Emby + auth stack stopped remotely via SSH
|
|
# 3. Pull updates locally — if WEEKLY_SYNC_UPDATES=true (zero extra downtime)
|
|
# 4. Pull updates remotely — if WEEKLY_SYNC_UPDATES_REMOTE=true
|
|
# 5. rsync WEEKLY_SYNC_SHARES — full clean mirror, containers stopped both sides
|
|
# 6. Start remote containers — correct order, delayed start respected
|
|
# 7. Start local containers — rebuild if new image pulled, docker start otherwise
|
|
# 8. WEEKLY_MAINTENANCE_SCRIPTS — weekly restarts etc. (docker_weekly_restart.sh)
|
|
#
|
|
# ==============================================================================================
|
|
# DESIGN PRINCIPLES
|
|
# ==============================================================================================
|
|
#
|
|
# Weekly Cadence Preserves Cache
|
|
# Emby builds a warm image cache on HOST2 throughout the week. Syncing nightly
|
|
# resets that cache — cold loads every morning for users. Weekly sync keeps
|
|
# the cache warm for 6 days, resets Sunday night while users sleep.
|
|
# play_state_sync covers watch/resume state every 30 min between weekly syncs.
|
|
#
|
|
# Zero-Downtime Updates
|
|
# Containers are already stopped for the sync window — image pulls happen at
|
|
# zero extra downtime. Both servers start on identical image versions after
|
|
# the window completes.
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL SAFEGUARDS
|
|
# ==============================================================================================
|
|
#
|
|
# Root check — stop/start containers and rsync require root
|
|
# acquire_lock — prevents concurrent weekly windows
|
|
# detect_hosts() — MY_ID in banner, summary, and notifications
|
|
# check_connectivity — verifies remote before any remote operations
|
|
# check_remote_rootfs — aborts rsync if remote rootfs nearly full
|
|
# DOCKER_TIMEOUT — all docker calls protected
|
|
# SSH_TIMEOUT — all SSH calls protected
|
|
# platform_require_cmd — notify validated before use
|
|
# Silent on success — runs weekly; only failures warrant notification
|
|
#
|
|
# ==============================================================================================
|
|
# CONFIGURATION
|
|
# ==============================================================================================
|
|
#
|
|
# master.conf
|
|
#
|
|
# WEEKLY_SYNC_SHARES — shares synced during window
|
|
# WEEKLY_MAINTENANCE_SCRIPTS — scripts run after sync
|
|
# WEEKLY_SYNC_UPDATES — toggle local container updates
|
|
# WEEKLY_SYNC_UPDATES_REMOTE — toggle remote container updates
|
|
# WEEKLY_RSYNC_ENABLED — enable/disable rsync section
|
|
#
|
|
# ==============================================================================================
|
|
# RUNTIME MODES
|
|
# ==============================================================================================
|
|
#
|
|
# weekly_sync_maintenance.sh
|
|
# Normal run.
|
|
#
|
|
# weekly_sync_maintenance.sh --dry-run
|
|
# Preview without stopping containers or syncing.
|
|
#
|
|
# weekly_sync_maintenance.sh --log
|
|
# Verbose per-share/per-job output.
|
|
#
|
|
# weekly_sync_maintenance.sh --status
|
|
# Show configuration and exit.
|
|
#
|
|
# ==============================================================================================
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
source "$SCRIPT_DIR/../load_config.sh"
|
|
|
|
ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
|
RSYNC_SCRIPT="$ECOSYSTEM_ROOT/Rsync/rsync.sh"
|
|
|
|
parse_args "$@"
|
|
|
|
DOCKER_TIMEOUT=30 # container stop/start needs longer than normal
|
|
SSH_TIMEOUT=30 # remote pulls can be slow
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Setup ━━━
|
|
# ==============================================================================================
|
|
if [[ "$EUID" -ne 0 ]]; then
|
|
error "Must be run as root"
|
|
exit 1
|
|
fi
|
|
|
|
|
|
acquire_lock
|
|
|
|
if ! command -v docker &>/dev/null; then
|
|
error "Docker command not found"
|
|
exit 1
|
|
fi
|
|
|
|
detect_hosts
|
|
resolve_remote_ip
|
|
|
|
WINDOW_START=$(date +%s)
|
|
PASS=()
|
|
FAIL=()
|
|
JOB_PASS=()
|
|
JOB_FAIL=()
|
|
|
|
# Load container lists from profile config
|
|
read -r -a MAINTENANCE_CONTAINERS <<< \
|
|
"${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-}"
|
|
|
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be stopped, no sync, no updates"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Status ━━━
|
|
# ==============================================================================================
|
|
if [[ "$SHOW_STATUS" == true ]]; then
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY WEEKLY SYNC STATUS ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_NET Remote: $REMOTE_ID ($REMOTE_SERVER_NAME)"
|
|
echo "$ICON_SYNC Rsync enabled: ${RSYNC_ENABLED:-true}"
|
|
echo "$ICON_SYNC Weekly enabled: ${WEEKLY_RSYNC_ENABLED:-false}"
|
|
echo "$ICON_GEAR Local updates: ${WEEKLY_SYNC_UPDATES:-false}"
|
|
echo "$ICON_GEAR Remote updates: ${WEEKLY_SYNC_UPDATES_REMOTE:-false}"
|
|
echo ""
|
|
echo "━━━ Weekly Sync Shares ━━━"
|
|
if [[ ${#WEEKLY_SYNC_SHARES[@]} -eq 0 ]]; then
|
|
warn " No WEEKLY_SYNC_SHARES configured"
|
|
else
|
|
for share in "${WEEKLY_SYNC_SHARES[@]}"; do
|
|
[[ -n "$share" ]] && echo " $ICON_SYNC $(basename "$share") ($share)"
|
|
done
|
|
fi
|
|
echo ""
|
|
echo "━━━ Weekly Maintenance Scripts ━━━"
|
|
if [[ ${#WEEKLY_MAINTENANCE_SCRIPTS[@]} -eq 0 ]]; then
|
|
echo " None configured"
|
|
else
|
|
for entry in "${WEEKLY_MAINTENANCE_SCRIPTS[@]}"; do
|
|
[[ -n "$entry" ]] && echo " $ICON_GEAR ${entry##*/}"
|
|
done
|
|
fi
|
|
echo ""
|
|
echo "━━━ Containers (from profile config) ━━━"
|
|
for c in "${MAINTENANCE_CONTAINERS[@]}"; do
|
|
[[ -n "$c" ]] && echo " $ICON_CONTAINERS $c"
|
|
done
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Pre-flight Checks ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_GEAR Weekly Sync Maintenance — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
|
echo ""
|
|
echo "━━━ $ICON_SHIELD Pre-flight ━━━"
|
|
|
|
check_connectivity
|
|
check_remote_rootfs
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Stop Containers ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_STOP $ICON_CONTAINERS Stop Containers ━━━"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — containers will not be stopped"
|
|
else
|
|
# Load container lists for stop functions
|
|
read -r -a CRITICAL_CONTAINER_NAMES <<< \
|
|
"${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-}"
|
|
read -r -a DELAYED_CONTAINERS <<< "${PROFILE_DELAYED_CONTAINERS[critical-data]:-}"
|
|
CONTAINER_DELAY="${PROFILE_CONTAINER_DELAY[critical-data]:-15}"
|
|
|
|
stop_local_containers
|
|
stop_containers
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Container Updates ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_GEAR Container Updates ━━━"
|
|
|
|
declare -A _weekly_needs_rebuild=()
|
|
|
|
if [[ "$WEEKLY_SYNC_UPDATES" == true ]]; then
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
for c in "${MAINTENANCE_CONTAINERS[@]}"; do
|
|
[[ -n "$c" ]] && warn "DRY RUN — would pull: $c"
|
|
done
|
|
else
|
|
log "Pulling local container updates..."
|
|
for c in "${MAINTENANCE_CONTAINERS[@]}"; do
|
|
[[ -z "$c" ]] && continue
|
|
IMAGE=$(timeout "$DOCKER_TIMEOUT" docker inspect \
|
|
"$c" --format '{{.Config.Image}}' 2>/dev/null)
|
|
if [[ -z "$IMAGE" ]]; then
|
|
log "$c — not found locally, skipping update"
|
|
continue
|
|
fi
|
|
_old_id=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "")
|
|
log "Pulling $IMAGE for $c..."
|
|
if docker pull "$IMAGE" >/dev/null 2>&1; then
|
|
_new_id=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "")
|
|
if [[ -n "$_old_id" && "$_old_id" != "$_new_id" ]]; then
|
|
log "$c — new image (${_old_id:7:12} → ${_new_id:7:12}) — will rebuild after sync"
|
|
_weekly_needs_rebuild["$c"]=1
|
|
else
|
|
log "$c — already current"
|
|
fi
|
|
else
|
|
warn "$c — pull failed, will start on existing image"
|
|
fi
|
|
done
|
|
unset _old_id _new_id
|
|
fi
|
|
else
|
|
echo "WEEKLY_SYNC_UPDATES=false — skipping local updates"
|
|
fi
|
|
|
|
if [[ "$WEEKLY_SYNC_UPDATES_REMOTE" == true ]]; then
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would pull updates on $REMOTE_SERVER_NAME"
|
|
else
|
|
log "Pulling remote container updates on $REMOTE_SERVER_NAME..."
|
|
for c in "${MAINTENANCE_CONTAINERS[@]}"; do
|
|
[[ -z "$c" ]] && continue
|
|
IMAGE=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
|
-o ConnectTimeout="$SSH_TIMEOUT" \
|
|
root@"$REMOTE_SERVER" \
|
|
"docker inspect $c --format '{{.Config.Image}}' 2>/dev/null" 2>/dev/null)
|
|
if [[ -z "$IMAGE" ]]; then
|
|
log "$c — not found on remote, skipping update"
|
|
continue
|
|
fi
|
|
log "Pulling $IMAGE for $c on $REMOTE_SERVER_NAME..."
|
|
if timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
|
-o ConnectTimeout="$SSH_TIMEOUT" \
|
|
root@"$REMOTE_SERVER" \
|
|
"docker pull $IMAGE" >/dev/null 2>&1; then
|
|
echo "$c — remote image updated ✅"
|
|
else
|
|
warn "$c — remote pull failed, will start on existing image"
|
|
fi
|
|
done
|
|
fi
|
|
else
|
|
echo "WEEKLY_SYNC_UPDATES_REMOTE=false — skipping remote updates"
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Critical Shares Sync ━━━
|
|
# ==============================================================================================
|
|
SHARE_COUNT=${#WEEKLY_SYNC_SHARES[@]}
|
|
|
|
echo ""
|
|
echo "━━━ $ICON_SYNC Critical Shares Sync — $SHARE_COUNT share(s) ━━━"
|
|
|
|
SYNC_START=$(date +%s)
|
|
JOB_NUM=0
|
|
|
|
if ! check_rsync_enabled "WEEKLY"; then
|
|
warn "Weekly rsync disabled — skipping all $SHARE_COUNT sync job(s)"
|
|
warn "Proceeding to container start and maintenance scripts..."
|
|
elif [[ "$SHARE_COUNT" -eq 0 ]]; then
|
|
warn "No WEEKLY_SYNC_SHARES configured — skipping sync"
|
|
warn "Check WEEKLY_SYNC_SHARES in master.conf"
|
|
else
|
|
RSYNC_DRY=""
|
|
[[ "$DRY_RUN" == true ]] && RSYNC_DRY="--dry-run"
|
|
|
|
for JOB in "${WEEKLY_SYNC_SHARES[@]}"; do
|
|
(( JOB_NUM++ ))
|
|
JOB_NAME=$(basename "$JOB")
|
|
|
|
echo ""
|
|
echo "━━━ [$JOB_NUM/$SHARE_COUNT] $JOB_NAME ━━━"
|
|
|
|
JOB_START=$(date +%s)
|
|
bash "$RSYNC_SCRIPT" "$JOB" $RSYNC_DRY
|
|
EXIT_CODE=$?
|
|
JOB_DUR=$(format_duration $(( $(date +%s) - JOB_START )))
|
|
|
|
if [[ "$EXIT_CODE" -eq 0 ]]; then
|
|
PASS+=("$JOB_NAME")
|
|
echo "$JOB_NAME — done in $JOB_DUR ✅"
|
|
else
|
|
FAIL+=("$JOB_NAME")
|
|
error "$JOB_NAME — failed after $JOB_DUR (exit $EXIT_CODE)"
|
|
fi
|
|
echo ""
|
|
done
|
|
fi
|
|
|
|
SYNC_END=$(date +%s)
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Start Containers ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_START $ICON_CONTAINERS Start Containers ━━━"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — containers will not be started"
|
|
else
|
|
start_containers
|
|
|
|
# Local start — rebuild containers that received a new image, docker start the rest
|
|
if [[ ${#LOCAL_RUNNING_CONTAINERS[@]} -eq 0 ]]; then
|
|
log "No local containers to restart."
|
|
else
|
|
for _c in "${LOCAL_RUNNING_CONTAINERS[@]}"; do
|
|
[[ -z "$_c" ]] && continue
|
|
_needs_delay=false
|
|
for _d in "${DELAYED_CONTAINERS[@]}"; do
|
|
[[ "$_c" == "$_d" ]] && _needs_delay=true && break
|
|
done
|
|
[[ "$_needs_delay" == true ]] && {
|
|
info "Waiting ${CONTAINER_DELAY}s before starting $_c..."
|
|
sleep "$CONTAINER_DELAY"
|
|
}
|
|
if [[ -n "${_weekly_needs_rebuild[$_c]:-}" ]]; then
|
|
log "Rebuilding $_c on new image..."
|
|
if platform_rebuild_container "$_c"; then
|
|
echo "$_c rebuilt on new image ✅"
|
|
else
|
|
warn "$_c rebuild failed — falling back to docker start"
|
|
docker start "$_c" >/dev/null 2>&1 || error "Failed to start $_c"
|
|
fi
|
|
else
|
|
docker start "$_c" >/dev/null 2>&1 && echo "$_c started" || error "Failed to start $_c"
|
|
fi
|
|
done
|
|
unset _c _d _needs_delay
|
|
fi
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Post-sync Jobs ━━━
|
|
# ==============================================================================================
|
|
if [[ ${#WEEKLY_MAINTENANCE_SCRIPTS[@]} -gt 0 ]]; then
|
|
echo ""
|
|
echo "━━━ $ICON_GEAR Post-sync Jobs ━━━"
|
|
for script_entry in "${WEEKLY_MAINTENANCE_SCRIPTS[@]}"; do
|
|
[[ -z "$script_entry" ]] && continue
|
|
echo ""
|
|
run_orch_child "$script_entry"
|
|
done
|
|
fi
|
|
|
|
WINDOW_END=$(date +%s)
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Summary ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY WEEKLY SYNC MAINTENANCE SUMMARY ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_TIME Window: $(date -d @"$WINDOW_START" '+%Y-%m-%d %H:%M:%S') → $(date -d @"$WINDOW_END" '+%H:%M:%S')"
|
|
echo "$ICON_TIME Duration: $(format_duration $(( WINDOW_END - WINDOW_START )))"
|
|
echo "$ICON_GEAR Updates: local=${WEEKLY_SYNC_UPDATES:-false} remote=${WEEKLY_SYNC_UPDATES_REMOTE:-false}"
|
|
echo ""
|
|
|
|
echo "$ICON_SYNC Sync jobs ($SHARE_COUNT):"
|
|
for job in "${PASS[@]}"; do echo " $ICON_DONE $job"; done
|
|
for job in "${FAIL[@]}"; do echo " $ICON_ERROR $job"; done
|
|
echo " Passed: ${#PASS[@]} Failed: ${#FAIL[@]}"
|
|
|
|
if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then
|
|
echo ""
|
|
echo "$ICON_GEAR Post-sync jobs:"
|
|
for job in "${JOB_PASS[@]}"; do echo " $ICON_DONE $job"; done
|
|
for job in "${JOB_FAIL[@]}"; do echo " $ICON_ERROR $job"; done
|
|
fi
|
|
|
|
TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} ))
|
|
|
|
echo ""
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — no changes made"
|
|
elif [[ "$TOTAL_FAIL" -eq 0 ]]; then
|
|
echo "$ICON_DONE Status: all complete ✅ — ${#PASS[@]} share(s) synced, ${#JOB_PASS[@]} job(s) run"
|
|
else
|
|
warn "Status: $TOTAL_FAIL failure(s)"
|
|
notify "Weekly maintenance failed on $(hostname) ($MY_ID) — sync: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \
|
|
"Weekly Maintenance" "warning"
|
|
fi
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
|
|
[[ "$TOTAL_FAIL" -gt 0 ]] && exit 1
|
|
exit 0 |