Gmer4Lfe
d5cf3db2ec
A gated-off section left nothing failed, so the weekly could run for hours and report "all complete" beside "0 shares synced"; skipped is now derived from what was expected rather than self-reported, and the verdict degrades to PARTIAL instead of flattering.
424 lines
17 KiB
Bash
Executable File
424 lines
17 KiB
Bash
Executable File
#!/bin/bash
|
|
# ==============================================================================================
|
|
# ============================= Weekly Sync Maintenance ========================================
|
|
# ==============================================================================================
|
|
#
|
|
# PURPOSE
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Weekly maintenance window orchestrator — clean Emby sync, container updates,
|
|
# and weekly restarts. Schedule: 30 2 * * 0 (Sunday 2:30am)
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL MODEL
|
|
# ==============================================================================================
|
|
#
|
|
# 1. Stop local containers — Emby + auth stack stopped locally
|
|
# 2. Stop remote containers — Emby + auth stack stopped remotely via SSH
|
|
# 3. Pull updates locally — if WEEKLY_SYNC_UPDATES=true (zero extra downtime)
|
|
# 4. Pull updates remotely — if WEEKLY_SYNC_UPDATES_REMOTE=true
|
|
# 5. rsync WEEKLY_SYNC_SHARES — full clean mirror, containers stopped both sides
|
|
# 6. Start remote containers — correct order, delayed start respected
|
|
# 7. Start local containers — rebuild if new image pulled, docker start otherwise
|
|
# 8. WEEKLY_MAINTENANCE_SCRIPTS — weekly restarts etc. (docker_weekly_restart.sh)
|
|
#
|
|
# ==============================================================================================
|
|
# DESIGN PRINCIPLES
|
|
# ==============================================================================================
|
|
#
|
|
# Weekly Cadence Preserves Cache
|
|
# Emby builds a warm image cache on HOST2 throughout the week. Syncing nightly
|
|
# resets that cache — cold loads every morning for users. Weekly sync keeps
|
|
# the cache warm for 6 days, resets Sunday night while users sleep.
|
|
# play_state_sync covers watch/resume state every 30 min between weekly syncs.
|
|
#
|
|
# Zero-Downtime Updates
|
|
# Containers are already stopped for the sync window — image pulls happen at
|
|
# zero extra downtime. Both servers start on identical image versions after
|
|
# the window completes.
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL SAFEGUARDS
|
|
# ==============================================================================================
|
|
#
|
|
# Root Enforcement
|
|
# Container stop/start and rsync both require root.
|
|
#
|
|
# Lock Acquisition
|
|
# acquire_lock prevents concurrent weekly windows. This window stops Emby and the auth
|
|
# stack — two overlapping runs would fight over the same critical containers.
|
|
#
|
|
# Host Detection
|
|
# detect_hosts() sets MY_ID for the banner, summary and notifications.
|
|
#
|
|
# Empty Job List Guard
|
|
# Exits with an error and a notification if WEEKLY_MAINTENANCE_SCRIPTS is empty, rather
|
|
# than taking the weekly outage window and doing nothing with it.
|
|
#
|
|
# Connectivity Check
|
|
# check_connectivity verifies the remote before any remote operation is attempted.
|
|
#
|
|
# Remote Rootfs Check
|
|
# check_remote_rootfs aborts rsync if the remote rootfs is nearly full.
|
|
#
|
|
# Timeout Protection
|
|
# DOCKER_TIMEOUT bounds every docker call and SSH_TIMEOUT every SSH call, so neither a
|
|
# hung daemon nor an unresponsive partner can hold the weekly window open indefinitely.
|
|
#
|
|
# Non-Fatal Steps
|
|
# A failing job is recorded and the remaining jobs still run.
|
|
#
|
|
# Silent on Success
|
|
# Runs weekly; only failures warrant a notification.
|
|
#
|
|
# ==============================================================================================
|
|
# CONFIGURATION
|
|
# ==============================================================================================
|
|
#
|
|
# master.conf
|
|
#
|
|
# WEEKLY_SYNC_SHARES — shares synced during window
|
|
# WEEKLY_MAINTENANCE_SCRIPTS — scripts run after sync
|
|
# WEEKLY_SYNC_UPDATES — toggle local container updates
|
|
# WEEKLY_SYNC_UPDATES_REMOTE — toggle remote container updates
|
|
# WEEKLY_RSYNC_ENABLED — enable/disable rsync section
|
|
#
|
|
# ==============================================================================================
|
|
# RUNTIME MODES
|
|
# ==============================================================================================
|
|
#
|
|
# weekly_sync_maintenance.sh
|
|
# Normal run.
|
|
#
|
|
# weekly_sync_maintenance.sh --dry-run
|
|
# Preview without stopping containers or syncing.
|
|
#
|
|
# weekly_sync_maintenance.sh --log
|
|
# Verbose per-share/per-job output.
|
|
#
|
|
# weekly_sync_maintenance.sh --status
|
|
# Show configuration and exit.
|
|
#
|
|
# ==============================================================================================
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
source "$SCRIPT_DIR/../load_config.sh"
|
|
|
|
ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
|
RSYNC_SCRIPT="$ECOSYSTEM_ROOT/Rsync/rsync.sh"
|
|
|
|
parse_args "$@"
|
|
|
|
DOCKER_TIMEOUT=30 # container stop/start needs longer than normal
|
|
SSH_TIMEOUT=30 # remote pulls can be slow
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Setup ━━━
|
|
# ==============================================================================================
|
|
if [[ "$EUID" -ne 0 ]]; then
|
|
error "Must be run as root"
|
|
exit 1
|
|
fi
|
|
|
|
|
|
acquire_lock
|
|
|
|
if ! command -v docker &>/dev/null; then
|
|
error "Docker command not found"
|
|
exit 1
|
|
fi
|
|
|
|
detect_hosts
|
|
|
|
# An unconfigured job list would run nothing and still report "0/0 passed" — indistinguishable
|
|
# from a healthy run. Fail loudly instead of silently doing no work.
|
|
if [[ ${#WEEKLY_MAINTENANCE_SCRIPTS[@]} -eq 0 ]]; then
|
|
error "WEEKLY_MAINTENANCE_SCRIPTS is empty — no weekly maintenance scripts will run"
|
|
error "Check WEEKLY_MAINTENANCE_SCRIPTS in master.conf"
|
|
notify "weekly maintenance scripts skipped on $(hostname) ($MY_ID) — WEEKLY_MAINTENANCE_SCRIPTS is empty" \
|
|
"$(basename "$0" .sh)" "warning"
|
|
exit 1
|
|
fi
|
|
resolve_remote_ip
|
|
|
|
WINDOW_START=$(date +%s)
|
|
PASS=()
|
|
FAIL=()
|
|
JOB_PASS=()
|
|
JOB_FAIL=()
|
|
|
|
# Load container lists from profile config
|
|
read -r -a MAINTENANCE_CONTAINERS <<< \
|
|
"${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-}"
|
|
|
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be stopped, no sync, no updates"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Status ━━━
|
|
# ==============================================================================================
|
|
if [[ "$SHOW_STATUS" == true ]]; then
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY WEEKLY SYNC STATUS ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_NET Remote: $REMOTE_ID ($REMOTE_SERVER_NAME)"
|
|
echo "$ICON_SYNC Rsync enabled: ${RSYNC_ENABLED:-true}"
|
|
echo "$ICON_SYNC Weekly enabled: ${WEEKLY_RSYNC_ENABLED:-false}"
|
|
echo "$ICON_GEAR Local updates: ${WEEKLY_SYNC_UPDATES:-false}"
|
|
echo "$ICON_GEAR Remote updates: ${WEEKLY_SYNC_UPDATES_REMOTE:-false}"
|
|
echo ""
|
|
echo "━━━ Weekly Sync Shares ━━━"
|
|
if [[ ${#WEEKLY_SYNC_SHARES[@]} -eq 0 ]]; then
|
|
warn " No WEEKLY_SYNC_SHARES configured"
|
|
else
|
|
for share in "${WEEKLY_SYNC_SHARES[@]}"; do
|
|
[[ -n "$share" ]] && echo " $ICON_SYNC $(basename "$share") ($share)"
|
|
done
|
|
fi
|
|
echo ""
|
|
echo "━━━ Weekly Maintenance Scripts ━━━"
|
|
if [[ ${#WEEKLY_MAINTENANCE_SCRIPTS[@]} -eq 0 ]]; then
|
|
echo " None configured"
|
|
else
|
|
for entry in "${WEEKLY_MAINTENANCE_SCRIPTS[@]}"; do
|
|
[[ -n "$entry" ]] && echo " $ICON_GEAR ${entry##*/}"
|
|
done
|
|
fi
|
|
echo ""
|
|
echo "━━━ Containers (from profile config) ━━━"
|
|
for c in "${MAINTENANCE_CONTAINERS[@]}"; do
|
|
[[ -n "$c" ]] && echo " $ICON_CONTAINERS $c"
|
|
done
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Pre-flight Checks ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_GEAR Weekly Sync Maintenance — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
|
echo ""
|
|
echo "━━━ $ICON_SHIELD Pre-flight ━━━"
|
|
|
|
check_connectivity
|
|
check_remote_rootfs
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Stop Containers ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_STOP $ICON_CONTAINERS Stop Containers ━━━"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — containers will not be stopped"
|
|
else
|
|
# Load container lists for stop functions
|
|
read -r -a CRITICAL_CONTAINER_NAMES <<< \
|
|
"${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-}"
|
|
read -r -a LOCAL_CRITICAL_CONTAINER_NAMES <<< \
|
|
"${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-}"
|
|
read -r -a DELAYED_CONTAINERS <<< "${PROFILE_DELAYED_CONTAINERS[critical-data]:-}"
|
|
CONTAINER_DELAY="${PROFILE_CONTAINER_DELAY[critical-data]:-15}"
|
|
|
|
stop_local_containers
|
|
stop_containers
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Container Updates ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_GEAR Container Updates ━━━"
|
|
|
|
declare -A _weekly_needs_rebuild=()
|
|
|
|
if [[ "$WEEKLY_SYNC_UPDATES" == true ]]; then
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
for c in "${MAINTENANCE_CONTAINERS[@]}"; do
|
|
[[ -n "$c" ]] && warn "DRY RUN — would pull: $c"
|
|
done
|
|
else
|
|
log "Pulling local container updates..."
|
|
for c in "${MAINTENANCE_CONTAINERS[@]}"; do
|
|
[[ -z "$c" ]] && continue
|
|
IMAGE=$(timeout "$DOCKER_TIMEOUT" docker inspect \
|
|
"$c" --format '{{.Config.Image}}' 2>/dev/null)
|
|
if [[ -z "$IMAGE" ]]; then
|
|
log "$c — not found locally, skipping update"
|
|
continue
|
|
fi
|
|
_old_id=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "")
|
|
log "Pulling $IMAGE for $c..."
|
|
if docker pull "$IMAGE" >/dev/null 2>&1; then
|
|
_new_id=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "")
|
|
if [[ -n "$_old_id" && "$_old_id" != "$_new_id" ]]; then
|
|
log "$c — new image (${_old_id:7:12} → ${_new_id:7:12}) — will rebuild after sync"
|
|
_weekly_needs_rebuild["$c"]=1
|
|
else
|
|
log "$c — already current"
|
|
fi
|
|
else
|
|
warn "$c — pull failed, will start on existing image"
|
|
fi
|
|
done
|
|
unset _old_id _new_id
|
|
fi
|
|
else
|
|
echo "WEEKLY_SYNC_UPDATES=false — skipping local updates"
|
|
fi
|
|
|
|
if [[ "$WEEKLY_SYNC_UPDATES_REMOTE" == true ]]; then
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would pull updates on $REMOTE_SERVER_NAME"
|
|
else
|
|
log "Pulling remote container updates on $REMOTE_SERVER_NAME..."
|
|
for c in "${MAINTENANCE_CONTAINERS[@]}"; do
|
|
[[ -z "$c" ]] && continue
|
|
IMAGE=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
|
-o ConnectTimeout="$SSH_TIMEOUT" \
|
|
root@"$REMOTE_SERVER" \
|
|
"docker inspect $c --format '{{.Config.Image}}' 2>/dev/null" 2>/dev/null)
|
|
if [[ -z "$IMAGE" ]]; then
|
|
log "$c — not found on remote, skipping update"
|
|
continue
|
|
fi
|
|
log "Pulling $IMAGE for $c on $REMOTE_SERVER_NAME..."
|
|
if timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
|
-o ConnectTimeout="$SSH_TIMEOUT" \
|
|
root@"$REMOTE_SERVER" \
|
|
"docker pull $IMAGE" >/dev/null 2>&1; then
|
|
echo "$c — remote image updated ✅"
|
|
else
|
|
warn "$c — remote pull failed, will start on existing image"
|
|
fi
|
|
done
|
|
fi
|
|
else
|
|
echo "WEEKLY_SYNC_UPDATES_REMOTE=false — skipping remote updates"
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Critical Shares Sync ━━━
|
|
# ==============================================================================================
|
|
SHARE_COUNT=${#WEEKLY_SYNC_SHARES[@]}
|
|
|
|
echo ""
|
|
echo "━━━ $ICON_SYNC Critical Shares Sync — $SHARE_COUNT share(s) ━━━"
|
|
|
|
SYNC_START=$(date +%s)
|
|
JOB_NUM=0
|
|
|
|
if ! check_rsync_enabled "WEEKLY"; then
|
|
warn "Weekly rsync disabled — skipping all $SHARE_COUNT sync job(s)"
|
|
warn "Proceeding to container start and maintenance scripts..."
|
|
elif [[ "$SHARE_COUNT" -eq 0 ]]; then
|
|
warn "No WEEKLY_SYNC_SHARES configured — skipping sync"
|
|
warn "Check WEEKLY_SYNC_SHARES in master.conf"
|
|
else
|
|
RSYNC_DRY=""
|
|
[[ "$DRY_RUN" == true ]] && RSYNC_DRY="--dry-run"
|
|
|
|
for JOB in "${WEEKLY_SYNC_SHARES[@]}"; do
|
|
(( JOB_NUM++ ))
|
|
JOB_NAME=$(basename "$JOB")
|
|
|
|
echo ""
|
|
echo "━━━ [$JOB_NUM/$SHARE_COUNT] $JOB_NAME ━━━"
|
|
|
|
JOB_START=$(date +%s)
|
|
bash "$RSYNC_SCRIPT" "$JOB" $RSYNC_DRY
|
|
EXIT_CODE=$?
|
|
JOB_DUR=$(format_duration $(( $(date +%s) - JOB_START )))
|
|
|
|
if [[ "$EXIT_CODE" -eq 0 ]]; then
|
|
PASS+=("$JOB_NAME")
|
|
echo "$JOB_NAME — done in $JOB_DUR ✅"
|
|
else
|
|
FAIL+=("$JOB_NAME")
|
|
error "$JOB_NAME — failed after $JOB_DUR (exit $EXIT_CODE)"
|
|
fi
|
|
echo ""
|
|
done
|
|
fi
|
|
|
|
SYNC_END=$(date +%s)
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Start Containers ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_START $ICON_CONTAINERS Start Containers ━━━"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — containers will not be started"
|
|
else
|
|
start_containers
|
|
|
|
# Local start — rebuild containers that received a new image, docker start the rest
|
|
if [[ ${#LOCAL_RUNNING_CONTAINERS[@]} -eq 0 ]]; then
|
|
log "No local containers to restart."
|
|
else
|
|
for _c in "${LOCAL_RUNNING_CONTAINERS[@]}"; do
|
|
[[ -z "$_c" ]] && continue
|
|
_needs_delay=false
|
|
for _d in "${DELAYED_CONTAINERS[@]}"; do
|
|
[[ "$_c" == "$_d" ]] && _needs_delay=true && break
|
|
done
|
|
[[ "$_needs_delay" == true ]] && {
|
|
info "Waiting ${CONTAINER_DELAY}s before starting $_c..."
|
|
sleep "$CONTAINER_DELAY"
|
|
}
|
|
if [[ -n "${_weekly_needs_rebuild[$_c]:-}" ]]; then
|
|
log "Rebuilding $_c on new image..."
|
|
if platform_rebuild_container "$_c"; then
|
|
echo "$_c rebuilt on new image ✅"
|
|
else
|
|
warn "$_c rebuild failed — falling back to docker start"
|
|
docker start "$_c" >/dev/null 2>&1 || error "Failed to start $_c"
|
|
fi
|
|
else
|
|
docker start "$_c" >/dev/null 2>&1 && echo "$_c started" || error "Failed to start $_c"
|
|
fi
|
|
done
|
|
unset _c _d _needs_delay
|
|
fi
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Post-sync Jobs ━━━
|
|
# ==============================================================================================
|
|
if [[ ${#WEEKLY_MAINTENANCE_SCRIPTS[@]} -gt 0 ]]; then
|
|
echo ""
|
|
echo "━━━ $ICON_GEAR Post-sync Jobs ━━━"
|
|
for script_entry in "${WEEKLY_MAINTENANCE_SCRIPTS[@]}"; do
|
|
[[ -z "$script_entry" ]] && continue
|
|
echo ""
|
|
run_orch_child "$script_entry"
|
|
done
|
|
fi
|
|
|
|
WINDOW_END=$(date +%s)
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Summary ━━━
|
|
# ==============================================================================================
|
|
# Per-unit detail first — the standard block that follows carries the verdict and the counts, not
|
|
# the names, and knowing WHICH share failed is the whole point of reading a log.
|
|
echo ""
|
|
echo "$ICON_GEAR Updates: local=${WEEKLY_SYNC_UPDATES:-false} remote=${WEEKLY_SYNC_UPDATES_REMOTE:-false}"
|
|
echo ""
|
|
echo "$ICON_SYNC Sync jobs ($SHARE_COUNT):"
|
|
for job in "${PASS[@]}"; do echo " $ICON_DONE $job"; done
|
|
for job in "${FAIL[@]}"; do echo " $ICON_ERROR $job"; done
|
|
|
|
if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then
|
|
echo ""
|
|
echo "$ICON_GEAR Post-sync jobs:"
|
|
for job in "${JOB_PASS[@]}"; do echo " $ICON_DONE $job"; done
|
|
for job in "${JOB_FAIL[@]}"; do echo " $ICON_ERROR $job"; done
|
|
fi
|
|
|
|
# Standard ending. Derives skipped from SHARE_COUNT vs what actually ran, so a gated-off section
|
|
# can no longer read as success — this is the run that printed "all complete — 0 shares synced".
|
|
orchestrator_summary "WEEKLY SYNC MAINTENANCE" "$WINDOW_START" "Weekly Maintenance"
|
|
exit $? |