massive update. Master conf split, now modular with a load sceriprt to drive all configs to scripts. with unraid scpecific safeguard tests , and improved standardized ux. including dynamic host detect, who am i who else it there. EVERY SINGLE SCRIPT UPDATED. DEBATING THAT THIS IS ACUALLY V2
This commit is contained in:
@@ -1,99 +1,193 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ----------------------------- Weekly Sync Maintenance ----------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ============================= Weekly Sync Maintenance ========================================
|
||||
# ==============================================================================================
|
||||
# Weekly maintenance window orchestrator — clean sync, container updates, weekly restarts.
|
||||
# Schedule: 30 2 * * 0 (Sunday 2:30am — fits before 3am network reboot)
|
||||
# Schedule: 30 2 * * 0 (Sunday 2:30am — before Sunday 7am coffee report)
|
||||
#
|
||||
# Execution order:
|
||||
# 1. Stop local containers — Emby + auth stack stopped locally
|
||||
# 2. Stop remote containers — Emby + auth stack stopped remotely via SSH
|
||||
# 3. Pull updates locally — if WEEKLY_SYNC_UPDATES=true
|
||||
# 4. Pull updates remotely — if WEEKLY_SYNC_UPDATES_REMOTE=true
|
||||
# 5. rsync Emby — full clean mirror, both instances stopped
|
||||
# 6. rsync Critical-Data — auth stack clean sync, databases flushed
|
||||
# 7. Start remote containers — correct order, delayed start respected
|
||||
# 8. Start local containers — correct order, delayed start respected
|
||||
# 9. docker_weekly_restart.sh — weekly container restarts
|
||||
# ── EXECUTION ORDER ───────────────────────────────────────────────────────────────────────────
|
||||
# 1. Stop local containers — Emby + auth stack stopped locally
|
||||
# 2. Stop remote containers — Emby + auth stack stopped remotely via SSH
|
||||
# 3. Pull updates locally — if WEEKLY_SYNC_UPDATES=true (zero extra downtime)
|
||||
# 4. Pull updates remotely — if WEEKLY_SYNC_UPDATES_REMOTE=true
|
||||
# 5. rsync WEEKLY_SYNC_SHARES — full clean mirror, containers stopped both sides
|
||||
# 6. Start remote containers — correct order, delayed start respected
|
||||
# 7. Start local containers — correct order, delayed start respected
|
||||
# 8. WEEKLY_MAINTENANCE_SCRIPTS — weekly restarts etc. (docker_weekly_restart.sh)
|
||||
#
|
||||
# Synced shares (WEEKLY_SYNC_SHARES in Master.conf):
|
||||
# /mnt/user/Media_Server/Emby — emby profile — full mirror, cache resets weekly
|
||||
# /mnt/user/appdata-Failover/Critical-Data — critical-data — auth stack clean state
|
||||
# ── WHY WEEKLY NOT NIGHTLY FOR EMBY ──────────────────────────────────────────────────────────
|
||||
# Emby builds a warm image cache on HOST2 throughout the week.
|
||||
# Syncing nightly resets cache — cold loads every morning for users.
|
||||
# Weekly sync: cache stays warm 6 days, resets Sunday night while users sleep.
|
||||
# emby-failover dirty sync covers watch states + library every 15min between weekly syncs.
|
||||
#
|
||||
# Why weekly instead of nightly for Emby:
|
||||
# Emby builds a warm image cache on HOST2 throughout the week
|
||||
# Syncing nightly resets cache — cold loads every morning for users
|
||||
# Weekly sync: cache stays warm 6 days, resets Sunday night while users sleep
|
||||
# emby-failover dirty sync covers watch states + library every 30-60min between syncs
|
||||
# ── CONTAINER UPDATES ─────────────────────────────────────────────────────────────────────────
|
||||
# Containers already stopped for sync — updates pull at zero extra downtime.
|
||||
# Both servers start on identical image versions after the window completes.
|
||||
# Toggle: WEEKLY_SYNC_UPDATES / WEEKLY_SYNC_UPDATES_REMOTE in master.conf
|
||||
#
|
||||
# Container updates during the window:
|
||||
# Containers already stopped for sync — updates pull at zero extra downtime
|
||||
# Both servers start on identical image versions after the window completes
|
||||
# Toggle: WEEKLY_SYNC_UPDATES / WEEKLY_SYNC_UPDATES_REMOTE in Master.conf
|
||||
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
|
||||
# detect_hosts() sets MY_ID — used in banner, summary, and notifications.
|
||||
# WEEKLY_SYNC_SHARES and WEEKLY_MAINTENANCE_SCRIPTS configured in master.conf.
|
||||
# Same script runs correctly on both servers.
|
||||
#
|
||||
# What triggers weekly_health_digest.sh:
|
||||
# NOT this script — weekly_health_digest.sh runs on its own Saturday schedule
|
||||
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
|
||||
# Root check — stop/start containers, rsync require root
|
||||
# acquire_lock — prevents concurrent weekly windows
|
||||
# check_connectivity — verifies remote before any remote operations
|
||||
# check_remote_rootfs — aborts if remote rootfs nearly full
|
||||
# DOCKER_TIMEOUT — all docker calls protected
|
||||
# SSH_TIMEOUT — all SSH calls protected
|
||||
# validate_unraid_cmd — notify validated before use
|
||||
# Silent on success — runs weekly, only failures warrant notification
|
||||
#
|
||||
# Configuration in Master.conf:
|
||||
# WEEKLY_SYNC_SHARES — shares synced during the maintenance window
|
||||
# WEEKLY_MAINTENANCE_SCRIPTS — scripts run after sync (docker_weekly_restart)
|
||||
# WEEKLY_SYNC_UPDATES — toggle container updates on/off
|
||||
# WEEKLY_SYNC_UPDATES_REMOTE — toggle remote container updates on/off
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# All configuration in Master.conf.
|
||||
# Supports --dry-run to walk through without stopping containers, syncing, or updating.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||||
# WEEKLY_SYNC_SHARES — shares synced during window
|
||||
# WEEKLY_MAINTENANCE_SCRIPTS — scripts run after sync
|
||||
# WEEKLY_SYNC_UPDATES — toggle local container updates
|
||||
# WEEKLY_SYNC_UPDATES_REMOTE — toggle remote container updates
|
||||
# WEEKLY_RSYNC_ENABLED — enable/disable rsync section
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# weekly_sync_maintenance.sh — normal run
|
||||
# weekly_sync_maintenance.sh --dry-run — preview without stopping containers or syncing
|
||||
# weekly_sync_maintenance.sh --log — verbose per-share/per-job output
|
||||
# weekly_sync_maintenance.sh --status — show configuration and exit
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
RSYNC_SCRIPT="$SCRIPT_DIR/../Rsync/rsync.sh"
|
||||
SCRIPTS_ROOT="$SCRIPT_DIR/.."
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
DOCKER_TIMEOUT=30 # container stop/start needs longer than normal
|
||||
SSH_TIMEOUT=30 # remote pulls can be slow
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Running as root"
|
||||
validate_unraid_cmd \
|
||||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||||
"" "" \
|
||||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||
|
||||
acquire_lock
|
||||
|
||||
detect_hosts
|
||||
resolve_remote_ip
|
||||
|
||||
# Load container list from emby profile — used for update pulls
|
||||
read -r -a MAINTENANCE_CONTAINERS <<< "${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-}"
|
||||
WINDOW_START=$(date +%s)
|
||||
PASS=()
|
||||
FAIL=()
|
||||
JOB_PASS=()
|
||||
JOB_FAIL=()
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SHIELD Pre-flight Checks ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Load container lists from profile config
|
||||
read -r -a MAINTENANCE_CONTAINERS <<< \
|
||||
"${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-}"
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be stopped, no sync, no updates"
|
||||
|
||||
# ── Helper — run a post-sync maintenance script ────────────────────────────────────────────────
|
||||
run_job() {
|
||||
local script_entry="$1"
|
||||
local extra_dry=""
|
||||
[[ "$DRY_RUN" == true ]] && extra_dry="--dry-run"
|
||||
|
||||
read -r -a script_args <<< "$script_entry"
|
||||
local script_path="$SCRIPTS_ROOT/${script_args[0]}"
|
||||
local script_name
|
||||
script_name=$(basename "${script_args[0]}")
|
||||
local extra_args=("${script_args[@]:1}")
|
||||
|
||||
if [[ ! -f "$script_path" ]]; then
|
||||
error "$script_name — not found at $script_path"
|
||||
JOB_FAIL+=("$script_name")
|
||||
return 1
|
||||
fi
|
||||
|
||||
log "Running: $script_name ${extra_args[*]}"
|
||||
# shellcheck disable=SC2086
|
||||
if bash "$script_path" "${extra_args[@]}" $extra_dry; then
|
||||
log "$script_name — done ✅"
|
||||
JOB_PASS+=("$script_name ${extra_args[*]}")
|
||||
else
|
||||
error "$script_name — failed (exit $?)"
|
||||
JOB_FAIL+=("$script_name ${extra_args[*]}")
|
||||
fi
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY WEEKLY SYNC STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_NET Remote: $REMOTE_ID ($REMOTE_SERVER_NAME)"
|
||||
echo "$ICON_SYNC Rsync enabled: ${RSYNC_ENABLED:-true}"
|
||||
echo "$ICON_SYNC Weekly enabled: ${WEEKLY_RSYNC_ENABLED:-false}"
|
||||
echo "$ICON_GEAR Local updates: ${WEEKLY_SYNC_UPDATES:-false}"
|
||||
echo "$ICON_GEAR Remote updates: ${WEEKLY_SYNC_UPDATES_REMOTE:-false}"
|
||||
echo ""
|
||||
echo "━━━ Weekly Sync Shares ━━━"
|
||||
if [[ ${#WEEKLY_SYNC_SHARES[@]} -eq 0 ]]; then
|
||||
warn " No WEEKLY_SYNC_SHARES configured"
|
||||
else
|
||||
for share in "${WEEKLY_SYNC_SHARES[@]:-}"; do
|
||||
[[ -n "$share" ]] && echo " $ICON_SYNC $(basename "$share") ($share)"
|
||||
done
|
||||
fi
|
||||
echo ""
|
||||
echo "━━━ Weekly Maintenance Scripts ━━━"
|
||||
if [[ ${#WEEKLY_MAINTENANCE_SCRIPTS[@]} -eq 0 ]]; then
|
||||
echo " None configured"
|
||||
else
|
||||
for entry in "${WEEKLY_MAINTENANCE_SCRIPTS[@]:-}"; do
|
||||
[[ -n "$entry" ]] && echo " $ICON_GEAR $(basename "${entry%% *}") ${entry#* }"
|
||||
done
|
||||
fi
|
||||
echo ""
|
||||
echo "━━━ Containers (from profile config) ━━━"
|
||||
for c in "${MAINTENANCE_CONTAINERS[@]:-}"; do
|
||||
[[ -n "$c" ]] && echo " $ICON_CONTAINERS $c"
|
||||
done
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Pre-flight Checks ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_SHIELD Pre-flight Checks ━━━"
|
||||
echo "━━━ $ICON_GEAR Weekly Sync Maintenance — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo ""
|
||||
echo "━━━ $ICON_SHIELD Pre-flight ━━━"
|
||||
|
||||
check_connectivity
|
||||
check_remote_rootfs
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_STOP $ICON_CONTAINERS Stop Containers ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Stop Containers ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_STOP $ICON_CONTAINERS Stop Containers ━━━"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — containers will not be stopped"
|
||||
else
|
||||
# Load critical-data + emby container list for stops
|
||||
read -r -a CRITICAL_CONTAINER_NAMES <<< "${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-}"
|
||||
# Load container lists for stop functions
|
||||
read -r -a CRITICAL_CONTAINER_NAMES <<< \
|
||||
"${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-}"
|
||||
read -r -a DELAYED_CONTAINERS <<< "${PROFILE_DELAYED_CONTAINERS[critical-data]:-}"
|
||||
CONTAINER_DELAY="${PROFILE_CONTAINER_DELAY[critical-data]:-15}"
|
||||
|
||||
@@ -101,123 +195,118 @@ else
|
||||
stop_containers
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Container Updates ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Container Updates ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Container Updates ━━━"
|
||||
|
||||
if [[ "$WEEKLY_SYNC_UPDATES" == true ]]; then
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would pull updates for local containers"
|
||||
for c in "${MAINTENANCE_CONTAINERS[@]}"; do
|
||||
[[ -z "$c" ]] && continue
|
||||
warn "DRY RUN — would docker pull: $c"
|
||||
for c in "${MAINTENANCE_CONTAINERS[@]:-}"; do
|
||||
[[ -n "$c" ]] && warn "DRY RUN — would pull: $c"
|
||||
done
|
||||
else
|
||||
info "Pulling local container updates..."
|
||||
for c in "${MAINTENANCE_CONTAINERS[@]}"; do
|
||||
log "Pulling local container updates..."
|
||||
for c in "${MAINTENANCE_CONTAINERS[@]:-}"; do
|
||||
[[ -z "$c" ]] && continue
|
||||
# Get image name from running or stopped container
|
||||
IMAGE=$(docker inspect "$c" --format '{{.Config.Image}}' 2>/dev/null)
|
||||
IMAGE=$(timeout "$DOCKER_TIMEOUT" docker inspect \
|
||||
"$c" --format '{{.Config.Image}}' 2>/dev/null)
|
||||
if [[ -z "$IMAGE" ]]; then
|
||||
log "$c — not found locally, skipping update"
|
||||
continue
|
||||
fi
|
||||
info "Pulling $IMAGE for $c..."
|
||||
log "Pulling $IMAGE for $c..."
|
||||
if docker pull "$IMAGE" >/dev/null 2>&1; then
|
||||
success "$c — image updated"
|
||||
log "$c — image updated ✅"
|
||||
else
|
||||
warn "$c — pull failed, will start on existing image"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
else
|
||||
info "WEEKLY_SYNC_UPDATES=false — skipping local updates"
|
||||
log "WEEKLY_SYNC_UPDATES=false — skipping local updates"
|
||||
fi
|
||||
|
||||
if [[ "$WEEKLY_SYNC_UPDATES_REMOTE" == true ]]; then
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would pull updates on $REMOTE_SERVER_NAME"
|
||||
else
|
||||
info "Pulling remote container updates on $REMOTE_SERVER_NAME..."
|
||||
for c in "${MAINTENANCE_CONTAINERS[@]}"; do
|
||||
log "Pulling remote container updates on $REMOTE_SERVER_NAME..."
|
||||
for c in "${MAINTENANCE_CONTAINERS[@]:-}"; do
|
||||
[[ -z "$c" ]] && continue
|
||||
IMAGE=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
||||
IMAGE=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" \
|
||||
root@"$REMOTE_SERVER" \
|
||||
"docker inspect $c --format '{{.Config.Image}}' 2>/dev/null" 2>/dev/null)
|
||||
if [[ -z "$IMAGE" ]]; then
|
||||
log "$c — not found on remote, skipping update"
|
||||
continue
|
||||
fi
|
||||
info "Pulling $IMAGE for $c on $REMOTE_SERVER_NAME..."
|
||||
if ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" "docker pull $IMAGE" >/dev/null 2>&1; then
|
||||
success "$c — remote image updated"
|
||||
log "Pulling $IMAGE for $c on $REMOTE_SERVER_NAME..."
|
||||
if timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" \
|
||||
root@"$REMOTE_SERVER" \
|
||||
"docker pull $IMAGE" >/dev/null 2>&1; then
|
||||
log "$c — remote image updated ✅"
|
||||
else
|
||||
warn "$c — remote pull failed, will start on existing image"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
else
|
||||
info "WEEKLY_SYNC_UPDATES_REMOTE=false — skipping remote updates"
|
||||
log "WEEKLY_SYNC_UPDATES_REMOTE=false — skipping remote updates"
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SYNC Critical Shares Sync ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
PASS=()
|
||||
FAIL=()
|
||||
TOTAL_START=$(date +%s)
|
||||
|
||||
SYNC_JOBS=("${WEEKLY_SYNC_SHARES[@]}")
|
||||
SHARE_COUNT=${#SYNC_JOBS[@]}
|
||||
# ==============================================================================================
|
||||
# ━━━ Critical Shares Sync ━━━
|
||||
# ==============================================================================================
|
||||
SHARE_COUNT=${#WEEKLY_SYNC_SHARES[@]}
|
||||
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Critical Shares Sync — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo "$ICON_SUMMARY Jobs: $SHARE_COUNT"
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Critical Shares Sync — $SHARE_COUNT share(s) ━━━"
|
||||
|
||||
SYNC_START=$(date +%s)
|
||||
JOB_NUM=0
|
||||
|
||||
# Tier 1 + Tier 2 rsync gate check
|
||||
if ! check_rsync_enabled "WEEKLY"; then
|
||||
warn "Rsync disabled — skipping all $SHARE_COUNT weekly sync jobs"
|
||||
warn "Proceeding to container updates and maintenance scripts..."
|
||||
warn "Weekly rsync disabled — skipping all $SHARE_COUNT sync job(s)"
|
||||
warn "Proceeding to container start and maintenance scripts..."
|
||||
elif [[ "$SHARE_COUNT" -eq 0 ]]; then
|
||||
warn "No WEEKLY_SYNC_SHARES configured — skipping sync"
|
||||
warn "Check WEEKLY_SYNC_SHARES in master.conf"
|
||||
else
|
||||
for JOB in "${SYNC_JOBS[@]}"; do
|
||||
((JOB_NUM++))
|
||||
RSYNC_DRY=""
|
||||
[[ "$DRY_RUN" == true ]] && RSYNC_DRY="--dry-run"
|
||||
|
||||
for JOB in "${WEEKLY_SYNC_SHARES[@]}"; do
|
||||
(( JOB_NUM++ ))
|
||||
JOB_NAME=$(basename "$JOB")
|
||||
|
||||
echo ""
|
||||
echo "━━━ [$JOB_NUM/$SHARE_COUNT] $JOB_NAME ━━━"
|
||||
|
||||
JOB_START=$(date +%s)
|
||||
JOB_START=$(date +%s)
|
||||
bash "$RSYNC_SCRIPT" "$JOB" $RSYNC_DRY
|
||||
EXIT_CODE=$?
|
||||
JOB_DUR=$(format_duration $(( $(date +%s) - JOB_START )))
|
||||
|
||||
# Containers already stopped — rsync profile won't try to stop them again
|
||||
# Pass --no-container-stop flag would be ideal but profiles handle this naturally
|
||||
# since containers are already stopped, stop_containers finds nothing running
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
bash "$RSYNC_SCRIPT" "$JOB" --dry-run
|
||||
else
|
||||
bash "$RSYNC_SCRIPT" "$JOB"
|
||||
fi
|
||||
if [[ "$EXIT_CODE" -eq 0 ]]; then
|
||||
PASS+=("$JOB_NAME")
|
||||
log "$JOB_NAME — done in $JOB_DUR ✅"
|
||||
else
|
||||
FAIL+=("$JOB_NAME")
|
||||
error "$JOB_NAME — failed after $JOB_DUR (exit $EXIT_CODE)"
|
||||
fi
|
||||
echo ""
|
||||
done
|
||||
fi
|
||||
|
||||
EXIT_CODE=$?
|
||||
JOB_END=$(date +%s)
|
||||
JOB_DURATION=$(format_duration $(( JOB_END - JOB_START )))
|
||||
SYNC_END=$(date +%s)
|
||||
|
||||
if [[ "$EXIT_CODE" -eq 0 ]]; then
|
||||
PASS+=("$JOB_NAME")
|
||||
success "$JOB_NAME — $ICON_SUCCESS done in $JOB_DURATION"
|
||||
else
|
||||
FAIL+=("$JOB_NAME")
|
||||
error "$JOB_NAME — $ICON_ERROR failed after $JOB_DURATION"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
done
|
||||
fi # end check_rsync_enabled "WEEKLY"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_START $ICON_CONTAINERS Start Containers ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Start Containers ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_START $ICON_CONTAINERS Start Containers ━━━"
|
||||
|
||||
@@ -228,88 +317,57 @@ else
|
||||
start_local_containers
|
||||
fi
|
||||
|
||||
TOTAL_END=$(date +%s)
|
||||
TOTAL_DURATION=$(format_duration $(( TOTAL_END - TOTAL_START )))
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Post-sync Jobs ━━━
|
||||
# docker_weekly_restart.sh and any other WEEKLY_MAINTENANCE_SCRIPTS run after sync
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
JOB_PASS=()
|
||||
JOB_FAIL=()
|
||||
SCRIPTS_ROOT="$SCRIPT_DIR/.."
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Post-sync Jobs ━━━
|
||||
# ==============================================================================================
|
||||
if [[ ${#WEEKLY_MAINTENANCE_SCRIPTS[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Post-sync Jobs ━━━"
|
||||
|
||||
for script_entry in "${WEEKLY_MAINTENANCE_SCRIPTS[@]}"; do
|
||||
for script_entry in "${WEEKLY_MAINTENANCE_SCRIPTS[@]:-}"; do
|
||||
[[ -z "$script_entry" ]] && continue
|
||||
script_args=($script_entry)
|
||||
script_path="$SCRIPTS_ROOT/${script_args[0]}"
|
||||
script_name=$(basename "${script_args[0]}")
|
||||
extra_args=("${script_args[@]:1}")
|
||||
|
||||
echo ""
|
||||
info "$ICON_START Running: $script_name"
|
||||
|
||||
if [[ ! -f "$script_path" ]]; then
|
||||
error "$script_name — not found at $script_path"
|
||||
JOB_FAIL+=("$script_name")
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would run: $script_name"
|
||||
JOB_PASS+=("$script_name (dry run)")
|
||||
elif bash "$script_path" "${extra_args[@]}"; then
|
||||
success "$script_name — done"
|
||||
JOB_PASS+=("$script_name")
|
||||
else
|
||||
error "$script_name — failed"
|
||||
JOB_FAIL+=("$script_name")
|
||||
fi
|
||||
run_job "$script_entry"
|
||||
done
|
||||
fi
|
||||
|
||||
WINDOW_END=$(date +%s)
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Summary ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY WEEKLY SYNC MAINTENANCE SUMMARY ━━━━━"
|
||||
echo "$ICON_TIME Duration: $TOTAL_DURATION"
|
||||
echo "$ICON_GEAR Updates: local=$WEEKLY_SYNC_UPDATES remote=$WEEKLY_SYNC_UPDATES_REMOTE"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_TIME Window: $(date -d @"$WINDOW_START" '+%Y-%m-%d %H:%M:%S') → $(date -d @"$WINDOW_END" '+%H:%M:%S')"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( WINDOW_END - WINDOW_START )))"
|
||||
echo "$ICON_GEAR Updates: local=${WEEKLY_SYNC_UPDATES:-false} remote=${WEEKLY_SYNC_UPDATES_REMOTE:-false}"
|
||||
echo ""
|
||||
|
||||
echo "$ICON_SYNC Sync jobs:"
|
||||
if [[ ${#PASS[@]} -gt 0 ]]; then
|
||||
for job in "${PASS[@]}"; do echo " $ICON_SUCCESS $job"; done
|
||||
fi
|
||||
if [[ ${#FAIL[@]} -gt 0 ]]; then
|
||||
for job in "${FAIL[@]}"; do echo " $ICON_ERROR $job"; done
|
||||
fi
|
||||
echo " Passed: ${#PASS[@]} Failed: ${#FAIL[@]}"
|
||||
echo "$ICON_SYNC Sync jobs ($SHARE_COUNT):"
|
||||
for job in "${PASS[@]:-}"; do echo " $ICON_DONE $job"; done
|
||||
for job in "${FAIL[@]:-}"; do echo " $ICON_ERROR $job"; done
|
||||
echo " Passed: ${#PASS[@]} Failed: ${#FAIL[@]}"
|
||||
|
||||
if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
echo "$ICON_GEAR Post-sync jobs:"
|
||||
for job in "${JOB_PASS[@]}"; do echo " $ICON_SUCCESS $job"; done
|
||||
for job in "${JOB_FAIL[@]}"; do echo " $ICON_ERROR $job"; done
|
||||
for job in "${JOB_PASS[@]:-}"; do echo " $ICON_DONE $job"; done
|
||||
for job in "${JOB_FAIL[@]:-}"; do echo " $ICON_ERROR $job"; done
|
||||
fi
|
||||
|
||||
echo ""
|
||||
|
||||
TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} ))
|
||||
|
||||
echo ""
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "$ICON_WARN Status: DRY RUN — no changes made"
|
||||
warn "DRY RUN — no changes made"
|
||||
elif [[ "$TOTAL_FAIL" -eq 0 ]]; then
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS ALL COMPLETE"
|
||||
notify "Weekly sync maintenance complete on $(hostname) — synced + updated (local=$WEEKLY_SYNC_UPDATES remote=$WEEKLY_SYNC_UPDATES_REMOTE)" "Weekly Maintenance" "normal"
|
||||
log "$ICON_DONE Status: all complete ✅ — ${#PASS[@]} share(s) synced, ${#JOB_PASS[@]} job(s) run"
|
||||
else
|
||||
echo "$ICON_ERROR Status: $TOTAL_FAIL failure(s) — check logs"
|
||||
notify "Weekly sync maintenance failed on $(hostname) — sync: ${#FAIL[@]} failed, jobs: ${#JOB_FAIL[@]} failed" "Weekly Maintenance" "warning"
|
||||
warn "Status: $TOTAL_FAIL failure(s)"
|
||||
notify "Weekly maintenance failed on $(hostname) ($MY_ID) — sync: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \
|
||||
"Weekly Maintenance" "warning"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ "$TOTAL_FAIL" -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
Reference in New Issue
Block a user