Gmer4Lfe
369a9e6c19
- Rename unRAID_Essentials/ → System_Essentials/ (git detects as rename) - Add Plugin/unraid/adapter.sh: 13 platform_*() functions providing OS-agnostic API for storage health, service management, mover, user scripts, notifications, disk temps, and platform command validation - Update load_config.sh: detect PLATFORM (unraid/truenas/unknown), export SCRIPTS_DIR, auto-source Plugin/$PLATFORM/adapter.sh after common.sh - Wire all call sites: replace direct rc.d, pgrep/pkill, var.ini, dynamix.cfg, disks.ini, and validate_unraid_cmd calls with platform_*() functions across watchdogs, orchestrators, and System_Essentials scripts - Update all documentation: rename refs, update webgui escalation logic, add platform adapter section to Plugin README, update main README with portability vision and corrected self-healing stack description
404 lines
18 KiB
Bash
Executable File
404 lines
18 KiB
Bash
Executable File
#!/bin/bash
|
|
# ==============================================================================================
|
|
# ============================= Weekly Sync Maintenance ========================================
|
|
# ==============================================================================================
|
|
# Weekly maintenance window orchestrator — clean sync, container updates, weekly restarts.
|
|
# Schedule: 30 2 * * 0 (Sunday 2:30am — before Sunday 7am coffee report)
|
|
#
|
|
# ── EXECUTION ORDER ───────────────────────────────────────────────────────────────────────────
|
|
# 1. Stop local containers — Emby + auth stack stopped locally
|
|
# 2. Stop remote containers — Emby + auth stack stopped remotely via SSH
|
|
# 3. Pull updates locally — if WEEKLY_SYNC_UPDATES=true (zero extra downtime)
|
|
# 4. Pull updates remotely — if WEEKLY_SYNC_UPDATES_REMOTE=true
|
|
# 5. rsync WEEKLY_SYNC_SHARES — full clean mirror, containers stopped both sides
|
|
# 6. Start remote containers — correct order, delayed start respected
|
|
# 7. Start local containers — correct order, delayed start respected
|
|
# 8. WEEKLY_MAINTENANCE_SCRIPTS — weekly restarts etc. (docker_weekly_restart.sh)
|
|
# 9. docker_update.sh --remainder — update all containers not in daily or weekly sync window
|
|
#
|
|
# ── WHY WEEKLY NOT NIGHTLY FOR EMBY ──────────────────────────────────────────────────────────
|
|
# Emby builds a warm image cache on HOST2 throughout the week.
|
|
# Syncing nightly resets cache — cold loads every morning for users.
|
|
# Weekly sync: cache stays warm 6 days, resets Sunday night while users sleep.
|
|
# emby-fallback dirty sync covers watch states + library every 30min between weekly syncs.
|
|
#
|
|
# ── CONTAINER UPDATES ─────────────────────────────────────────────────────────────────────────
|
|
# Containers already stopped for sync — updates pull at zero extra downtime.
|
|
# Both servers start on identical image versions after the window completes.
|
|
# Toggle: WEEKLY_SYNC_UPDATES / WEEKLY_SYNC_UPDATES_REMOTE in master.conf
|
|
#
|
|
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
|
|
# detect_hosts() sets MY_ID — used in banner, summary, and notifications.
|
|
# WEEKLY_SYNC_SHARES and WEEKLY_MAINTENANCE_SCRIPTS configured in master.conf.
|
|
# Same script runs correctly on both servers.
|
|
#
|
|
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
|
|
# Root check — stop/start containers, rsync require root
|
|
# acquire_lock — prevents concurrent weekly windows
|
|
# check_connectivity — verifies remote before any remote operations
|
|
# check_remote_rootfs — aborts if remote rootfs nearly full
|
|
# DOCKER_TIMEOUT — all docker calls protected
|
|
# SSH_TIMEOUT — all SSH calls protected
|
|
# platform_require_cmd — notify validated before use
|
|
# Silent on success — runs weekly, only failures warrant notification
|
|
#
|
|
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
|
# WEEKLY_SYNC_SHARES — shares synced during window
|
|
# WEEKLY_MAINTENANCE_SCRIPTS — scripts run after sync
|
|
# WEEKLY_SYNC_UPDATES — toggle local container updates
|
|
# WEEKLY_SYNC_UPDATES_REMOTE — toggle remote container updates
|
|
# WEEKLY_RSYNC_ENABLED — enable/disable rsync section
|
|
#
|
|
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
|
# weekly_sync_maintenance.sh — normal run
|
|
# weekly_sync_maintenance.sh --dry-run — preview without stopping containers or syncing
|
|
# weekly_sync_maintenance.sh --log — verbose per-share/per-job output
|
|
# weekly_sync_maintenance.sh --status — show configuration and exit
|
|
# ==============================================================================================
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
source "$SCRIPT_DIR/../load_config.sh"
|
|
|
|
RSYNC_SCRIPT="$SCRIPT_DIR/../Rsync/rsync.sh"
|
|
SCRIPTS_ROOT="$SCRIPT_DIR/.."
|
|
|
|
parse_args "$@"
|
|
|
|
DOCKER_TIMEOUT=30 # container stop/start needs longer than normal
|
|
SSH_TIMEOUT=30 # remote pulls can be slow
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Setup ━━━
|
|
# ==============================================================================================
|
|
if [[ "$EUID" -ne 0 ]]; then
|
|
error "Must be run as root"
|
|
exit 1
|
|
fi
|
|
|
|
platform_require_cmd \
|
|
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
|
"" "" \
|
|
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
|
|
|
acquire_lock
|
|
|
|
if ! command -v docker &>/dev/null; then
|
|
error "Docker command not found"
|
|
exit 1
|
|
fi
|
|
|
|
detect_hosts
|
|
resolve_remote_ip
|
|
|
|
WINDOW_START=$(date +%s)
|
|
PASS=()
|
|
FAIL=()
|
|
JOB_PASS=()
|
|
JOB_FAIL=()
|
|
|
|
# Load container lists from profile config
|
|
read -r -a MAINTENANCE_CONTAINERS <<< \
|
|
"${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-}"
|
|
|
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be stopped, no sync, no updates"
|
|
|
|
# ── Helper — run a post-sync maintenance script ────────────────────────────────────────────────
|
|
run_job() {
|
|
local script_entry="$1"
|
|
local extra_dry=""
|
|
[[ "$DRY_RUN" == true ]] && extra_dry="--dry-run"
|
|
|
|
read -r -a script_args <<< "$script_entry"
|
|
local script_path="$SCRIPTS_ROOT/${script_args[0]}"
|
|
local script_name
|
|
script_name=$(basename "${script_args[0]}")
|
|
local extra_args=("${script_args[@]:1}")
|
|
|
|
if [[ ! -f "$script_path" ]]; then
|
|
error "$script_name — not found at $script_path"
|
|
JOB_FAIL+=("$script_name")
|
|
return 1
|
|
fi
|
|
|
|
log "Running: $script_name ${extra_args[*]}"
|
|
# shellcheck disable=SC2086
|
|
if bash "$script_path" "${extra_args[@]}" $extra_dry; then
|
|
log "$script_name — done ✅"
|
|
JOB_PASS+=("$script_name ${extra_args[*]}")
|
|
else
|
|
error "$script_name — failed (exit $?)"
|
|
JOB_FAIL+=("$script_name ${extra_args[*]}")
|
|
fi
|
|
}
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Status ━━━
|
|
# ==============================================================================================
|
|
if [[ "$SHOW_STATUS" == true ]]; then
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY WEEKLY SYNC STATUS ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_NET Remote: $REMOTE_ID ($REMOTE_SERVER_NAME)"
|
|
echo "$ICON_SYNC Rsync enabled: ${RSYNC_ENABLED:-true}"
|
|
echo "$ICON_SYNC Weekly enabled: ${WEEKLY_RSYNC_ENABLED:-false}"
|
|
echo "$ICON_GEAR Local updates: ${WEEKLY_SYNC_UPDATES:-false}"
|
|
echo "$ICON_GEAR Remote updates: ${WEEKLY_SYNC_UPDATES_REMOTE:-false}"
|
|
echo ""
|
|
echo "━━━ Weekly Sync Shares ━━━"
|
|
if [[ ${#WEEKLY_SYNC_SHARES[@]} -eq 0 ]]; then
|
|
warn " No WEEKLY_SYNC_SHARES configured"
|
|
else
|
|
for share in "${WEEKLY_SYNC_SHARES[@]}"; do
|
|
[[ -n "$share" ]] && echo " $ICON_SYNC $(basename "$share") ($share)"
|
|
done
|
|
fi
|
|
echo ""
|
|
echo "━━━ Weekly Maintenance Scripts ━━━"
|
|
if [[ ${#WEEKLY_MAINTENANCE_SCRIPTS[@]} -eq 0 ]]; then
|
|
echo " None configured"
|
|
else
|
|
for entry in "${WEEKLY_MAINTENANCE_SCRIPTS[@]}"; do
|
|
[[ -n "$entry" ]] && echo " $ICON_GEAR ${entry##*/}"
|
|
done
|
|
fi
|
|
echo ""
|
|
echo "━━━ Containers (from profile config) ━━━"
|
|
for c in "${MAINTENANCE_CONTAINERS[@]}"; do
|
|
[[ -n "$c" ]] && echo " $ICON_CONTAINERS $c"
|
|
done
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Pre-flight Checks ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_GEAR Weekly Sync Maintenance — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
|
echo ""
|
|
echo "━━━ $ICON_SHIELD Pre-flight ━━━"
|
|
|
|
check_connectivity
|
|
check_remote_rootfs
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Stop Containers ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_STOP $ICON_CONTAINERS Stop Containers ━━━"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — containers will not be stopped"
|
|
else
|
|
# Load container lists for stop functions
|
|
read -r -a CRITICAL_CONTAINER_NAMES <<< \
|
|
"${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-}"
|
|
read -r -a DELAYED_CONTAINERS <<< "${PROFILE_DELAYED_CONTAINERS[critical-data]:-}"
|
|
CONTAINER_DELAY="${PROFILE_CONTAINER_DELAY[critical-data]:-15}"
|
|
|
|
stop_local_containers
|
|
stop_containers
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Container Updates ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_GEAR Container Updates ━━━"
|
|
|
|
if [[ "$WEEKLY_SYNC_UPDATES" == true ]]; then
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
for c in "${MAINTENANCE_CONTAINERS[@]}"; do
|
|
[[ -n "$c" ]] && warn "DRY RUN — would pull: $c"
|
|
done
|
|
else
|
|
log "Pulling local container updates..."
|
|
for c in "${MAINTENANCE_CONTAINERS[@]}"; do
|
|
[[ -z "$c" ]] && continue
|
|
IMAGE=$(timeout "$DOCKER_TIMEOUT" docker inspect \
|
|
"$c" --format '{{.Config.Image}}' 2>/dev/null)
|
|
if [[ -z "$IMAGE" ]]; then
|
|
log "$c — not found locally, skipping update"
|
|
continue
|
|
fi
|
|
log "Pulling $IMAGE for $c..."
|
|
if docker pull "$IMAGE" >/dev/null 2>&1; then
|
|
log "$c — image updated ✅"
|
|
else
|
|
warn "$c — pull failed, will start on existing image"
|
|
fi
|
|
done
|
|
fi
|
|
else
|
|
echo "WEEKLY_SYNC_UPDATES=false — skipping local updates"
|
|
fi
|
|
|
|
if [[ "$WEEKLY_SYNC_UPDATES_REMOTE" == true ]]; then
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would pull updates on $REMOTE_SERVER_NAME"
|
|
else
|
|
log "Pulling remote container updates on $REMOTE_SERVER_NAME..."
|
|
for c in "${MAINTENANCE_CONTAINERS[@]}"; do
|
|
[[ -z "$c" ]] && continue
|
|
IMAGE=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
|
-o ConnectTimeout="$SSH_TIMEOUT" \
|
|
root@"$REMOTE_SERVER" \
|
|
"docker inspect $c --format '{{.Config.Image}}' 2>/dev/null" 2>/dev/null)
|
|
if [[ -z "$IMAGE" ]]; then
|
|
log "$c — not found on remote, skipping update"
|
|
continue
|
|
fi
|
|
log "Pulling $IMAGE for $c on $REMOTE_SERVER_NAME..."
|
|
if timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
|
-o ConnectTimeout="$SSH_TIMEOUT" \
|
|
root@"$REMOTE_SERVER" \
|
|
"docker pull $IMAGE" >/dev/null 2>&1; then
|
|
log "$c — remote image updated ✅"
|
|
else
|
|
warn "$c — remote pull failed, will start on existing image"
|
|
fi
|
|
done
|
|
fi
|
|
else
|
|
echo "WEEKLY_SYNC_UPDATES_REMOTE=false — skipping remote updates"
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Critical Shares Sync ━━━
|
|
# ==============================================================================================
|
|
SHARE_COUNT=${#WEEKLY_SYNC_SHARES[@]}
|
|
|
|
echo ""
|
|
echo "━━━ $ICON_SYNC Critical Shares Sync — $SHARE_COUNT share(s) ━━━"
|
|
|
|
SYNC_START=$(date +%s)
|
|
JOB_NUM=0
|
|
|
|
if ! check_rsync_enabled "WEEKLY"; then
|
|
warn "Weekly rsync disabled — skipping all $SHARE_COUNT sync job(s)"
|
|
warn "Proceeding to container start and maintenance scripts..."
|
|
elif [[ "$SHARE_COUNT" -eq 0 ]]; then
|
|
warn "No WEEKLY_SYNC_SHARES configured — skipping sync"
|
|
warn "Check WEEKLY_SYNC_SHARES in master.conf"
|
|
else
|
|
RSYNC_DRY=""
|
|
[[ "$DRY_RUN" == true ]] && RSYNC_DRY="--dry-run"
|
|
|
|
for JOB in "${WEEKLY_SYNC_SHARES[@]}"; do
|
|
(( JOB_NUM++ ))
|
|
JOB_NAME=$(basename "$JOB")
|
|
|
|
echo ""
|
|
echo "━━━ [$JOB_NUM/$SHARE_COUNT] $JOB_NAME ━━━"
|
|
|
|
JOB_START=$(date +%s)
|
|
bash "$RSYNC_SCRIPT" "$JOB" $RSYNC_DRY
|
|
EXIT_CODE=$?
|
|
JOB_DUR=$(format_duration $(( $(date +%s) - JOB_START )))
|
|
|
|
if [[ "$EXIT_CODE" -eq 0 ]]; then
|
|
PASS+=("$JOB_NAME")
|
|
log "$JOB_NAME — done in $JOB_DUR ✅"
|
|
else
|
|
FAIL+=("$JOB_NAME")
|
|
error "$JOB_NAME — failed after $JOB_DUR (exit $EXIT_CODE)"
|
|
fi
|
|
echo ""
|
|
done
|
|
fi
|
|
|
|
SYNC_END=$(date +%s)
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Start Containers ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_START $ICON_CONTAINERS Start Containers ━━━"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — containers will not be started"
|
|
else
|
|
start_containers
|
|
start_local_containers
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Post-sync Jobs ━━━
|
|
# ==============================================================================================
|
|
if [[ ${#WEEKLY_MAINTENANCE_SCRIPTS[@]} -gt 0 ]]; then
|
|
echo ""
|
|
echo "━━━ $ICON_GEAR Post-sync Jobs ━━━"
|
|
for script_entry in "${WEEKLY_MAINTENANCE_SCRIPTS[@]}"; do
|
|
[[ -z "$script_entry" ]] && continue
|
|
echo ""
|
|
run_job "$script_entry"
|
|
done
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Remainder Container Updates ━━━
|
|
# ==============================================================================================
|
|
# Updates all running containers not already covered by daily or the weekly sync window.
|
|
# Runs last — weekly sync-window containers are back up before this pulls their peers.
|
|
echo ""
|
|
echo "━━━ $ICON_CONTAINERS Remainder Container Updates ━━━"
|
|
|
|
DOCKER_UPDATE_SCRIPT="$SCRIPTS_ROOT/Docker_Essentials/docker_update.sh"
|
|
if [[ ! -f "$DOCKER_UPDATE_SCRIPT" ]]; then
|
|
warn "docker_update.sh not found — skipping remainder updates"
|
|
JOB_FAIL+=("docker_update.sh --remainder")
|
|
else
|
|
_remainder_args=("--remainder")
|
|
[[ "$DRY_RUN" == true ]] && _remainder_args+=("--dry-run")
|
|
if bash "$DOCKER_UPDATE_SCRIPT" "${_remainder_args[@]}"; then
|
|
echo "Remainder updates complete ✅"
|
|
JOB_PASS+=("docker_update.sh --remainder")
|
|
else
|
|
warn "Remainder updates completed with errors"
|
|
JOB_FAIL+=("docker_update.sh --remainder")
|
|
fi
|
|
unset _remainder_args
|
|
fi
|
|
|
|
WINDOW_END=$(date +%s)
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Summary ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY WEEKLY SYNC MAINTENANCE SUMMARY ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_TIME Window: $(date -d @"$WINDOW_START" '+%Y-%m-%d %H:%M:%S') → $(date -d @"$WINDOW_END" '+%H:%M:%S')"
|
|
echo "$ICON_TIME Duration: $(format_duration $(( WINDOW_END - WINDOW_START )))"
|
|
echo "$ICON_GEAR Updates: local=${WEEKLY_SYNC_UPDATES:-false} remote=${WEEKLY_SYNC_UPDATES_REMOTE:-false}"
|
|
echo ""
|
|
|
|
echo "$ICON_SYNC Sync jobs ($SHARE_COUNT):"
|
|
for job in "${PASS[@]}"; do echo " $ICON_DONE $job"; done
|
|
for job in "${FAIL[@]}"; do echo " $ICON_ERROR $job"; done
|
|
echo " Passed: ${#PASS[@]} Failed: ${#FAIL[@]}"
|
|
|
|
if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then
|
|
echo ""
|
|
echo "$ICON_GEAR Post-sync jobs:"
|
|
for job in "${JOB_PASS[@]}"; do echo " $ICON_DONE $job"; done
|
|
for job in "${JOB_FAIL[@]}"; do echo " $ICON_ERROR $job"; done
|
|
fi
|
|
|
|
TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} ))
|
|
|
|
echo ""
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — no changes made"
|
|
elif [[ "$TOTAL_FAIL" -eq 0 ]]; then
|
|
echo "$ICON_DONE Status: all complete ✅ — ${#PASS[@]} share(s) synced, ${#JOB_PASS[@]} job(s) run"
|
|
else
|
|
warn "Status: $TOTAL_FAIL failure(s)"
|
|
notify "Weekly maintenance failed on $(hostname) ($MY_ID) — sync: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \
|
|
"Weekly Maintenance" "warning"
|
|
fi
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
|
|
[[ "$TOTAL_FAIL" -gt 0 ]] && exit 1
|
|
exit 0 |