Stop the media seed from holding the onboard, and therefore the partnership, open for weeks
A first seed is ~28 TB behind a 12.5 MB/s bwlimit, and it ran inline as Step 9d, so the phase-2 flag every status reader depends on was written only after it finished.
This commit is contained in:
Gmer4Lfe
Executable
+152
@@ -0,0 +1,152 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ============================= Media Share Seed ===============================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Pushes every share in this host's DAILY_SYNC_SHARES to the partner with
|
||||
# Rsync/rsync.sh --seed. Run once after a partnership onboard, when the mirror's
|
||||
# arr databases know about all the content but the disks behind them are empty.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# WHY THIS IS ITS OWN SCRIPT
|
||||
# ==============================================================================================
|
||||
#
|
||||
# This was Step 9d inside partnership_onboard.sh, running inline. On HOST1 that
|
||||
# is thirteen shares and roughly 28 TB, and DEFAULT_RSYNC_OPTS caps the transfer
|
||||
# at --bwlimit=12500 (12.5 MB/s), so a first seed is a multi-week transfer.
|
||||
#
|
||||
# Inline, it held the onboard open for all of it — and everything after it in the
|
||||
# script waited: the webhook listener, the master.conf push, service discovery,
|
||||
# container grouping, and the HOST<n>_PHASE2_DONE flag that every status reader
|
||||
# uses to decide the partnership is established. So the owner's card sat on
|
||||
# "Waiting for HOST2 to install Varaverk and complete onboard" and the mirror's
|
||||
# checklist sat on "1 required item left: Partnership" for as long as the copy
|
||||
# took, while the partnership underneath them was already fully wired.
|
||||
#
|
||||
# It also meant the onboard job record stayed "running" for weeks, which the
|
||||
# already-running guard in run_job.sh correctly reads as a reason to refuse every
|
||||
# subsequent onboard invocation.
|
||||
#
|
||||
# Moving the data movement out gives it its own job id, its own log, and its own
|
||||
# progress in the UI, and lets it be re-run or cancelled without touching onboard.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# DESIGN PRINCIPLES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Resumable By Construction
|
||||
# Every share is a separate rsync.sh call, and rsync.sh runs --inplace --partial.
|
||||
# Interrupting this script costs the current file, not the current share, and a
|
||||
# re-run picks up where it stopped. Stopping it is cheap; that is deliberate.
|
||||
#
|
||||
# Gate Read From Disk
|
||||
# RSYNC_ENABLED is read out of master.conf here rather than trusted from the
|
||||
# sourced environment. Onboard's Step 9c rewrites that file moments before
|
||||
# dispatching this script, and each rsync.sh below sources it fresh anyway.
|
||||
# With the gate closed rsync.sh moves nothing and still exits 0, so every share
|
||||
# would be counted as seeded — refuse once instead of reporting fourteen no-ops.
|
||||
#
|
||||
# One Failed Share Is Not A Failed Seed
|
||||
# A share whose backing disk is unmounted on the partner fails its own rsync and
|
||||
# is named in the summary. The remaining shares still run. Exit is non-zero only
|
||||
# when nothing at all was seeded.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# acquire_lock "skip" — a second seed cannot run beside the first
|
||||
# Gate check — refuses when RSYNC_ENABLED is not true
|
||||
# Empty list check — refuses when DAILY_SYNC_SHARES is empty
|
||||
# Per-share accounting — failures are listed by name, not summed away
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Rsync/media_seed.sh — seed every DAILY_SYNC_SHARES entry
|
||||
# Rsync/media_seed.sh --dry-run — pass --dry-run down to rsync.sh
|
||||
# Rsync/media_seed.sh --log — verbose output
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
SCRIPTS_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
source "$SCRIPTS_ROOT/load_config.sh"
|
||||
parse_args "$@"
|
||||
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
acquire_lock "skip"
|
||||
trap "_release_all_locks" EXIT
|
||||
|
||||
detect_hosts
|
||||
|
||||
START=$(date +%s)
|
||||
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Media Share Seed — $MY_ID ($LOCAL_SERVER_NAME) — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo ""
|
||||
|
||||
# See "Gate Read From Disk" above. The trailing comment is cut before the value is compared:
|
||||
# master.conf writes this as `RSYNC_ENABLED=true # Tier 1 — global gate, overrides everything
|
||||
# below`, so stopping at `cut -d= -f2` yields "true#Tier1—globalgate,…" and never matches.
|
||||
_gate=$(grep -m1 -E '^[[:space:]]*RSYNC_ENABLED=' "$SCRIPTS_ROOT/Configurations/master.conf" 2>/dev/null \
|
||||
| cut -d= -f2- | cut -d'#' -f1 | tr -d '"'"'" | tr -d '[:space:]')
|
||||
|
||||
if [[ "$DRY_RUN" == false && "$_gate" != "true" ]]; then
|
||||
error "RSYNC_ENABLED is '${_gate:-unset}' — rsync.sh would move nothing"
|
||||
error "Arm it in master.conf, then re-run this script"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ "${#DAILY_SYNC_SHARES[@]}" -eq 0 ]]; then
|
||||
error "DAILY_SYNC_SHARES is empty for $MY_ID — nothing to seed"
|
||||
error "Configure HOST${MY_ID: -1}_DAILY_SYNC_SHARES in host${MY_ID: -1}.conf"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo " Shares: ${#DAILY_SYNC_SHARES[@]}"
|
||||
echo " Note: a first seed of a full media library runs for days — this is expected"
|
||||
echo ""
|
||||
|
||||
_flags=(--seed)
|
||||
[[ "$DRY_RUN" == true ]] && _flags+=(--dry-run)
|
||||
[[ "$ENABLE_LOGGING" == true ]] && _flags+=(--log)
|
||||
|
||||
SEEDED=0
|
||||
FAILED_SHARES=()
|
||||
|
||||
for _share in "${DAILY_SYNC_SHARES[@]}"; do
|
||||
echo ""
|
||||
echo "━━━ Seeding: $_share ━━━"
|
||||
if bash "$SCRIPTS_ROOT/Rsync/rsync.sh" "$_share" "${_flags[@]}"; then
|
||||
(( SEEDED++ )) || true
|
||||
else
|
||||
warn "Seed failed for $_share — re-run: Rsync/rsync.sh $_share --seed"
|
||||
FAILED_SHARES+=("$_share")
|
||||
fi
|
||||
done
|
||||
unset _share _flags
|
||||
|
||||
END=$(date +%s)
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY MEDIA SEED SUMMARY ━━━━━"
|
||||
echo " Duration: $(format_duration $(( END - START )))"
|
||||
echo " Seeded: ${SEEDED}/${#DAILY_SYNC_SHARES[@]} share(s)"
|
||||
if [[ "${#FAILED_SHARES[@]}" -gt 0 ]]; then
|
||||
echo " Failed: ${FAILED_SHARES[*]}"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
if [[ "$SEEDED" -eq 0 ]]; then
|
||||
error "No shares seeded"
|
||||
exit 1
|
||||
fi
|
||||
[[ "${#FAILED_SHARES[@]}" -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
Reference in New Issue
Block a user