#!/bin/bash # ============================================================================================== # ============================= Media Share Seed =============================================== # ============================================================================================== # # PURPOSE # ───────────────────────────────────────────────────────────────────────────── # Pushes every share in this host's DAILY_SYNC_SHARES to the partner with # Rsync/rsync.sh --seed. This is Phase 3 — run after a partnership onboard has # completed, when the mirror's arr databases know about all the content but the # disks behind them are empty. The partnership does not wait on it. # # ============================================================================================== # WHY THIS IS ITS OWN SCRIPT # ============================================================================================== # # This was Step 9d inside partnership_onboard.sh, running inline, and is now Phase 3 — # reached only by partnership_onboard.sh --phase3-only. On HOST1 that # is thirteen shares and roughly 28 TB, and DEFAULT_RSYNC_OPTS caps the transfer # at --bwlimit=12500 (12.5 MB/s), so a first seed is a multi-week transfer. # # Inline, it held the onboard open for all of it — and everything after it in the # script waited: the webhook listener, the master.conf push, service discovery, # container grouping, and the HOST_PHASE2_DONE flag that every status reader # uses to decide the partnership is established. So the owner's card sat on # "Waiting for HOST2 to install Varaverk and complete onboard" and the mirror's # checklist sat on "1 required item left: Partnership" for as long as the copy # took, while the partnership underneath them was already fully wired. # # It also meant the onboard job record stayed "running" for weeks, which the # already-running guard in run_job.sh correctly reads as a reason to refuse every # subsequent onboard invocation. # # Moving the data movement out gives it its own job id, its own log, and its own # progress in the UI, and lets it be re-run or cancelled without touching onboard. # # ============================================================================================== # DESIGN PRINCIPLES # ============================================================================================== # # Resumable By Construction # Every share is a separate rsync.sh call, and rsync.sh runs --inplace --partial. # Interrupting this script costs the current file, not the current share, and a # re-run picks up where it stopped. Stopping it is cheap; that is deliberate. # # Two Gates, And They Mean Different Things # MEDIA_SEED_ENABLED (master.conf, Tier 2) switches off this transfer and only # this transfer — for a partner being filled from a physically moved disk, or # one that already holds the library. Off is a decision, so it exits 0. # RSYNC_ENABLED (Tier 1) switches off every rsync on the host. Reaching this # script with Tier 1 closed is a misconfiguration, so it exits 1. # # Gate Read From Disk # Both are read out of master.conf here rather than trusted from the sourced # environment. Phase 3 rewrites that file moments before dispatching # this script, and each rsync.sh below sources it fresh anyway. With Tier 1 # closed rsync.sh moves nothing and still exits 0, so every share would be # counted as seeded — refuse once instead of reporting fourteen no-ops. # # One Failed Share Is Not A Failed Seed # A share whose backing disk is unmounted on the partner fails its own rsync and # is named in the summary. The remaining shares still run. Exit is non-zero only # when nothing at all was seeded. # # ============================================================================================== # OPERATIONAL SAFEGUARDS # ============================================================================================== # # acquire_lock "skip" — a second seed cannot run beside the first # Gate checks — MEDIA_SEED_ENABLED then RSYNC_ENABLED, in that order # Empty list check — refuses when DAILY_SYNC_SHARES is empty # Per-share accounting — failures are listed by name, not summed away # # An absent MEDIA_SEED_ENABLED reads as on, never as off. # The toggle was added after this script shipped, so a master.conf that has not been # through a conf_upgrade does not have the key. Defaulting an unset toggle to off would # silently stop seeding on every node that has not upgraded — a change in behaviour # delivered by a missing line, which is the hardest kind to notice. # # ============================================================================================== # RUNTIME MODES # ============================================================================================== # # Partnership/partnership_onboard.sh --phase3-only — the normal entry point: checks # Phase 2 is done, sets the gate posture, dispatches this # # Rsync/media_seed.sh — seed every DAILY_SYNC_SHARES entry directly # Rsync/media_seed.sh --dry-run — pass --dry-run down to rsync.sh # Rsync/media_seed.sh --log — verbose output # # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" SCRIPTS_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" source "$SCRIPTS_ROOT/load_config.sh" parse_args "$@" if [[ "$EUID" -ne 0 ]]; then error "Must be run as root" exit 1 fi acquire_lock "skip" trap "_release_all_locks" EXIT detect_hosts START=$(date +%s) echo "" echo "━━━ $ICON_SYNC Media Share Seed — $MY_ID ($LOCAL_SERVER_NAME) — $(date '+%Y-%m-%d %H:%M:%S') ━━━" echo "" # See "Gate Read From Disk" above. The trailing comment is cut before the value is compared: # master.conf writes these as `RSYNC_ENABLED=true # Tier 1 — global gate, overrides everything # below`, so stopping at `cut -d= -f2` yields "true#Tier1—globalgate,…" and never matches. _read_gate() { grep -m1 -E "^[[:space:]]*$1=" "$SCRIPTS_ROOT/Configurations/master.conf" 2>/dev/null \ | cut -d= -f2- | cut -d'#' -f1 | tr -d '"'"'" | tr -d '[:space:]' } # Tier 2 first, because it is the more specific answer and the operator deserves to be told # which switch stopped this. An unset MEDIA_SEED_ENABLED is treated as on: it was added after # the seed already existed, so a conf that predates it must keep behaving the way it did. _seed_gate=$(_read_gate MEDIA_SEED_ENABLED) if [[ "$DRY_RUN" == false && -n "$_seed_gate" && "$_seed_gate" != "true" ]]; then warn "MEDIA_SEED_ENABLED is '$_seed_gate' — the media seed is switched off in master.conf" warn "Set it true there if the partner should be filled by rsync rather than by hand" exit 0 fi _gate=$(_read_gate RSYNC_ENABLED) if [[ "$DRY_RUN" == false && "$_gate" != "true" ]]; then error "RSYNC_ENABLED is '${_gate:-unset}' — rsync.sh would move nothing" error "Arm it in master.conf, then re-run this script" exit 1 fi if [[ "${#DAILY_SYNC_SHARES[@]}" -eq 0 ]]; then error "DAILY_SYNC_SHARES is empty for $MY_ID — nothing to seed" error "Configure HOST${MY_ID: -1}_DAILY_SYNC_SHARES in host${MY_ID: -1}.conf" exit 1 fi echo " Shares: ${#DAILY_SYNC_SHARES[@]}" echo " Note: a first seed of a full media library runs for days — this is expected" echo "" _flags=(--seed) [[ "$DRY_RUN" == true ]] && _flags+=(--dry-run) [[ "$ENABLE_LOGGING" == true ]] && _flags+=(--log) SEEDED=0 FAILED_SHARES=() for _share in "${DAILY_SYNC_SHARES[@]}"; do echo "" echo "━━━ Seeding: $_share ━━━" if bash "$SCRIPTS_ROOT/Rsync/rsync.sh" "$_share" "${_flags[@]}"; then (( SEEDED++ )) || true else warn "Seed failed for $_share — re-run: Rsync/rsync.sh $_share --seed" FAILED_SHARES+=("$_share") fi done unset _share _flags END=$(date +%s) echo "" echo "━━━━━ $ICON_SUMMARY MEDIA SEED SUMMARY ━━━━━" echo " Duration: $(format_duration $(( END - START )))" echo " Seeded: ${SEEDED}/${#DAILY_SYNC_SHARES[@]} share(s)" if [[ "${#FAILED_SHARES[@]}" -gt 0 ]]; then echo " Failed: ${FAILED_SHARES[*]}" fi echo "━━━━━━━━━━━━━━━━━━━━━━━" if [[ "$SEEDED" -eq 0 ]]; then error "No shares seeded" exit 1 fi if [[ "${#FAILED_SHARES[@]}" -gt 0 ]]; then warn "Phase 3 incomplete — the flag stays unset while any share is unseeded" exit 1 fi # Phase 3 is complete only when every share landed. This is what the Partnership tab reads to # stop offering ▶ Phase 3, so writing it on a partial run would retire the button over a partner # still missing shares — and a partial seed is the normal outcome of the first attempt, because # a disk unmounted on the far side fails its share and no other. # # Written here rather than by the dispatcher: the dispatcher returns in seconds and knows only # that the job started. Weeks separate those two facts. if [[ -n "${REMOTE_ID:-}" ]]; then set_state_var "$(platform_setup_db_path)" "${REMOTE_ID}_PHASE3_DONE" "true" platform_push_setup_state >/dev/null 2>&1 \ || warn "Phase 3 flag written locally but not pushed to the partner" echo "Phase 3 complete — ${REMOTE_ID}_PHASE3_DONE=true ✅" fi exit 0