Varaverk/Rsync/media_seed.sh

#!/bin/bash
# ==============================================================================================
# ============================= Media Share Seed ===============================================
# ==============================================================================================
#
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# Pushes every share in this host's DAILY_SYNC_SHARES to the partner with
# Rsync/rsync.sh --seed. This is Phase 3 — run after a partnership onboard has
# completed, when the mirror's arr databases know about all the content but the
# disks behind them are empty. The partnership does not wait on it.
#
# ==============================================================================================
# WHY THIS IS ITS OWN SCRIPT
# ==============================================================================================
#
# This was Step 9d inside partnership_onboard.sh, running inline, and is now Phase 3 —
# reached only by partnership_onboard.sh --phase3-only. On HOST1 that
# is thirteen shares and roughly 28 TB, and DEFAULT_RSYNC_OPTS caps the transfer
# at --bwlimit=12500 (12.5 MB/s), so a first seed is a multi-week transfer.
#
# Inline, it held the onboard open for all of it — and everything after it in the
# script waited: the webhook listener, the master.conf push, service discovery,
# container grouping, and the HOST<n>_PHASE2_DONE flag that every status reader
# uses to decide the partnership is established. So the owner's card sat on
# "Waiting for HOST2 to install Varaverk and complete onboard" and the mirror's
# checklist sat on "1 required item left: Partnership" for as long as the copy
# took, while the partnership underneath them was already fully wired.
#
# It also meant the onboard job record stayed "running" for weeks, which the
# already-running guard in run_job.sh correctly reads as a reason to refuse every
# subsequent onboard invocation.
#
# Moving the data movement out gives it its own job id, its own log, and its own
# progress in the UI, and lets it be re-run or cancelled without touching onboard.
#
# ==============================================================================================
# OPERATIONAL MODEL
# ==============================================================================================
#
# One share at a time, in DAILY_SYNC_SHARES order, each handed to Rsync/rsync.sh --seed. The
# transfer itself, its bandwidth cap and its resume behaviour are all rsync.sh's — this script
# decides what to seed and in what order, never how.
#
# Sequential on purpose. The cap that matters is DEFAULT_RSYNC_OPTS' --bwlimit, which is a limit
# per rsync rather than per host; running shares in parallel would multiply it by the number of
# shares and saturate the link the rest of the ecosystem is still using.
#
# Refuses before it starts rather than partway. RSYNC_ENABLED must be true and DAILY_SYNC_SHARES
# must be non-empty, both checked up front — a multi-week transfer is the wrong place to
# discover that the global gate was off.
#
# Nothing waits on this. Phase 3 runs after the partnership is already established, so a seed
# that is still copying weeks later blocks no flag, no status card and no job record.
#
# ==============================================================================================
# DESIGN PRINCIPLES
# ==============================================================================================
#
# Resumable By Construction
#   Every share is a separate rsync.sh call, and rsync.sh runs --inplace --partial.
#   Interrupting this script costs the current file, not the current share, and a
#   re-run picks up where it stopped. Stopping it is cheap; that is deliberate.
#
# Two Gates, And They Mean Different Things
#   MEDIA_SEED_ENABLED (master.conf, Tier 2) switches off this transfer and only
#   this transfer — for a partner being filled from a physically moved disk, or
#   one that already holds the library. Off is a decision, so it exits 0.
#   RSYNC_ENABLED (Tier 1) switches off every rsync on the host. Reaching this
#   script with Tier 1 closed is a misconfiguration, so it exits 1.
#
# Gate Read From Disk
#   Both are read out of master.conf here rather than trusted from the sourced
#   environment. Phase 3 rewrites that file moments before dispatching
#   this script, and each rsync.sh below sources it fresh anyway. With Tier 1
#   closed rsync.sh moves nothing and still exits 0, so every share would be
#   counted as seeded — refuse once instead of reporting fourteen no-ops.
#
# One Failed Share Is Not A Failed Seed
#   A share whose backing disk is unmounted on the partner fails its own rsync and
#   is named in the summary. The remaining shares still run. Exit is non-zero only
#   when nothing at all was seeded.
#
# ==============================================================================================
# OPERATIONAL SAFEGUARDS
# ==============================================================================================
#
#   acquire_lock "skip"   — a second seed cannot run beside the first
#   Gate checks           — MEDIA_SEED_ENABLED then RSYNC_ENABLED, in that order
#   Empty list check      — refuses when DAILY_SYNC_SHARES is empty
#   Per-share accounting  — failures are listed by name, not summed away
#
#   An absent MEDIA_SEED_ENABLED reads as on, never as off.
#       The toggle was added after this script shipped, so a master.conf that has not been
#       through a conf_upgrade does not have the key. Defaulting an unset toggle to off would
#       silently stop seeding on every node that has not upgraded — a change in behaviour
#       delivered by a missing line, which is the hardest kind to notice.
#
# ==============================================================================================
# RUNTIME MODES
# ==============================================================================================
#
#   Partnership/partnership_onboard.sh --phase3-only   — the normal entry point: checks
#                                      Phase 2 is done, sets the gate posture, dispatches this
#
#   Rsync/media_seed.sh              — seed every DAILY_SYNC_SHARES entry directly
#   Rsync/media_seed.sh --dry-run    — pass --dry-run down to rsync.sh
#   Rsync/media_seed.sh --log        — verbose output
#
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
SCRIPTS_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
source "$SCRIPTS_ROOT/load_config.sh"
parse_args "$@"

if [[ "$EUID" -ne 0 ]]; then
    error "Must be run as root"
    exit 1
fi

acquire_lock "skip"
trap "_release_all_locks" EXIT

detect_hosts

START=$(date +%s)

echo ""
echo "━━━ $ICON_SYNC Media Share Seed — $MY_ID ($LOCAL_SERVER_NAME) — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
echo ""

# See "Gate Read From Disk" above. The trailing comment is cut before the value is compared:
# master.conf writes these as `RSYNC_ENABLED=true    # Tier 1 — global gate, overrides everything
# below`, so stopping at `cut -d= -f2` yields "true#Tier1—globalgate,…" and never matches.
_read_gate() {
    grep -m1 -E "^[[:space:]]*$1=" "$SCRIPTS_ROOT/Configurations/master.conf" 2>/dev/null \
        | cut -d= -f2- | cut -d'#' -f1 | tr -d '"'"'" | tr -d '[:space:]'
}

# Tier 2 first, because it is the more specific answer and the operator deserves to be told
# which switch stopped this. An unset MEDIA_SEED_ENABLED is treated as on: it was added after
# the seed already existed, so a conf that predates it must keep behaving the way it did.
_seed_gate=$(_read_gate MEDIA_SEED_ENABLED)
if [[ "$DRY_RUN" == false && -n "$_seed_gate" && "$_seed_gate" != "true" ]]; then
    warn "MEDIA_SEED_ENABLED is '$_seed_gate' — the media seed is switched off in master.conf"
    warn "Set it true there if the partner should be filled by rsync rather than by hand"
    exit 0
fi

_gate=$(_read_gate RSYNC_ENABLED)
if [[ "$DRY_RUN" == false && "$_gate" != "true" ]]; then
    error "RSYNC_ENABLED is '${_gate:-unset}' — rsync.sh would move nothing"
    error "Arm it in master.conf, then re-run this script"
    exit 1
fi

if [[ "${#DAILY_SYNC_SHARES[@]}" -eq 0 ]]; then
    error "DAILY_SYNC_SHARES is empty for $MY_ID — nothing to seed"
    error "Configure HOST${MY_ID: -1}_DAILY_SYNC_SHARES in host${MY_ID: -1}.conf"
    exit 1
fi

echo "  Shares:  ${#DAILY_SYNC_SHARES[@]}"
echo "  Note:    a first seed of a full media library runs for days — this is expected"
echo ""

_flags=(--seed)
[[ "$DRY_RUN" == true ]] && _flags+=(--dry-run)
[[ "$ENABLE_LOGGING" == true ]] && _flags+=(--log)

SEEDED=0
FAILED_SHARES=()

for _share in "${DAILY_SYNC_SHARES[@]}"; do
    echo ""
    echo "━━━ Seeding: $_share ━━━"
    if bash "$SCRIPTS_ROOT/Rsync/rsync.sh" "$_share" "${_flags[@]}"; then
        (( SEEDED++ )) || true
    else
        warn "Seed failed for $_share — re-run: Rsync/rsync.sh $_share --seed"
        FAILED_SHARES+=("$_share")
    fi
done
unset _share _flags

END=$(date +%s)
echo ""
echo "━━━━━ $ICON_SUMMARY MEDIA SEED SUMMARY ━━━━━"
echo "  Duration: $(format_duration $(( END - START )))"
echo "  Seeded:   ${SEEDED}/${#DAILY_SYNC_SHARES[@]} share(s)"
if [[ "${#FAILED_SHARES[@]}" -gt 0 ]]; then
    echo "  Failed:   ${FAILED_SHARES[*]}"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━"

if [[ "$SEEDED" -eq 0 ]]; then
    error "No shares seeded"
    exit 1
fi
if [[ "${#FAILED_SHARES[@]}" -gt 0 ]]; then
    warn "Phase 3 incomplete — the flag stays unset while any share is unseeded"
    exit 1
fi

# Phase 3 is complete only when every share landed. This is what the Partnership tab reads to
# stop offering ▶ Phase 3, so writing it on a partial run would retire the button over a partner
# still missing shares — and a partial seed is the normal outcome of the first attempt, because
# a disk unmounted on the far side fails its share and no other.
#
# Written here rather than by the dispatcher: the dispatcher returns in seconds and knows only
# that the job started. Weeks separate those two facts.
if [[ -n "${REMOTE_ID:-}" ]]; then
    set_state_var "$(platform_setup_db_path)" "${REMOTE_ID}_PHASE3_DONE" "true"
    platform_push_setup_state >/dev/null 2>&1 \
        || warn "Phase 3 flag written locally but not pushed to the partner"
    echo "Phase 3 complete — ${REMOTE_ID}_PHASE3_DONE=true ✅"
fi
exit 0