From ad623353bc91ff3de915264cb0b065dd14f9578e Mon Sep 17 00:00:00 2001 From: Gmer4Lfe Date: Mon, 17 Aug 2026 07:40:00 -0400 Subject: [PATCH] Stop the media seed from holding the onboard, and therefore the partnership, open for weeks A first seed is ~28 TB behind a 12.5 MB/s bwlimit, and it ran inline as Step 9d, so the phase-2 flag every status reader depends on was written only after it finished. --- Partnership/partnership_onboard.sh | 130 +++++++++++----------- Plugin/unraid/api/media_seed.php | 131 ++++++++++++++++++++++ Plugin/unraid/include/partnership.php | 59 ++++++++++ Plugin/unraid/pages/partnership.php | 49 +++++++++ Rsync/README-Rsync.md | 6 + Rsync/media_seed.sh | 152 ++++++++++++++++++++++++++ 6 files changed, 466 insertions(+), 61 deletions(-) create mode 100644 Plugin/unraid/api/media_seed.php create mode 100755 Rsync/media_seed.sh diff --git a/Partnership/partnership_onboard.sh b/Partnership/partnership_onboard.sh index 8036364..36ee3a5 100755 --- a/Partnership/partnership_onboard.sh +++ b/Partnership/partnership_onboard.sh @@ -34,16 +34,17 @@ # Step 9b: Webhook setup — register download webhook in arrs on both servers # Step 9c: Arm sync gates — open RSYNC/CONF_SYNC/ARR_SYNC in master.conf, which the # template ships closed so a fresh node cannot sync early. -# Ahead of the seed because the seed is itself an rsync and -# Tier 1 stops every rsync, including that one -# Step 9d: Media seed — rsync all DAILY_SYNC_SHARES to mirror (--seed) -# prevents arrs treating every file as missing after bootstrap +# Still ahead of the Step 13 seed, which is itself an rsync +# and dies at the Tier 1 gate if this has not run # Step 9e: Webhook listener — start listener on mirror (runs continuously, no reboot needed) # Step 10: Conf push — push master.conf + setup state to all listed hosts # Step 11: Service discovery — conf_populate.sh on the mirror, last, once the stacks it # would discover are actually deployed there # Step 12: Container grouping — file our containers on the mirror under "-Fallback", # icon resolved here and passed over: the mirror has no Emby key +# Step 13: Media seed — dispatch Rsync/media_seed.sh detached, after the phase flag +# is written. It is a multi-week transfer and must never be +# the thing standing between an onboard and its own summary # # ============================================================================================== # DESIGN PRINCIPLES @@ -159,7 +160,7 @@ # Skip webhook registration in arrs (Step 9b) # # Partnership/partnership_onboard.sh --skip-media-seed -# Skip initial media share rsync to mirror (Step 9d) +# Skip dispatching the background media seed (Step 13) # Use when mirror already has files or you want to seed manually # # Partnership/partnership_onboard.sh --skip-webhook-listener @@ -489,8 +490,6 @@ SERVICES_FAILED=0 ONBOARD_OK=false ARR_SYNC_OK=false WEBHOOK_SETUP_OK=false -MEDIA_SEED_OK=false -MEDIA_SEED_COUNT=0 WEBHOOK_LISTENER_OK=false MASTER_PUSH_OK=false @@ -931,57 +930,6 @@ else unset _master_conf fi -# ── Step 9d: Seed media shares on mirror ───────────────────────────────────────────────────── -# arr_sync.sh bootstrapped the databases — mirror's arrs now know about all content but -# have no files on disk. Without this rsync, every imported item looks missing and arrs -# will immediately queue searches. --seed skips the empty-remote guard and does a clean push. -echo "" -echo "━━━ Step 9d — Media Share Seed ━━━" - -# Read the gate back off disk rather than trusting the value sourced at startup: Step 9c has -# just rewritten master.conf, and each rsync.sh below is a separate process that will source -# the file fresh. A closed gate here means every share would exit 0 having moved nothing, so -# say so once and skip, instead of counting fourteen no-ops as fourteen successes. -# -# The trailing comment is cut before the value is read. master.conf writes this gate as -# RSYNC_ENABLED=true # Tier 1 — global gate, overrides everything below -# so stopping at `cut -d= -f2` and squeezing whitespace yields "true#Tier1—globalgate,…", -# which is not "true" — and the seed would then skip itself on every single run, including -# the ones where the gate is open. -_rsync_gate=$(grep -m1 -E '^[[:space:]]*RSYNC_ENABLED=' "$SCRIPTS_ROOT/Configurations/master.conf" 2>/dev/null \ - | cut -d= -f2- | cut -d'#' -f1 | tr -d '"'"'" | tr -d '[:space:]') - -if [[ "$SKIP_MEDIA_SEED" == true ]]; then - warn "Skipping (--skip-media-seed)" -elif [[ "$DRY_RUN" == false && "$_rsync_gate" != "true" ]]; then - warn "RSYNC_ENABLED is '$_rsync_gate' — skipping media seed, rsync.sh would move nothing" - warn "Arm it and re-run: Rsync/rsync.sh --seed" -elif [[ "${#DAILY_SYNC_SHARES[@]}" -eq 0 ]]; then - warn "DAILY_SYNC_SHARES empty for $MY_ID — skipping media seed" - warn "Configure HOST${MY_ID: -1}_DAILY_SYNC_SHARES in host${MY_ID: -1}.conf and run Rsync/rsync.sh --seed manually" -else - echo " Seeding ${#DAILY_SYNC_SHARES[@]} share(s) to $MIRROR — this may take a while" - _rsync_script="$SCRIPTS_ROOT/Rsync/rsync.sh" - _seed_flags=(--seed) - [[ "$DRY_RUN" == true ]] && _seed_flags+=(--dry-run) - [[ "$ENABLE_LOGGING" == true ]] && _seed_flags+=(--log) - for _share in "${DAILY_SYNC_SHARES[@]}"; do - echo " Seeding: $_share" - if bash "$_rsync_script" "$_share" "${_seed_flags[@]}"; then - (( MEDIA_SEED_COUNT++ )) || true - else - warn " Seed failed for $_share — re-run: Rsync/rsync.sh $_share --seed" - fi - done - if [[ "$MEDIA_SEED_COUNT" -gt 0 ]]; then - echo "Media seed complete — ${MEDIA_SEED_COUNT}/${#DAILY_SYNC_SHARES[@]} share(s) ✅" - MEDIA_SEED_OK=true - else - warn "Media seed: no shares completed — check errors above" - fi - unset _rsync_script _seed_flags _share -fi - # ── Step 9e: Start webhook listener on mirror ───────────────────────────────────────────────── # Listener is in ARRAY_START_SCRIPTS so it starts on next boot, but the mirror's array is # already running — kick it now so events are captured immediately after onboard. @@ -1121,6 +1069,66 @@ fi # ── Write Phase 2 completion state ──────────────────────────────────────────────────────────── [[ "$ONBOARD_OK" == true && "$DRY_RUN" == false ]] && write_onboard_phase "$MIRROR_ID" 2 +# ── Step 13: Dispatch the media seed ────────────────────────────────────────────────────────── +# Last, detached, and after the phase flag is already on disk — see the header of +# Rsync/media_seed.sh for why. Briefly: on HOST1 this is thirteen shares and ~28 TB behind a +# 12.5 MB/s bwlimit, so inline it held this script open for weeks. Everything below it waited, +# including write_onboard_phase above, so the owner's card stayed on "Waiting for HOST2 to +# install Varaverk" and the mirror's checklist on "1 required item left: Partnership" for the +# entire transfer, describing a partnership that was already established. The onboard job +# record also stayed "running", which run_job.sh's already-running guard reads as a reason to +# refuse every later onboard. +# +# setsid, not nohup + &: the seed must survive this script exiting and must not share a process +# group with it, or stopping the onboard takes the transfer down with it. +echo "" +echo "━━━ $ICON_SYNC Step 13 — Media Share Seed (dispatch) ━━━" + +MEDIA_SEED_DISPATCHED=false +_seed_job="Rsync/media_seed.sh" +_seed_script="$SCRIPTS_ROOT/Rsync/media_seed.sh" +_runner="$SCRIPTS_ROOT/Plugin/$PLATFORM/run_job.sh" + +if [[ "$SKIP_MEDIA_SEED" == true ]]; then + warn "Skipping (--skip-media-seed)" +elif [[ "$ONBOARD_OK" == false ]]; then + warn "Skipping — onboard did not complete" +elif [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would dispatch $_seed_job" +elif [[ ! -f "$_seed_script" || ! -f "$_runner" ]]; then + warn "Seed runner not found — run manually: bash $_seed_script" +else + _seed_flags=() + [[ "$ENABLE_LOGGING" == true ]] && _seed_flags+=(--log) + setsid bash "$_runner" "$_seed_job" "$_seed_script" "${_seed_flags[@]}" \ + >/dev/null 2>&1 < /dev/null & + disown 2>/dev/null || true + + # Confirm the runner actually took the job rather than reporting the launch. run_job.sh + # writes its stat file before exec'ing the script, so a fresh "running" record with a live + # pid is the difference between dispatched and merely attempted. + _seed_stat="/var/log/varaverk/Rsync/media_seed.json" + for _i in 1 2 3 4 5 6 7 8 9 10; do + if [[ -f "$_seed_stat" ]] \ + && grep -q '"status":"running"' "$_seed_stat" 2>/dev/null \ + && (( $(date +%s) - $(stat -c %Y "$_seed_stat") < 60 )); then + MEDIA_SEED_DISPATCHED=true + break + fi + sleep 1 + done + unset _seed_stat _i _seed_flags + + if [[ "$MEDIA_SEED_DISPATCHED" == true ]]; then + echo "Seed running as job $_seed_job ✅" + echo " ${#DAILY_SYNC_SHARES[@]} share(s) — watch it on the Scheduler tab, not here" + echo " A first full seed runs for days. The partnership is live now; this only fills disks." + else + warn "Seed did not start — run manually: bash $_seed_script" + fi +fi +unset _seed_job _seed_script _runner + # ── Summary ─────────────────────────────────────────────────────────────────────────────────── END=$(date +%s) echo "" @@ -1145,13 +1153,13 @@ echo " Step 8 — Onboard: $(_ok "$ONBOARD_OK")" echo " Step 9 — Arr bootstrap: $( [[ "$SKIP_ARR_SYNC" == true || "$ONBOARD_OK" == false ]] && echo "skipped" || echo "$(_ok "$ARR_SYNC_OK")" )" echo " Step 9b — Webhook setup: $(_skip "$SKIP_WEBHOOK_SETUP" "$WEBHOOK_SETUP_OK")" echo " Step 9c — Arm sync gates: $( [[ "$SKIP_ARM" == true ]] && echo "skipped (--no-arm)" || { [[ "$ONBOARD_OK" == false ]] && echo "skipped" || echo "$(_ok "$ARM_OK")"; } )" -echo " Step 9d — Media seed: $( [[ "$SKIP_MEDIA_SEED" == true ]] && echo "skipped" \ - || { [[ "$DRY_RUN" == false && "${_rsync_gate:-}" != "true" ]] && echo "skipped (rsync gate closed)" \ - || echo "${MEDIA_SEED_COUNT}/${#DAILY_SYNC_SHARES[@]} shares $(_ok "$MEDIA_SEED_OK")"; } )" echo " Step 9e — Webhook listener: $(_skip "$SKIP_WEBHOOK_LISTENER" "$WEBHOOK_LISTENER_OK")" echo " Step 10 — Conf push: $( [[ "$ONBOARD_OK" == false ]] && echo "skipped" || echo "$(_ok "$MASTER_PUSH_OK")" )" echo " Step 11 — Discovery: $( [[ "$POPULATE_OK" == skipped ]] && echo "skipped (unreachable)" || _ok "$POPULATE_OK" )" echo " Step 12 — Grouping: $( [[ "$FOLDER_OK" == skipped ]] && echo "skipped" || _ok "$FOLDER_OK" )" +echo " Step 13 — Media seed: $( [[ "$SKIP_MEDIA_SEED" == true ]] && echo "skipped" \ + || { [[ "$MEDIA_SEED_DISPATCHED" == true ]] && echo "dispatched — runs in background ✅" \ + || echo "not dispatched ❌"; } )" echo "" if [[ "$ONBOARD_OK" == true ]]; then diff --git a/Plugin/unraid/api/media_seed.php b/Plugin/unraid/api/media_seed.php new file mode 100644 index 0000000..158f5e9 --- /dev/null +++ b/Plugin/unraid/api/media_seed.php @@ -0,0 +1,131 @@ + reaches the rsync and the ssh beneath it. Signalling the pid alone +// would reap the wrapper and leave the transfer running with no record pointing at it. +// +// The recorded pid is verified to still be that job before it is signalled. +// A stale json from a run killed by a reboot can name a pid the kernel has since reused. +// /proc//cmdline is checked for the seed script's own path first, so at worst this +// refuses to stop something; it cannot kill an unrelated process. +// +// Starting is delegated, not duplicated. +// run_job.sh already refuses to start a job that is running, and media_seed.sh takes its +// own lock. This endpoint does not re-implement either check — it dispatches and reports +// what the record then says. +// +// REQUEST +// POST action=start dispatch the seed detached +// POST action=stop terminate a running seed +// +// RESPONSE +// {"ok":true,"status":"running"} start: the job record went live within the wait window +// {"ok":true,"stopped":true} stop: the process group was signalled +// {"ok":false,"error":string} wrong method, unknown action, or nothing to act on +// +// DEPENDS ON +// Rsync/media_seed.sh the job itself +// Plugin/unraid/run_job.sh the wrapper that writes the record +// include/config.php SCRIPTS_DIR +// ═══════════════════════════════════════════════════════════════════════════════════════════════ +header('Content-Type: application/json'); +require_once dirname(__DIR__) . '/include/config.php'; + +if ($_SERVER['REQUEST_METHOD'] !== 'POST') { + echo json_encode(['ok' => false, 'error' => 'POST only']); + exit; +} + +$action = $_POST['action'] ?? ''; +$dir = rtrim(SCRIPTS_DIR, '/'); +$script = $dir . '/Rsync/media_seed.sh'; +$runner = $dir . '/Plugin/unraid/run_job.sh'; +$stat = '/var/log/varaverk/Rsync/media_seed.json'; + +if ($action === 'start') { + if (!is_file($script) || !is_file($runner)) { + echo json_encode(['ok' => false, 'error' => 'media_seed.sh not found on this host']); + exit; + } + $cmd = 'setsid /bin/bash ' . escapeshellarg($runner) + . ' ' . escapeshellarg('Rsync/media_seed.sh') + . ' ' . escapeshellarg($script) + . ' --manual >/dev/null 2>&1 true, 'status' => 'running']); + exit; + } + } + usleep(500000); + } + echo json_encode(['ok' => false, 'error' => 'Seed did not start — check Rsync/media_seed.log']); + exit; +} + +if ($action === 'stop') { + if (!is_file($stat)) { + echo json_encode(['ok' => false, 'error' => 'No seed has been run on this host']); + exit; + } + $j = json_decode((string)file_get_contents($stat), true); + $pid = (int)($j['pid'] ?? 0); + if (($j['status'] ?? '') !== 'running' || !$pid || !is_dir("/proc/$pid")) { + echo json_encode(['ok' => false, 'error' => 'Seed is not running']); + exit; + } + + // See "The recorded pid is verified" above — cmdline is NUL-separated, so the script path + // is matched against the raw bytes rather than a split. + $cmdline = @file_get_contents("/proc/$pid/cmdline") ?: ''; + if (strpos($cmdline, 'media_seed.sh') === false) { + echo json_encode(['ok' => false, 'error' => 'Recorded PID is no longer the seed — record is stale']); + exit; + } + + shell_exec('kill -TERM -' . $pid . ' 2>/dev/null'); + usleep(400000); + if (is_dir("/proc/$pid")) shell_exec('kill -KILL -' . $pid . ' 2>/dev/null'); + + echo json_encode(['ok' => true, 'stopped' => true]); + exit; +} + +echo json_encode(['ok' => false, 'error' => 'Unknown action']); diff --git a/Plugin/unraid/include/partnership.php b/Plugin/unraid/include/partnership.php index 30f6f91..cc33d00 100644 --- a/Plugin/unraid/include/partnership.php +++ b/Plugin/unraid/include/partnership.php @@ -230,6 +230,64 @@ function vv_pt_remote_system(string $ip, string $sshKey): array { ]; } +// ── Media seed progress ─────────────────────────────────────────────────────── +// +// Onboard Step 13 dispatches Rsync/media_seed.sh detached, because a first seed of a full +// media library is a multi-week transfer and used to hold the onboard — and therefore the +// phase-2 flag, and therefore this whole page — open for the duration. +// +// Detaching it means nothing on screen would mention it at all unless something reads its +// job record, which is what this does. The seed is a push from this host to the partner, so +// it is rendered on the partner's card even though every byte of evidence for it is local. +// +// Returns [] when there has never been a seed. A stale "running" record whose pid is gone is +// reported as stopped rather than running: a record is not a process. +function vv_pt_media_seed(): array { + $stat = '/var/log/varaverk/Rsync/media_seed.json'; + $log = '/var/log/varaverk/Rsync/media_seed.log'; + if (!is_file($stat)) return []; + + $j = json_decode((string)file_get_contents($stat), true); + if (!is_array($j)) return []; + + $status = (string)($j['status'] ?? ''); + $pid = (int)($j['pid'] ?? 0); + if ($status === 'running' && (!$pid || !is_dir("/proc/$pid"))) { + $status = 'stopped'; + } + + $share = ''; + $line = ''; + if (is_file($log)) { + // rsync --info=progress2 rewrites one line with \r, so the tail is read as bytes and + // split on both terminators — splitting on \n alone yields a single enormous "line" + // holding every progress repaint of the current file. + $fh = fopen($log, 'r'); + if ($fh) { + fseek($fh, 0, SEEK_END); + $size = ftell($fh); + fseek($fh, max(0, $size - 65536)); + $tail = (string)fread($fh, 65536); + fclose($fh); + $parts = preg_split('/[\r\n]+/', $tail) ?: []; + for ($i = count($parts) - 1; $i >= 0; $i--) { + $p = trim($parts[$i]); + if ($p === '') continue; + if ($line === '') $line = $p; + if (preg_match('/Seeding:\s*(\S+)/', $p, $m)) { $share = $m[1]; break; } + } + } + } + + return array_filter([ + 'status' => $status, + 'start' => isset($j['start']) ? (int)$j['start'] : null, + 'end' => isset($j['end']) ? (int)$j['end'] : null, + 'share' => $share, + 'line' => mb_substr($line, 0, 160), + ], fn($v) => $v !== null && $v !== ''); +} + // ── Per-node data ───────────────────────────────────────────────────────────── function vv_pt_nodes(): array { @@ -362,6 +420,7 @@ function vv_pt_nodes(): array { 'api_key_set' => $apiKeySet, 'api_key_preview' => $apiKeyPreview, 'metrics' => $metrics, + 'media_seed' => $isMe ? [] : vv_pt_media_seed(), ]; } return $nodes; diff --git a/Plugin/unraid/pages/partnership.php b/Plugin/unraid/pages/partnership.php index 9069f0c..2ddf7ba 100644 --- a/Plugin/unraid/pages/partnership.php +++ b/Plugin/unraid/pages/partnership.php @@ -379,6 +379,30 @@ function vvPtHostChanged(el) { document.getElementById('vv-pt-hosts-save').style.display = any ? '' : 'none'; } +// Stop the background media seed. Safe to press: rsync.sh runs --inplace --partial, so this +// costs the file in flight and a later start resumes the share rather than restarting it. +async function vvPtStopSeed(btn) { + if (!await vvConfirm('Stop the media seed?\n\nThe transfer resumes from where it stopped when restarted.')) return; + btn.disabled = true; btn.textContent = '⟳'; + try { + const r = await fetch('/plugins/varaverk/api/media_seed.php', { + method: 'POST', + headers: {'Content-Type': 'application/x-www-form-urlencoded'}, + body: new URLSearchParams({ + csrf_token: typeof csrf_token !== 'undefined' ? csrf_token : '', + action: 'stop' + }) + }); + const d = await r.json(); + if (!d.ok) { vvAlert('Stop failed: ' + (d.error ?? 'Unknown error')); btn.disabled = false; btn.textContent = '■ Stop'; return; } + btn.textContent = '✓'; + if (typeof vvPtLoad === 'function') vvPtLoad(); + } catch (e) { + btn.disabled = false; btn.textContent = '■ Stop'; + vvAlert('Error: ' + e); + } +} + async function vvPtSaveHosts(btn) { const changedEls = document.querySelectorAll('.vv-pt-host-inp.changed'); if (!changedEls.length) return; @@ -1123,6 +1147,31 @@ function _renderActions(nodes, cfg) { style="opacity:.4;font-size:10px;">🗑 Keys `; + + // Media seed strip. The seed is dispatched detached by onboard Step 13 and runs for + // days, so without this the page would show a finished onboard and no sign that + // terabytes are still moving underneath it. + const ms = remote.media_seed || {}; + if (ms.status) { + const seedRunning = ms.status === 'running'; + const seedColor = seedRunning ? '#ff9800' + : ms.status === 'ok' ? '#4caf50' + : ms.status === 'stopped' ? '#666' : '#f44336'; + const seedLabel = seedRunning ? 'Seeding' + : ms.status === 'ok' ? 'Seed complete' + : ms.status === 'stopped' ? 'Seed stopped' : 'Seed failed'; + html += `
+
+ ${seedLabel} + ${ms.share ? `${vvEscHtml(ms.share)}` : ''} + ${seedRunning ? `` : ''} +
+ ${ms.line ? `
${vvEscHtml(ms.line)}
` : ''} +
`; + } } // ── Delete Keys panel ────────────────────────────────────────────────── diff --git a/Rsync/README-Rsync.md b/Rsync/README-Rsync.md index 239f1b9..617d360 100644 --- a/Rsync/README-Rsync.md +++ b/Rsync/README-Rsync.md @@ -104,6 +104,12 @@ For full profile definitions see `Manual-Rsync.md`. | Script | Role | When It Runs | |--------|------|-------------| | `rsync.sh` | Core transfer engine — profile resolution, pre-flights, container management, transfer, bandwidth logging | Called by orchestrators; directly for manual syncs | +| `media_seed.sh` | First-fill of a new partner — loops every `DAILY_SYNC_SHARES` entry through `rsync.sh --seed` | Dispatched detached by partnership onboard Step 13; restartable from the Partnership tab | + +`media_seed.sh` is the one script here that is expected to run for weeks. A first seed of +HOST1's library is ~28 TB against a 12.5 MB/s `--bwlimit`, which is why onboard dispatches it +rather than waiting on it, and why the Partnership tab gives it a Stop button — `rsync.sh` runs +`--inplace --partial`, so stopping costs the file in flight, not the share. --- diff --git a/Rsync/media_seed.sh b/Rsync/media_seed.sh new file mode 100755 index 0000000..439aaef --- /dev/null +++ b/Rsync/media_seed.sh @@ -0,0 +1,152 @@ +#!/bin/bash +# ============================================================================================== +# ============================= Media Share Seed =============================================== +# ============================================================================================== +# +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Pushes every share in this host's DAILY_SYNC_SHARES to the partner with +# Rsync/rsync.sh --seed. Run once after a partnership onboard, when the mirror's +# arr databases know about all the content but the disks behind them are empty. +# +# ============================================================================================== +# WHY THIS IS ITS OWN SCRIPT +# ============================================================================================== +# +# This was Step 9d inside partnership_onboard.sh, running inline. On HOST1 that +# is thirteen shares and roughly 28 TB, and DEFAULT_RSYNC_OPTS caps the transfer +# at --bwlimit=12500 (12.5 MB/s), so a first seed is a multi-week transfer. +# +# Inline, it held the onboard open for all of it — and everything after it in the +# script waited: the webhook listener, the master.conf push, service discovery, +# container grouping, and the HOST_PHASE2_DONE flag that every status reader +# uses to decide the partnership is established. So the owner's card sat on +# "Waiting for HOST2 to install Varaverk and complete onboard" and the mirror's +# checklist sat on "1 required item left: Partnership" for as long as the copy +# took, while the partnership underneath them was already fully wired. +# +# It also meant the onboard job record stayed "running" for weeks, which the +# already-running guard in run_job.sh correctly reads as a reason to refuse every +# subsequent onboard invocation. +# +# Moving the data movement out gives it its own job id, its own log, and its own +# progress in the UI, and lets it be re-run or cancelled without touching onboard. +# +# ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# Resumable By Construction +# Every share is a separate rsync.sh call, and rsync.sh runs --inplace --partial. +# Interrupting this script costs the current file, not the current share, and a +# re-run picks up where it stopped. Stopping it is cheap; that is deliberate. +# +# Gate Read From Disk +# RSYNC_ENABLED is read out of master.conf here rather than trusted from the +# sourced environment. Onboard's Step 9c rewrites that file moments before +# dispatching this script, and each rsync.sh below sources it fresh anyway. +# With the gate closed rsync.sh moves nothing and still exits 0, so every share +# would be counted as seeded — refuse once instead of reporting fourteen no-ops. +# +# One Failed Share Is Not A Failed Seed +# A share whose backing disk is unmounted on the partner fails its own rsync and +# is named in the summary. The remaining shares still run. Exit is non-zero only +# when nothing at all was seeded. +# +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# acquire_lock "skip" — a second seed cannot run beside the first +# Gate check — refuses when RSYNC_ENABLED is not true +# Empty list check — refuses when DAILY_SYNC_SHARES is empty +# Per-share accounting — failures are listed by name, not summed away +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# Rsync/media_seed.sh — seed every DAILY_SYNC_SHARES entry +# Rsync/media_seed.sh --dry-run — pass --dry-run down to rsync.sh +# Rsync/media_seed.sh --log — verbose output +# +# ============================================================================================== + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SCRIPTS_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +source "$SCRIPTS_ROOT/load_config.sh" +parse_args "$@" + +if [[ "$EUID" -ne 0 ]]; then + error "Must be run as root" + exit 1 +fi + +acquire_lock "skip" +trap "_release_all_locks" EXIT + +detect_hosts + +START=$(date +%s) + +echo "" +echo "━━━ $ICON_SYNC Media Share Seed — $MY_ID ($LOCAL_SERVER_NAME) — $(date '+%Y-%m-%d %H:%M:%S') ━━━" +echo "" + +# See "Gate Read From Disk" above. The trailing comment is cut before the value is compared: +# master.conf writes this as `RSYNC_ENABLED=true # Tier 1 — global gate, overrides everything +# below`, so stopping at `cut -d= -f2` yields "true#Tier1—globalgate,…" and never matches. +_gate=$(grep -m1 -E '^[[:space:]]*RSYNC_ENABLED=' "$SCRIPTS_ROOT/Configurations/master.conf" 2>/dev/null \ + | cut -d= -f2- | cut -d'#' -f1 | tr -d '"'"'" | tr -d '[:space:]') + +if [[ "$DRY_RUN" == false && "$_gate" != "true" ]]; then + error "RSYNC_ENABLED is '${_gate:-unset}' — rsync.sh would move nothing" + error "Arm it in master.conf, then re-run this script" + exit 1 +fi + +if [[ "${#DAILY_SYNC_SHARES[@]}" -eq 0 ]]; then + error "DAILY_SYNC_SHARES is empty for $MY_ID — nothing to seed" + error "Configure HOST${MY_ID: -1}_DAILY_SYNC_SHARES in host${MY_ID: -1}.conf" + exit 1 +fi + +echo " Shares: ${#DAILY_SYNC_SHARES[@]}" +echo " Note: a first seed of a full media library runs for days — this is expected" +echo "" + +_flags=(--seed) +[[ "$DRY_RUN" == true ]] && _flags+=(--dry-run) +[[ "$ENABLE_LOGGING" == true ]] && _flags+=(--log) + +SEEDED=0 +FAILED_SHARES=() + +for _share in "${DAILY_SYNC_SHARES[@]}"; do + echo "" + echo "━━━ Seeding: $_share ━━━" + if bash "$SCRIPTS_ROOT/Rsync/rsync.sh" "$_share" "${_flags[@]}"; then + (( SEEDED++ )) || true + else + warn "Seed failed for $_share — re-run: Rsync/rsync.sh $_share --seed" + FAILED_SHARES+=("$_share") + fi +done +unset _share _flags + +END=$(date +%s) +echo "" +echo "━━━━━ $ICON_SUMMARY MEDIA SEED SUMMARY ━━━━━" +echo " Duration: $(format_duration $(( END - START )))" +echo " Seeded: ${SEEDED}/${#DAILY_SYNC_SHARES[@]} share(s)" +if [[ "${#FAILED_SHARES[@]}" -gt 0 ]]; then + echo " Failed: ${FAILED_SHARES[*]}" +fi +echo "━━━━━━━━━━━━━━━━━━━━━━━" + +if [[ "$SEEDED" -eq 0 ]]; then + error "No shares seeded" + exit 1 +fi +[[ "${#FAILED_SHARES[@]}" -gt 0 ]] && exit 1 +exit 0