Stop the media seed from holding the onboard, and therefore the partnership, open for weeks
A first seed is ~28 TB behind a 12.5 MB/s bwlimit, and it ran inline as Step 9d, so the phase-2 flag every status reader depends on was written only after it finished.
This commit is contained in:
Gmer4Lfe
@@ -34,16 +34,17 @@
|
||||
# Step 9b: Webhook setup — register download webhook in arrs on both servers
|
||||
# Step 9c: Arm sync gates — open RSYNC/CONF_SYNC/ARR_SYNC in master.conf, which the
|
||||
# template ships closed so a fresh node cannot sync early.
|
||||
# Ahead of the seed because the seed is itself an rsync and
|
||||
# Tier 1 stops every rsync, including that one
|
||||
# Step 9d: Media seed — rsync all DAILY_SYNC_SHARES to mirror (--seed)
|
||||
# prevents arrs treating every file as missing after bootstrap
|
||||
# Still ahead of the Step 13 seed, which is itself an rsync
|
||||
# and dies at the Tier 1 gate if this has not run
|
||||
# Step 9e: Webhook listener — start listener on mirror (runs continuously, no reboot needed)
|
||||
# Step 10: Conf push — push master.conf + setup state to all listed hosts
|
||||
# Step 11: Service discovery — conf_populate.sh on the mirror, last, once the stacks it
|
||||
# would discover are actually deployed there
|
||||
# Step 12: Container grouping — file our containers on the mirror under "<OwnerShort>-Fallback",
|
||||
# icon resolved here and passed over: the mirror has no Emby key
|
||||
# Step 13: Media seed — dispatch Rsync/media_seed.sh detached, after the phase flag
|
||||
# is written. It is a multi-week transfer and must never be
|
||||
# the thing standing between an onboard and its own summary
|
||||
#
|
||||
# ==============================================================================================
|
||||
# DESIGN PRINCIPLES
|
||||
@@ -159,7 +160,7 @@
|
||||
# Skip webhook registration in arrs (Step 9b)
|
||||
#
|
||||
# Partnership/partnership_onboard.sh --skip-media-seed
|
||||
# Skip initial media share rsync to mirror (Step 9d)
|
||||
# Skip dispatching the background media seed (Step 13)
|
||||
# Use when mirror already has files or you want to seed manually
|
||||
#
|
||||
# Partnership/partnership_onboard.sh --skip-webhook-listener
|
||||
@@ -489,8 +490,6 @@ SERVICES_FAILED=0
|
||||
ONBOARD_OK=false
|
||||
ARR_SYNC_OK=false
|
||||
WEBHOOK_SETUP_OK=false
|
||||
MEDIA_SEED_OK=false
|
||||
MEDIA_SEED_COUNT=0
|
||||
WEBHOOK_LISTENER_OK=false
|
||||
MASTER_PUSH_OK=false
|
||||
|
||||
@@ -931,57 +930,6 @@ else
|
||||
unset _master_conf
|
||||
fi
|
||||
|
||||
# ── Step 9d: Seed media shares on mirror ─────────────────────────────────────────────────────
|
||||
# arr_sync.sh bootstrapped the databases — mirror's arrs now know about all content but
|
||||
# have no files on disk. Without this rsync, every imported item looks missing and arrs
|
||||
# will immediately queue searches. --seed skips the empty-remote guard and does a clean push.
|
||||
echo ""
|
||||
echo "━━━ Step 9d — Media Share Seed ━━━"
|
||||
|
||||
# Read the gate back off disk rather than trusting the value sourced at startup: Step 9c has
|
||||
# just rewritten master.conf, and each rsync.sh below is a separate process that will source
|
||||
# the file fresh. A closed gate here means every share would exit 0 having moved nothing, so
|
||||
# say so once and skip, instead of counting fourteen no-ops as fourteen successes.
|
||||
#
|
||||
# The trailing comment is cut before the value is read. master.conf writes this gate as
|
||||
# RSYNC_ENABLED=true # Tier 1 — global gate, overrides everything below
|
||||
# so stopping at `cut -d= -f2` and squeezing whitespace yields "true#Tier1—globalgate,…",
|
||||
# which is not "true" — and the seed would then skip itself on every single run, including
|
||||
# the ones where the gate is open.
|
||||
_rsync_gate=$(grep -m1 -E '^[[:space:]]*RSYNC_ENABLED=' "$SCRIPTS_ROOT/Configurations/master.conf" 2>/dev/null \
|
||||
| cut -d= -f2- | cut -d'#' -f1 | tr -d '"'"'" | tr -d '[:space:]')
|
||||
|
||||
if [[ "$SKIP_MEDIA_SEED" == true ]]; then
|
||||
warn "Skipping (--skip-media-seed)"
|
||||
elif [[ "$DRY_RUN" == false && "$_rsync_gate" != "true" ]]; then
|
||||
warn "RSYNC_ENABLED is '$_rsync_gate' — skipping media seed, rsync.sh would move nothing"
|
||||
warn "Arm it and re-run: Rsync/rsync.sh <share> --seed"
|
||||
elif [[ "${#DAILY_SYNC_SHARES[@]}" -eq 0 ]]; then
|
||||
warn "DAILY_SYNC_SHARES empty for $MY_ID — skipping media seed"
|
||||
warn "Configure HOST${MY_ID: -1}_DAILY_SYNC_SHARES in host${MY_ID: -1}.conf and run Rsync/rsync.sh --seed manually"
|
||||
else
|
||||
echo " Seeding ${#DAILY_SYNC_SHARES[@]} share(s) to $MIRROR — this may take a while"
|
||||
_rsync_script="$SCRIPTS_ROOT/Rsync/rsync.sh"
|
||||
_seed_flags=(--seed)
|
||||
[[ "$DRY_RUN" == true ]] && _seed_flags+=(--dry-run)
|
||||
[[ "$ENABLE_LOGGING" == true ]] && _seed_flags+=(--log)
|
||||
for _share in "${DAILY_SYNC_SHARES[@]}"; do
|
||||
echo " Seeding: $_share"
|
||||
if bash "$_rsync_script" "$_share" "${_seed_flags[@]}"; then
|
||||
(( MEDIA_SEED_COUNT++ )) || true
|
||||
else
|
||||
warn " Seed failed for $_share — re-run: Rsync/rsync.sh $_share --seed"
|
||||
fi
|
||||
done
|
||||
if [[ "$MEDIA_SEED_COUNT" -gt 0 ]]; then
|
||||
echo "Media seed complete — ${MEDIA_SEED_COUNT}/${#DAILY_SYNC_SHARES[@]} share(s) ✅"
|
||||
MEDIA_SEED_OK=true
|
||||
else
|
||||
warn "Media seed: no shares completed — check errors above"
|
||||
fi
|
||||
unset _rsync_script _seed_flags _share
|
||||
fi
|
||||
|
||||
# ── Step 9e: Start webhook listener on mirror ─────────────────────────────────────────────────
|
||||
# Listener is in ARRAY_START_SCRIPTS so it starts on next boot, but the mirror's array is
|
||||
# already running — kick it now so events are captured immediately after onboard.
|
||||
@@ -1121,6 +1069,66 @@ fi
|
||||
# ── Write Phase 2 completion state ────────────────────────────────────────────────────────────
|
||||
[[ "$ONBOARD_OK" == true && "$DRY_RUN" == false ]] && write_onboard_phase "$MIRROR_ID" 2
|
||||
|
||||
# ── Step 13: Dispatch the media seed ──────────────────────────────────────────────────────────
|
||||
# Last, detached, and after the phase flag is already on disk — see the header of
|
||||
# Rsync/media_seed.sh for why. Briefly: on HOST1 this is thirteen shares and ~28 TB behind a
|
||||
# 12.5 MB/s bwlimit, so inline it held this script open for weeks. Everything below it waited,
|
||||
# including write_onboard_phase above, so the owner's card stayed on "Waiting for HOST2 to
|
||||
# install Varaverk" and the mirror's checklist on "1 required item left: Partnership" for the
|
||||
# entire transfer, describing a partnership that was already established. The onboard job
|
||||
# record also stayed "running", which run_job.sh's already-running guard reads as a reason to
|
||||
# refuse every later onboard.
|
||||
#
|
||||
# setsid, not nohup + &: the seed must survive this script exiting and must not share a process
|
||||
# group with it, or stopping the onboard takes the transfer down with it.
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Step 13 — Media Share Seed (dispatch) ━━━"
|
||||
|
||||
MEDIA_SEED_DISPATCHED=false
|
||||
_seed_job="Rsync/media_seed.sh"
|
||||
_seed_script="$SCRIPTS_ROOT/Rsync/media_seed.sh"
|
||||
_runner="$SCRIPTS_ROOT/Plugin/$PLATFORM/run_job.sh"
|
||||
|
||||
if [[ "$SKIP_MEDIA_SEED" == true ]]; then
|
||||
warn "Skipping (--skip-media-seed)"
|
||||
elif [[ "$ONBOARD_OK" == false ]]; then
|
||||
warn "Skipping — onboard did not complete"
|
||||
elif [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would dispatch $_seed_job"
|
||||
elif [[ ! -f "$_seed_script" || ! -f "$_runner" ]]; then
|
||||
warn "Seed runner not found — run manually: bash $_seed_script"
|
||||
else
|
||||
_seed_flags=()
|
||||
[[ "$ENABLE_LOGGING" == true ]] && _seed_flags+=(--log)
|
||||
setsid bash "$_runner" "$_seed_job" "$_seed_script" "${_seed_flags[@]}" \
|
||||
>/dev/null 2>&1 < /dev/null &
|
||||
disown 2>/dev/null || true
|
||||
|
||||
# Confirm the runner actually took the job rather than reporting the launch. run_job.sh
|
||||
# writes its stat file before exec'ing the script, so a fresh "running" record with a live
|
||||
# pid is the difference between dispatched and merely attempted.
|
||||
_seed_stat="/var/log/varaverk/Rsync/media_seed.json"
|
||||
for _i in 1 2 3 4 5 6 7 8 9 10; do
|
||||
if [[ -f "$_seed_stat" ]] \
|
||||
&& grep -q '"status":"running"' "$_seed_stat" 2>/dev/null \
|
||||
&& (( $(date +%s) - $(stat -c %Y "$_seed_stat") < 60 )); then
|
||||
MEDIA_SEED_DISPATCHED=true
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
unset _seed_stat _i _seed_flags
|
||||
|
||||
if [[ "$MEDIA_SEED_DISPATCHED" == true ]]; then
|
||||
echo "Seed running as job $_seed_job ✅"
|
||||
echo " ${#DAILY_SYNC_SHARES[@]} share(s) — watch it on the Scheduler tab, not here"
|
||||
echo " A first full seed runs for days. The partnership is live now; this only fills disks."
|
||||
else
|
||||
warn "Seed did not start — run manually: bash $_seed_script"
|
||||
fi
|
||||
fi
|
||||
unset _seed_job _seed_script _runner
|
||||
|
||||
# ── Summary ───────────────────────────────────────────────────────────────────────────────────
|
||||
END=$(date +%s)
|
||||
echo ""
|
||||
@@ -1145,13 +1153,13 @@ echo " Step 8 — Onboard: $(_ok "$ONBOARD_OK")"
|
||||
echo " Step 9 — Arr bootstrap: $( [[ "$SKIP_ARR_SYNC" == true || "$ONBOARD_OK" == false ]] && echo "skipped" || echo "$(_ok "$ARR_SYNC_OK")" )"
|
||||
echo " Step 9b — Webhook setup: $(_skip "$SKIP_WEBHOOK_SETUP" "$WEBHOOK_SETUP_OK")"
|
||||
echo " Step 9c — Arm sync gates: $( [[ "$SKIP_ARM" == true ]] && echo "skipped (--no-arm)" || { [[ "$ONBOARD_OK" == false ]] && echo "skipped" || echo "$(_ok "$ARM_OK")"; } )"
|
||||
echo " Step 9d — Media seed: $( [[ "$SKIP_MEDIA_SEED" == true ]] && echo "skipped" \
|
||||
|| { [[ "$DRY_RUN" == false && "${_rsync_gate:-}" != "true" ]] && echo "skipped (rsync gate closed)" \
|
||||
|| echo "${MEDIA_SEED_COUNT}/${#DAILY_SYNC_SHARES[@]} shares $(_ok "$MEDIA_SEED_OK")"; } )"
|
||||
echo " Step 9e — Webhook listener: $(_skip "$SKIP_WEBHOOK_LISTENER" "$WEBHOOK_LISTENER_OK")"
|
||||
echo " Step 10 — Conf push: $( [[ "$ONBOARD_OK" == false ]] && echo "skipped" || echo "$(_ok "$MASTER_PUSH_OK")" )"
|
||||
echo " Step 11 — Discovery: $( [[ "$POPULATE_OK" == skipped ]] && echo "skipped (unreachable)" || _ok "$POPULATE_OK" )"
|
||||
echo " Step 12 — Grouping: $( [[ "$FOLDER_OK" == skipped ]] && echo "skipped" || _ok "$FOLDER_OK" )"
|
||||
echo " Step 13 — Media seed: $( [[ "$SKIP_MEDIA_SEED" == true ]] && echo "skipped" \
|
||||
|| { [[ "$MEDIA_SEED_DISPATCHED" == true ]] && echo "dispatched — runs in background ✅" \
|
||||
|| echo "not dispatched ❌"; } )"
|
||||
echo ""
|
||||
|
||||
if [[ "$ONBOARD_OK" == true ]]; then
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
<?php
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
// PURPOSE
|
||||
// Start and stop the background media seed — Rsync/media_seed.sh, the multi-week first push
|
||||
// of every DAILY_SYNC_SHARES entry to the partner. Onboard Step 13 dispatches it once; this
|
||||
// endpoint is how the Partnership tab restarts it after a failure and stops it on demand.
|
||||
//
|
||||
// OPERATIONAL MODEL
|
||||
// The seed used to run inline inside partnership_onboard.sh, which meant the only control a
|
||||
// person had over ~28 TB of transfer was killing the onboard. It is its own job now, with its
|
||||
// own record at /var/log/varaverk/Rsync/media_seed.json, and these two actions are its whole
|
||||
// control surface. Progress is read by include/partnership.php and rendered on the partner's
|
||||
// card; nothing here reports progress.
|
||||
//
|
||||
// DESIGN PRINCIPLES
|
||||
// Stopping is safe by construction, and that is why a Stop button exists at all.
|
||||
// rsync.sh runs --inplace --partial, so a stopped seed loses the file in flight, not the
|
||||
// share, and a later start resumes rather than restarting. Anything else and the button
|
||||
// would be a trap on a transfer measured in weeks.
|
||||
//
|
||||
// The job id is a literal.
|
||||
// Nothing about which job runs comes from the request, so there is no path, no script
|
||||
// name and no argument for a caller to influence. This endpoint can start exactly one
|
||||
// script and signal exactly one recorded pid.
|
||||
//
|
||||
// OPERATIONAL SAFEGUARDS
|
||||
// POST only, for both actions. Unraid's CSRF guard is POST-only and jQuery-injected, so a
|
||||
// GET here would be both unguarded and, from native fetch(), silently unauthenticated.
|
||||
// See README-unraid.md.
|
||||
//
|
||||
// Stop signals the process group, not the pid.
|
||||
// run_job.sh is dispatched under setsid, so its pid is its process group leader and
|
||||
// kill -TERM -<pgid> reaches the rsync and the ssh beneath it. Signalling the pid alone
|
||||
// would reap the wrapper and leave the transfer running with no record pointing at it.
|
||||
//
|
||||
// The recorded pid is verified to still be that job before it is signalled.
|
||||
// A stale json from a run killed by a reboot can name a pid the kernel has since reused.
|
||||
// /proc/<pid>/cmdline is checked for the seed script's own path first, so at worst this
|
||||
// refuses to stop something; it cannot kill an unrelated process.
|
||||
//
|
||||
// Starting is delegated, not duplicated.
|
||||
// run_job.sh already refuses to start a job that is running, and media_seed.sh takes its
|
||||
// own lock. This endpoint does not re-implement either check — it dispatches and reports
|
||||
// what the record then says.
|
||||
//
|
||||
// REQUEST
|
||||
// POST action=start dispatch the seed detached
|
||||
// POST action=stop terminate a running seed
|
||||
//
|
||||
// RESPONSE
|
||||
// {"ok":true,"status":"running"} start: the job record went live within the wait window
|
||||
// {"ok":true,"stopped":true} stop: the process group was signalled
|
||||
// {"ok":false,"error":string} wrong method, unknown action, or nothing to act on
|
||||
//
|
||||
// DEPENDS ON
|
||||
// Rsync/media_seed.sh the job itself
|
||||
// Plugin/unraid/run_job.sh the wrapper that writes the record
|
||||
// include/config.php SCRIPTS_DIR
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
header('Content-Type: application/json');
|
||||
require_once dirname(__DIR__) . '/include/config.php';
|
||||
|
||||
if ($_SERVER['REQUEST_METHOD'] !== 'POST') {
|
||||
echo json_encode(['ok' => false, 'error' => 'POST only']);
|
||||
exit;
|
||||
}
|
||||
|
||||
$action = $_POST['action'] ?? '';
|
||||
$dir = rtrim(SCRIPTS_DIR, '/');
|
||||
$script = $dir . '/Rsync/media_seed.sh';
|
||||
$runner = $dir . '/Plugin/unraid/run_job.sh';
|
||||
$stat = '/var/log/varaverk/Rsync/media_seed.json';
|
||||
|
||||
if ($action === 'start') {
|
||||
if (!is_file($script) || !is_file($runner)) {
|
||||
echo json_encode(['ok' => false, 'error' => 'media_seed.sh not found on this host']);
|
||||
exit;
|
||||
}
|
||||
$cmd = 'setsid /bin/bash ' . escapeshellarg($runner)
|
||||
. ' ' . escapeshellarg('Rsync/media_seed.sh')
|
||||
. ' ' . escapeshellarg($script)
|
||||
. ' --manual >/dev/null 2>&1 </dev/null &';
|
||||
shell_exec($cmd);
|
||||
|
||||
// Report what the record says, not that the command was issued. run_job.sh writes its
|
||||
// stat file before running the script, so a live record is the difference between a seed
|
||||
// that started and one that was refused for already running or died on its gate check.
|
||||
for ($i = 0; $i < 10; $i++) {
|
||||
if (is_file($stat)) {
|
||||
$j = json_decode((string)file_get_contents($stat), true);
|
||||
if (is_array($j) && ($j['status'] ?? '') === 'running'
|
||||
&& time() - filemtime($stat) < 60) {
|
||||
echo json_encode(['ok' => true, 'status' => 'running']);
|
||||
exit;
|
||||
}
|
||||
}
|
||||
usleep(500000);
|
||||
}
|
||||
echo json_encode(['ok' => false, 'error' => 'Seed did not start — check Rsync/media_seed.log']);
|
||||
exit;
|
||||
}
|
||||
|
||||
if ($action === 'stop') {
|
||||
if (!is_file($stat)) {
|
||||
echo json_encode(['ok' => false, 'error' => 'No seed has been run on this host']);
|
||||
exit;
|
||||
}
|
||||
$j = json_decode((string)file_get_contents($stat), true);
|
||||
$pid = (int)($j['pid'] ?? 0);
|
||||
if (($j['status'] ?? '') !== 'running' || !$pid || !is_dir("/proc/$pid")) {
|
||||
echo json_encode(['ok' => false, 'error' => 'Seed is not running']);
|
||||
exit;
|
||||
}
|
||||
|
||||
// See "The recorded pid is verified" above — cmdline is NUL-separated, so the script path
|
||||
// is matched against the raw bytes rather than a split.
|
||||
$cmdline = @file_get_contents("/proc/$pid/cmdline") ?: '';
|
||||
if (strpos($cmdline, 'media_seed.sh') === false) {
|
||||
echo json_encode(['ok' => false, 'error' => 'Recorded PID is no longer the seed — record is stale']);
|
||||
exit;
|
||||
}
|
||||
|
||||
shell_exec('kill -TERM -' . $pid . ' 2>/dev/null');
|
||||
usleep(400000);
|
||||
if (is_dir("/proc/$pid")) shell_exec('kill -KILL -' . $pid . ' 2>/dev/null');
|
||||
|
||||
echo json_encode(['ok' => true, 'stopped' => true]);
|
||||
exit;
|
||||
}
|
||||
|
||||
echo json_encode(['ok' => false, 'error' => 'Unknown action']);
|
||||
@@ -230,6 +230,64 @@ function vv_pt_remote_system(string $ip, string $sshKey): array {
|
||||
];
|
||||
}
|
||||
|
||||
// ── Media seed progress ───────────────────────────────────────────────────────
|
||||
//
|
||||
// Onboard Step 13 dispatches Rsync/media_seed.sh detached, because a first seed of a full
|
||||
// media library is a multi-week transfer and used to hold the onboard — and therefore the
|
||||
// phase-2 flag, and therefore this whole page — open for the duration.
|
||||
//
|
||||
// Detaching it means nothing on screen would mention it at all unless something reads its
|
||||
// job record, which is what this does. The seed is a push from this host to the partner, so
|
||||
// it is rendered on the partner's card even though every byte of evidence for it is local.
|
||||
//
|
||||
// Returns [] when there has never been a seed. A stale "running" record whose pid is gone is
|
||||
// reported as stopped rather than running: a record is not a process.
|
||||
function vv_pt_media_seed(): array {
|
||||
$stat = '/var/log/varaverk/Rsync/media_seed.json';
|
||||
$log = '/var/log/varaverk/Rsync/media_seed.log';
|
||||
if (!is_file($stat)) return [];
|
||||
|
||||
$j = json_decode((string)file_get_contents($stat), true);
|
||||
if (!is_array($j)) return [];
|
||||
|
||||
$status = (string)($j['status'] ?? '');
|
||||
$pid = (int)($j['pid'] ?? 0);
|
||||
if ($status === 'running' && (!$pid || !is_dir("/proc/$pid"))) {
|
||||
$status = 'stopped';
|
||||
}
|
||||
|
||||
$share = '';
|
||||
$line = '';
|
||||
if (is_file($log)) {
|
||||
// rsync --info=progress2 rewrites one line with \r, so the tail is read as bytes and
|
||||
// split on both terminators — splitting on \n alone yields a single enormous "line"
|
||||
// holding every progress repaint of the current file.
|
||||
$fh = fopen($log, 'r');
|
||||
if ($fh) {
|
||||
fseek($fh, 0, SEEK_END);
|
||||
$size = ftell($fh);
|
||||
fseek($fh, max(0, $size - 65536));
|
||||
$tail = (string)fread($fh, 65536);
|
||||
fclose($fh);
|
||||
$parts = preg_split('/[\r\n]+/', $tail) ?: [];
|
||||
for ($i = count($parts) - 1; $i >= 0; $i--) {
|
||||
$p = trim($parts[$i]);
|
||||
if ($p === '') continue;
|
||||
if ($line === '') $line = $p;
|
||||
if (preg_match('/Seeding:\s*(\S+)/', $p, $m)) { $share = $m[1]; break; }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return array_filter([
|
||||
'status' => $status,
|
||||
'start' => isset($j['start']) ? (int)$j['start'] : null,
|
||||
'end' => isset($j['end']) ? (int)$j['end'] : null,
|
||||
'share' => $share,
|
||||
'line' => mb_substr($line, 0, 160),
|
||||
], fn($v) => $v !== null && $v !== '');
|
||||
}
|
||||
|
||||
// ── Per-node data ─────────────────────────────────────────────────────────────
|
||||
|
||||
function vv_pt_nodes(): array {
|
||||
@@ -362,6 +420,7 @@ function vv_pt_nodes(): array {
|
||||
'api_key_set' => $apiKeySet,
|
||||
'api_key_preview' => $apiKeyPreview,
|
||||
'metrics' => $metrics,
|
||||
'media_seed' => $isMe ? [] : vv_pt_media_seed(),
|
||||
];
|
||||
}
|
||||
return $nodes;
|
||||
|
||||
@@ -379,6 +379,30 @@ function vvPtHostChanged(el) {
|
||||
document.getElementById('vv-pt-hosts-save').style.display = any ? '' : 'none';
|
||||
}
|
||||
|
||||
// Stop the background media seed. Safe to press: rsync.sh runs --inplace --partial, so this
|
||||
// costs the file in flight and a later start resumes the share rather than restarting it.
|
||||
async function vvPtStopSeed(btn) {
|
||||
if (!await vvConfirm('Stop the media seed?\n\nThe transfer resumes from where it stopped when restarted.')) return;
|
||||
btn.disabled = true; btn.textContent = '⟳';
|
||||
try {
|
||||
const r = await fetch('/plugins/varaverk/api/media_seed.php', {
|
||||
method: 'POST',
|
||||
headers: {'Content-Type': 'application/x-www-form-urlencoded'},
|
||||
body: new URLSearchParams({
|
||||
csrf_token: typeof csrf_token !== 'undefined' ? csrf_token : '',
|
||||
action: 'stop'
|
||||
})
|
||||
});
|
||||
const d = await r.json();
|
||||
if (!d.ok) { vvAlert('Stop failed: ' + (d.error ?? 'Unknown error')); btn.disabled = false; btn.textContent = '■ Stop'; return; }
|
||||
btn.textContent = '✓';
|
||||
if (typeof vvPtLoad === 'function') vvPtLoad();
|
||||
} catch (e) {
|
||||
btn.disabled = false; btn.textContent = '■ Stop';
|
||||
vvAlert('Error: ' + e);
|
||||
}
|
||||
}
|
||||
|
||||
async function vvPtSaveHosts(btn) {
|
||||
const changedEls = document.querySelectorAll('.vv-pt-host-inp.changed');
|
||||
if (!changedEls.length) return;
|
||||
@@ -1123,6 +1147,31 @@ function _renderActions(nodes, cfg) {
|
||||
style="opacity:.4;font-size:10px;">🗑 Keys</button>
|
||||
</div>
|
||||
</div>`;
|
||||
|
||||
// Media seed strip. The seed is dispatched detached by onboard Step 13 and runs for
|
||||
// days, so without this the page would show a finished onboard and no sign that
|
||||
// terabytes are still moving underneath it.
|
||||
const ms = remote.media_seed || {};
|
||||
if (ms.status) {
|
||||
const seedRunning = ms.status === 'running';
|
||||
const seedColor = seedRunning ? '#ff9800'
|
||||
: ms.status === 'ok' ? '#4caf50'
|
||||
: ms.status === 'stopped' ? '#666' : '#f44336';
|
||||
const seedLabel = seedRunning ? 'Seeding'
|
||||
: ms.status === 'ok' ? 'Seed complete'
|
||||
: ms.status === 'stopped' ? 'Seed stopped' : 'Seed failed';
|
||||
html += `<div style="margin-top:9px;padding-top:8px;border-top:1px solid #181818;">
|
||||
<div style="display:flex;align-items:center;gap:7px;flex-wrap:wrap;">
|
||||
<span style="font-size:9px;font-weight:600;color:${seedColor};">${seedLabel}</span>
|
||||
${ms.share ? `<span style="font-size:9px;color:#444;font-family:monospace;">${vvEscHtml(ms.share)}</span>` : ''}
|
||||
${seedRunning ? `<button class="vv-pt-action-btn warn" onclick="vvPtStopSeed(this)"
|
||||
style="font-size:9px;opacity:.55;margin-left:auto;">■ Stop</button>` : ''}
|
||||
</div>
|
||||
${ms.line ? `<div style="margin-top:3px;font-size:9px;color:#2e2e2e;font-family:monospace;
|
||||
white-space:nowrap;overflow:hidden;text-overflow:ellipsis;"
|
||||
>${vvEscHtml(ms.line)}</div>` : ''}
|
||||
</div>`;
|
||||
}
|
||||
}
|
||||
|
||||
// ── Delete Keys panel ──────────────────────────────────────────────────
|
||||
|
||||
@@ -104,6 +104,12 @@ For full profile definitions see `Manual-Rsync.md`.
|
||||
| Script | Role | When It Runs |
|
||||
|--------|------|-------------|
|
||||
| `rsync.sh` | Core transfer engine — profile resolution, pre-flights, container management, transfer, bandwidth logging | Called by orchestrators; directly for manual syncs |
|
||||
| `media_seed.sh` | First-fill of a new partner — loops every `DAILY_SYNC_SHARES` entry through `rsync.sh --seed` | Dispatched detached by partnership onboard Step 13; restartable from the Partnership tab |
|
||||
|
||||
`media_seed.sh` is the one script here that is expected to run for weeks. A first seed of
|
||||
HOST1's library is ~28 TB against a 12.5 MB/s `--bwlimit`, which is why onboard dispatches it
|
||||
rather than waiting on it, and why the Partnership tab gives it a Stop button — `rsync.sh` runs
|
||||
`--inplace --partial`, so stopping costs the file in flight, not the share.
|
||||
|
||||
---
|
||||
|
||||
|
||||
Executable
+152
@@ -0,0 +1,152 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ============================= Media Share Seed ===============================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Pushes every share in this host's DAILY_SYNC_SHARES to the partner with
|
||||
# Rsync/rsync.sh --seed. Run once after a partnership onboard, when the mirror's
|
||||
# arr databases know about all the content but the disks behind them are empty.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# WHY THIS IS ITS OWN SCRIPT
|
||||
# ==============================================================================================
|
||||
#
|
||||
# This was Step 9d inside partnership_onboard.sh, running inline. On HOST1 that
|
||||
# is thirteen shares and roughly 28 TB, and DEFAULT_RSYNC_OPTS caps the transfer
|
||||
# at --bwlimit=12500 (12.5 MB/s), so a first seed is a multi-week transfer.
|
||||
#
|
||||
# Inline, it held the onboard open for all of it — and everything after it in the
|
||||
# script waited: the webhook listener, the master.conf push, service discovery,
|
||||
# container grouping, and the HOST<n>_PHASE2_DONE flag that every status reader
|
||||
# uses to decide the partnership is established. So the owner's card sat on
|
||||
# "Waiting for HOST2 to install Varaverk and complete onboard" and the mirror's
|
||||
# checklist sat on "1 required item left: Partnership" for as long as the copy
|
||||
# took, while the partnership underneath them was already fully wired.
|
||||
#
|
||||
# It also meant the onboard job record stayed "running" for weeks, which the
|
||||
# already-running guard in run_job.sh correctly reads as a reason to refuse every
|
||||
# subsequent onboard invocation.
|
||||
#
|
||||
# Moving the data movement out gives it its own job id, its own log, and its own
|
||||
# progress in the UI, and lets it be re-run or cancelled without touching onboard.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# DESIGN PRINCIPLES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Resumable By Construction
|
||||
# Every share is a separate rsync.sh call, and rsync.sh runs --inplace --partial.
|
||||
# Interrupting this script costs the current file, not the current share, and a
|
||||
# re-run picks up where it stopped. Stopping it is cheap; that is deliberate.
|
||||
#
|
||||
# Gate Read From Disk
|
||||
# RSYNC_ENABLED is read out of master.conf here rather than trusted from the
|
||||
# sourced environment. Onboard's Step 9c rewrites that file moments before
|
||||
# dispatching this script, and each rsync.sh below sources it fresh anyway.
|
||||
# With the gate closed rsync.sh moves nothing and still exits 0, so every share
|
||||
# would be counted as seeded — refuse once instead of reporting fourteen no-ops.
|
||||
#
|
||||
# One Failed Share Is Not A Failed Seed
|
||||
# A share whose backing disk is unmounted on the partner fails its own rsync and
|
||||
# is named in the summary. The remaining shares still run. Exit is non-zero only
|
||||
# when nothing at all was seeded.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# acquire_lock "skip" — a second seed cannot run beside the first
|
||||
# Gate check — refuses when RSYNC_ENABLED is not true
|
||||
# Empty list check — refuses when DAILY_SYNC_SHARES is empty
|
||||
# Per-share accounting — failures are listed by name, not summed away
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Rsync/media_seed.sh — seed every DAILY_SYNC_SHARES entry
|
||||
# Rsync/media_seed.sh --dry-run — pass --dry-run down to rsync.sh
|
||||
# Rsync/media_seed.sh --log — verbose output
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
SCRIPTS_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
source "$SCRIPTS_ROOT/load_config.sh"
|
||||
parse_args "$@"
|
||||
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
acquire_lock "skip"
|
||||
trap "_release_all_locks" EXIT
|
||||
|
||||
detect_hosts
|
||||
|
||||
START=$(date +%s)
|
||||
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Media Share Seed — $MY_ID ($LOCAL_SERVER_NAME) — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo ""
|
||||
|
||||
# See "Gate Read From Disk" above. The trailing comment is cut before the value is compared:
|
||||
# master.conf writes this as `RSYNC_ENABLED=true # Tier 1 — global gate, overrides everything
|
||||
# below`, so stopping at `cut -d= -f2` yields "true#Tier1—globalgate,…" and never matches.
|
||||
_gate=$(grep -m1 -E '^[[:space:]]*RSYNC_ENABLED=' "$SCRIPTS_ROOT/Configurations/master.conf" 2>/dev/null \
|
||||
| cut -d= -f2- | cut -d'#' -f1 | tr -d '"'"'" | tr -d '[:space:]')
|
||||
|
||||
if [[ "$DRY_RUN" == false && "$_gate" != "true" ]]; then
|
||||
error "RSYNC_ENABLED is '${_gate:-unset}' — rsync.sh would move nothing"
|
||||
error "Arm it in master.conf, then re-run this script"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ "${#DAILY_SYNC_SHARES[@]}" -eq 0 ]]; then
|
||||
error "DAILY_SYNC_SHARES is empty for $MY_ID — nothing to seed"
|
||||
error "Configure HOST${MY_ID: -1}_DAILY_SYNC_SHARES in host${MY_ID: -1}.conf"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo " Shares: ${#DAILY_SYNC_SHARES[@]}"
|
||||
echo " Note: a first seed of a full media library runs for days — this is expected"
|
||||
echo ""
|
||||
|
||||
_flags=(--seed)
|
||||
[[ "$DRY_RUN" == true ]] && _flags+=(--dry-run)
|
||||
[[ "$ENABLE_LOGGING" == true ]] && _flags+=(--log)
|
||||
|
||||
SEEDED=0
|
||||
FAILED_SHARES=()
|
||||
|
||||
for _share in "${DAILY_SYNC_SHARES[@]}"; do
|
||||
echo ""
|
||||
echo "━━━ Seeding: $_share ━━━"
|
||||
if bash "$SCRIPTS_ROOT/Rsync/rsync.sh" "$_share" "${_flags[@]}"; then
|
||||
(( SEEDED++ )) || true
|
||||
else
|
||||
warn "Seed failed for $_share — re-run: Rsync/rsync.sh $_share --seed"
|
||||
FAILED_SHARES+=("$_share")
|
||||
fi
|
||||
done
|
||||
unset _share _flags
|
||||
|
||||
END=$(date +%s)
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY MEDIA SEED SUMMARY ━━━━━"
|
||||
echo " Duration: $(format_duration $(( END - START )))"
|
||||
echo " Seeded: ${SEEDED}/${#DAILY_SYNC_SHARES[@]} share(s)"
|
||||
if [[ "${#FAILED_SHARES[@]}" -gt 0 ]]; then
|
||||
echo " Failed: ${FAILED_SHARES[*]}"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
if [[ "$SEEDED" -eq 0 ]]; then
|
||||
error "No shares seeded"
|
||||
exit 1
|
||||
fi
|
||||
[[ "${#FAILED_SHARES[@]}" -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
Reference in New Issue
Block a user