diff --git a/Media/play_state_sync.sh b/Media/play_state_sync.sh
index caf6b63..37141da 100755
--- a/Media/play_state_sync.sh
+++ b/Media/play_state_sync.sh
@@ -435,29 +435,19 @@ for lname in "${!USER_MAP[@]}"; do
if [[ -z "$_iid" ]]; then
_ptype="${_pkey%%:*}"
_pval="${_pkey##*:}"
- _search_field=""
-
case "$_ptype" in
imdb) _search_field="imdb.${_pval}" ;;
tmdb) _search_field="tmdb.${_pval##movie:}" ;;
tvdb)
- # AnyProviderIdEquals is unreliable for TVDB in Jellyfin 10.x —
- # returns the entire library. Search by season+episode instead,
- # then validate by ProviderIds.Tvdb.
- # pkey format: tvdb:ep:TVDB_EP_ID:sSEASONePEP
- _tvdb_ep_id="${_pkey#tvdb:ep:}"
- _tvdb_ep_id="${_tvdb_ep_id%%:*}"
- if [[ "$_pval" =~ ^s([0-9]+)e([0-9]+)$ ]]; then
- _iid=$(_api_get "${SRV_URL[$_si]}" "${SRV_KEY[$_si]}" \
- "Items?IncludeItemTypes=Episode&ParentIndexNumber=${BASH_REMATCH[1]}&IndexNumber=${BASH_REMATCH[2]}&Recursive=true&Fields=ProviderIds&Limit=500" 2>/dev/null \
- | jq -r --arg tvdb "$_tvdb_ep_id" \
- '.Items[] | select(.ProviderIds.Tvdb == $tvdb) | .Id' 2>/dev/null | head -1)
- fi
+ # _pkey format: tvdb:ep:{tvdb_id}:s{season}e{ep}
+ # ##*: gives "s7e2" (wrong); strip prefix then first :
+ _tvdb_num="${_pkey#tvdb:ep:}"; _tvdb_num="${_tvdb_num%%:*}"
+ _search_field="tvdb.${_tvdb_num}"
;;
- mb) : ;; # skip music if not found
+ mb) _search_field="" ;; # skip music if not found
esac
- if [[ -z "$_iid" && -n "$_search_field" ]]; then
+ if [[ -n "$_search_field" ]]; then
_iid=$(_api_get "${SRV_URL[$_si]}" "${SRV_KEY[$_si]}" \
"Items?AnyProviderIdEquals=${_search_field}&Recursive=true&Fields=ProviderIds&Limit=1" 2>/dev/null \
| jq -r '.Items[0].Id // empty' 2>/dev/null)
@@ -477,8 +467,17 @@ for lname in "${!USER_MAP[@]}"; do
if [[ "$_auth_played" == "true" ]]; then
# Mark as played with date
_date_param=""
- [[ "$_auth_lplayed" != "null" && -n "$_auth_lplayed" ]] && \
- _date_param="?DatePlayed=${_auth_lplayed//[: ]/%3A}"
+ if [[ "$_auth_lplayed" != "null" && -n "$_auth_lplayed" ]]; then
+ # Emby's PlayedItems endpoint rejects 7-digit fractional seconds (.0000000)
+ # with HTTP 500; strip to whole seconds before encoding
+ if [[ "$_auth_lplayed" == *.* ]]; then
+ _lp="${_auth_lplayed%.*}"
+ [[ "$_auth_lplayed" == *Z ]] && _lp+="Z"
+ else
+ _lp="$_auth_lplayed"
+ fi
+ _date_param="?DatePlayed=${_lp//[: ]/%3A}"
+ fi
_http=$(_api_post "${SRV_URL[$_si]}" "${SRV_KEY[$_si]}" \
"Users/${_uid}/PlayedItems/${_iid}${_date_param}")
if [[ "$_http" == "200" || "$_http" == "201" ]]; then
diff --git a/Plugin/unraid/Varaverk.page b/Plugin/unraid/Varaverk.page
index 82e978e..fa7afd8 100644
--- a/Plugin/unraid/Varaverk.page
+++ b/Plugin/unraid/Varaverk.page
@@ -24,9 +24,9 @@ unset($_master, $_h1m, $_host1_blank, $_my_hostid, $_conf_missing);
// Determine active tab
$tab = $_GET['tab'] ?? 'monitor';
-$validTabs = ['monitor', 'scheduler', 'docker', 'watchdog', 'partnership', 'fallback', 'arrs'];
+$validTabs = ['monitor', 'scheduler', 'docker', 'watchdog', 'partnership', 'fallback', 'arrs', 'rsync'];
if (!in_array($tab, $validTabs)) $tab = 'monitor';
-$tabLabels = ['monitor' => 'Monitor', 'scheduler' => 'Scheduler', 'docker' => 'Docker', 'watchdog' => 'Watchdog', 'partnership' => 'Partnership', 'fallback' => 'FallBack', 'arrs' => 'Arrs'];
+$tabLabels = ['monitor' => 'Monitor', 'scheduler' => 'Scheduler', 'docker' => 'Docker', 'watchdog' => 'Watchdog', 'partnership' => 'Partnership', 'fallback' => 'FallBack', 'arrs' => 'Arrs', 'rsync' => 'Rsync'];
?>
diff --git a/Plugin/unraid/api/manual_sync.php b/Plugin/unraid/api/manual_sync.php
new file mode 100644
index 0000000..cfcf878
--- /dev/null
+++ b/Plugin/unraid/api/manual_sync.php
@@ -0,0 +1,241 @@
+ $host, 'ssh_key' => $sshKey];
+}
+
+function _ms_target(string $slot, string $sshKey): array {
+ $vars = vv_conf_vars();
+ $hostname = $vars[strtoupper($slot)] ?? '';
+ if (!$hostname) return ['ok' => false, 'error' => 'Unknown host: ' . $slot];
+ if (!$sshKey || !file_exists($sshKey))
+ return ['ok' => false, 'error' => 'SSH key not configured on this host'];
+ $ip = vv_resolve_tailscale_ip($hostname);
+ if (!$ip) return ['ok' => false, 'error' => 'Cannot reach ' . $hostname . ' via Tailscale'];
+ return ['ok' => true, 'ip' => $ip, 'hostname' => $hostname];
+}
+
+$action = trim($_GET['action'] ?? $_POST['action'] ?? '');
+
+// ── hosts ─────────────────────────────────────────────────────────────────────
+if ($action === 'hosts') {
+ ['host' => $current, 'ssh_key' => $sshKey] = _ms_caller();
+ $all = vv_arr_known_hosts();
+ $peers = vv_pt_ts_peers();
+ $out = [];
+ foreach ($all as $slot => $hostname) {
+ if ($slot === $current) continue;
+ $label = strtolower($hostname);
+ $ts = $peers[$label] ?? [];
+ $out[] = [
+ 'slot' => $slot,
+ 'id' => strtoupper($slot),
+ 'hostname' => $hostname,
+ 'online' => $ts['online'] ?? null,
+ 'ip' => $ts['ip'] ?? null,
+ ];
+ }
+ echo json_encode(['ok' => true, 'hosts' => $out, 'has_key' => !empty($sshKey) && file_exists($sshKey), 'ssh_key' => $sshKey]);
+ exit;
+}
+
+// ── browse remote (SSH) ───────────────────────────────────────────────────────
+if ($action === 'browse') {
+ $slot = trim($_GET['host'] ?? '');
+ $path = trim($_GET['path'] ?? '/mnt/user');
+ if (!preg_match('#^/[^\0]*$#', $path) || str_contains($path, '..')) {
+ echo json_encode(['ok' => false, 'error' => 'Invalid path']); exit;
+ }
+ ['ssh_key' => $sshKey] = _ms_caller();
+ $t = _ms_target($slot, $sshKey);
+ if (!$t['ok']) { echo json_encode($t); exit; }
+
+ $clean = rtrim($path, '/') ?: '/';
+ $out = vv_pt_ssh($t['ip'], $sshKey,
+ 'find ' . escapeshellarg($clean) . ' -maxdepth 1 -mindepth 1 -type d 2>/dev/null',
+ 8);
+ $dirs = array_values(array_filter(array_map('trim', explode("\n", $out))));
+ sort($dirs);
+ $dirs = array_slice($dirs, 0, 200);
+ // If empty, do a quick SSH echo to distinguish "no dirs" from "SSH failed"
+ if (empty($dirs)) {
+ $ping = trim(vv_pt_ssh($t['ip'], $sshKey, 'echo ok', 4));
+ if ($ping !== 'ok') {
+ echo json_encode(['ok' => false, 'error' => 'SSH connection failed to ' . $t['hostname']]);
+ exit;
+ }
+ }
+ $parent = ($clean !== '/') ? (dirname($clean) ?: '/') : null;
+ echo json_encode(['ok' => true, 'path' => $clean, 'dirs' => $dirs, 'parent' => $parent]);
+ exit;
+}
+
+// ── browse local ──────────────────────────────────────────────────────────────
+if ($action === 'browse_local') {
+ $path = trim($_GET['path'] ?? '/mnt/user');
+ if (!preg_match('#^/[^\0]*$#', $path) || str_contains($path, '..')) {
+ echo json_encode(['ok' => false, 'error' => 'Invalid path']); exit;
+ }
+ $clean = rtrim($path, '/') ?: '/';
+ if (!is_dir($clean)) {
+ echo json_encode(['ok' => false, 'error' => 'Not a directory: ' . $clean]); exit;
+ }
+ $out = shell_exec('find ' . escapeshellarg($clean) . ' -maxdepth 1 -mindepth 1 -type d 2>/dev/null | sort | head -200') ?: '';
+ $dirs = array_values(array_filter(array_map('trim', explode("\n", $out))));
+ $parent = ($clean !== '/') ? (dirname($clean) ?: '/') : null;
+ echo json_encode(['ok' => true, 'path' => $clean, 'dirs' => $dirs, 'parent' => $parent]);
+ exit;
+}
+
+// ── run (background, returns token) ──────────────────────────────────────────
+if ($_SERVER['REQUEST_METHOD'] === 'POST' && $action === 'run') {
+ $local = trim($_POST['local'] ?? '');
+ $slot = trim($_POST['host'] ?? '');
+ $remotePath = trim($_POST['remote_path'] ?? '');
+ $user = preg_replace('/[^a-z0-9_.-]/i', '', trim($_POST['user'] ?? 'root')) ?: 'root';
+ $bwLimit = max(0, (int)($_POST['bw_limit'] ?? 0));
+ $useKey = ($_POST['use_key'] ?? '1') !== '0';
+ $rawFlags = trim($_POST['flags'] ?? '');
+ $flags = $rawFlags !== '' ? preg_replace('/[`$!|&;><(){}\[\]\\\\]/', '', $rawFlags) : '-av --stats';
+
+ foreach (['local' => $local, 'host' => $slot, 'remote_path' => $remotePath] as $f => $v) {
+ if (!$v) { echo json_encode(['ok' => false, 'error' => 'Missing: ' . $f]); exit; }
+ }
+ foreach ([$local, $remotePath] as $p) {
+ if (!str_starts_with($p, '/') || str_contains($p, '..') || preg_match('/[\x00\n\r`$]/', $p)) {
+ echo json_encode(['ok' => false, 'error' => 'Invalid path: ' . $p]); exit;
+ }
+ }
+
+ // ── Safeguards ────────────────────────────────────────────────────────────
+
+ // Block syncing from/to dangerous system paths
+ $blocked = ['/', '/proc', '/sys', '/dev', '/run', '/etc', '/bin', '/sbin',
+ '/usr', '/lib', '/lib64', '/boot/EFI', '/tmp'];
+ foreach ($blocked as $b) {
+ if (rtrim($local, '/') === $b) {
+ echo json_encode(['ok' => false, 'error' => 'Refusing to sync from system path: ' . $b]); exit;
+ }
+ if (rtrim($remotePath, '/') === $b) {
+ echo json_encode(['ok' => false, 'error' => 'Refusing to sync to system path: ' . $b]); exit;
+ }
+ }
+
+ // Local source must exist
+ if (!file_exists($local)) {
+ echo json_encode(['ok' => false, 'error' => 'Local source does not exist: ' . $local]); exit;
+ }
+
+ // Prevent concurrent manual syncs
+ foreach (glob('/tmp/vv_ms_*.pid') ?: [] as $pf) {
+ $pid = (int)trim(@file_get_contents($pf) ?: '0');
+ if ($pid > 0 && file_exists('/proc/' . $pid)) {
+ echo json_encode(['ok' => false, 'error' => 'Another manual sync is already running — stop it first.']); exit;
+ }
+ @unlink($pf); // stale
+ }
+
+ ['ssh_key' => $sshKey] = _ms_caller();
+ $t = _ms_target($slot, $sshKey);
+ if (!$t['ok']) { echo json_encode($t); exit; }
+
+ // Remote destination must exist
+ $destCheck = trim(vv_pt_ssh($t['ip'], $sshKey,
+ 'test -d ' . escapeshellarg($remotePath) . ' && echo ok || echo missing', 5));
+ if ($destCheck === 'missing') {
+ echo json_encode(['ok' => false, 'error' => 'Remote destination does not exist: ' . $remotePath]); exit;
+ }
+ if ($destCheck !== 'ok') {
+ // SSH check inconclusive — log warning but proceed; rsync will fail cleanly if needed
+ // (e.g. Tailscale not yet up, rsync error will surface in output)
+ }
+
+ // ── Build and launch ──────────────────────────────────────────────────────
+
+ $token = bin2hex(random_bytes(8));
+ $logFile = '/tmp/vv_ms_' . $token . '.log';
+ $pidFile = '/tmp/vv_ms_' . $token . '.pid';
+
+ if ($useKey && $sshKey && file_exists($sshKey)) {
+ $sshOpts = 'ssh -i ' . escapeshellarg($sshKey) . ' -o StrictHostKeyChecking=no -o BatchMode=yes -o ConnectTimeout=10';
+ } else {
+ $sshOpts = 'ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10';
+ }
+ if ($bwLimit) $flags .= ' --bwlimit=' . (int)$bwLimit;
+
+ $src = escapeshellarg(rtrim($local, '/') . '/');
+ $dst = escapeshellarg($user . '@' . $t['ip'] . ':' . rtrim($remotePath, '/') . '/');
+
+ // Start rsync in background, capture its PID for stop support
+ $inner = "rsync $flags -e " . escapeshellarg($sshOpts) . " $src $dst >> " . escapeshellarg($logFile) . " 2>&1 &"
+ . " RSYNC_PID=\$!;"
+ . " echo \$RSYNC_PID > " . escapeshellarg($pidFile) . ";"
+ . " wait \$RSYNC_PID;"
+ . " echo __DONE__ >> " . escapeshellarg($logFile) . ";"
+ . " rm -f " . escapeshellarg($pidFile);
+ shell_exec('nohup bash -c ' . escapeshellarg($inner) . ' &>/dev/null &');
+
+ echo json_encode(['ok' => true, 'token' => $token]);
+ exit;
+}
+
+// ── stop ──────────────────────────────────────────────────────────────────────
+if ($_SERVER['REQUEST_METHOD'] === 'POST' && $action === 'stop') {
+ $token = preg_replace('/[^a-f0-9]/', '', trim($_POST['token'] ?? ''));
+ if (!$token || strlen($token) !== 16) {
+ echo json_encode(['ok' => false, 'error' => 'Invalid token']); exit;
+ }
+ $pidFile = '/tmp/vv_ms_' . $token . '.pid';
+ $logFile = '/tmp/vv_ms_' . $token . '.log';
+
+ $pid = (int)trim(@file_get_contents($pidFile) ?: '0');
+ if ($pid > 0 && file_exists('/proc/' . $pid)) {
+ shell_exec('kill -TERM ' . $pid . ' 2>/dev/null');
+ usleep(400000); // 400ms grace
+ if (file_exists('/proc/' . $pid)) shell_exec('kill -KILL ' . $pid . ' 2>/dev/null');
+ }
+ @unlink($pidFile);
+ @file_put_contents($logFile, "\n\n--- Cancelled by user ---\n__DONE__\n", FILE_APPEND);
+ echo json_encode(['ok' => true]);
+ exit;
+}
+
+// ── poll ──────────────────────────────────────────────────────────────────────
+if ($action === 'poll') {
+ $token = preg_replace('/[^a-f0-9]/', '', trim($_GET['token'] ?? ''));
+ if (!$token || strlen($token) !== 16) {
+ echo json_encode(['ok' => false, 'error' => 'Invalid token']); exit;
+ }
+ $logFile = '/tmp/vv_ms_' . $token . '.log';
+ $pidFile = '/tmp/vv_ms_' . $token . '.pid';
+ if (!file_exists($logFile)) {
+ echo json_encode(['ok' => true, 'output' => '', 'done' => false, 'started' => false]); exit;
+ }
+ $content = file_get_contents($logFile) ?: '';
+ $done = str_contains($content, '__DONE__');
+ if ($done) {
+ $content = str_replace(['__DONE__', "\n\n\n"], ['', "\n\n"], $content);
+ @unlink($logFile);
+ @unlink($pidFile);
+ }
+ // Check if rsync process is actually alive (catches crashes without __DONE__)
+ $pid = (int)trim(@file_get_contents($pidFile) ?: '0');
+ if (!$done && $pid > 0 && !file_exists('/proc/' . $pid)) {
+ $done = true;
+ $content .= "\n\n--- Process ended unexpectedly ---";
+ @unlink($pidFile);
+ }
+ echo json_encode(['ok' => true, 'output' => $content, 'done' => $done, 'started' => true]);
+ exit;
+}
+
+echo json_encode(['ok' => false, 'error' => 'Unknown action']);
diff --git a/Plugin/unraid/api/rsync.php b/Plugin/unraid/api/rsync.php
new file mode 100644
index 0000000..bcfc10a
--- /dev/null
+++ b/Plugin/unraid/api/rsync.php
@@ -0,0 +1,47 @@
+ $p[0],
+ 'time' => $p[1],
+ 'profile' => $p[2],
+ 'duration' => (int)$p[3],
+ 'status' => trim($p[4]),
+ 'bytes' => isset($p[5]) ? (int)$p[5] : 0,
+ ];
+ }
+}
+
+echo json_encode([
+ 'enabled' => $base['enabled'],
+ 'windows' => $base['windows'],
+ 'active' => $base['active'],
+ 'last_sync' => $base['last_sync'],
+ 'bw_history' => $history,
+ 'bw_warn_gb' => $warnGb,
+ 'settings' => [
+ 'bw_limit' => (int)($vars['BW_LIMIT'] ?? 0),
+ 'retry_count' => (int)($vars['RETRY_COUNT'] ?? 3),
+ 'sleep' => (int)($vars['SLEEP'] ?? 300),
+ 'bw_warn_gb' => (float)($vars['BANDWIDTH_WARN_GB'] ?? 50),
+ ],
+ 'ts' => time(),
+]);
diff --git a/Plugin/unraid/api/rsync_profiles.php b/Plugin/unraid/api/rsync_profiles.php
new file mode 100644
index 0000000..7616a5c
--- /dev/null
+++ b/Plugin/unraid/api/rsync_profiles.php
@@ -0,0 +1,132 @@
+ 'rsync_opts',
+ 'PROFILE_BW_LIMIT' => 'bw_limit',
+ 'PROFILE_RETRY_COUNT' => 'retry_count',
+ 'PROFILE_SLEEP' => 'sleep',
+ 'PROFILE_CRITICAL_CONTAINER_NAMES' => 'critical_containers',
+ 'PROFILE_DELAYED_CONTAINERS' => 'delayed_containers',
+ 'PROFILE_CONTAINER_DELAY' => 'container_delay',
+ 'PROFILE_EXCLUDE_DIRS' => 'exclude_dirs',
+ 'PROFILE_REMOTE_RESTART_CONTAINERS' => 'remote_restart',
+];
+
+// Parse [key]="value" or [key]=bare entries from an assoc_array body
+function _rp_parse_assoc(string $body): array {
+ $result = [];
+ preg_match_all('/\[([^\]]+)\]\s*=\s*(?:"([^"]*)"|([^\s#\n]*))/', $body, $m, PREG_SET_ORDER);
+ foreach ($m as $match) {
+ $key = $match[1];
+ $val = $match[2] !== '' ? $match[2] : ($match[3] ?? '');
+ $result[$key] = $val;
+ }
+ return $result;
+}
+
+// Rebuild assoc_array body from entries map
+function _rp_build_assoc(array $entries): string {
+ $lines = [];
+ foreach ($entries as $k => $v) {
+ // Quote if empty, has spaces or special shell chars
+ if ($v === '' || preg_match('/[\s\$\!\[\]\(\)\|\'`\\\\]/', $v)) {
+ $lines[] = ' [' . $k . ']="' . str_replace('"', '\\"', $v) . '"';
+ } else {
+ $lines[] = ' [' . $k . ']=' . $v;
+ }
+ }
+ return implode("\n", $lines);
+}
+
+// Read all PROFILE_* arrays from master.conf → [profile_name => [field => value]]
+function _rp_read_all(): array {
+ $raw = vv_read_conf_raw('master.conf');
+ $result = [];
+ foreach (RP_ARRAYS as $varName => $fieldKey) {
+ if (!preg_match('/declare\s+-A\s+' . preg_quote($varName, '/') . '\s*=\s*\((.*?)\)/s', $raw, $m)) continue;
+ foreach (_rp_parse_assoc($m[1]) as $profile => $value) {
+ $result[$profile][$fieldKey] = $value;
+ }
+ }
+ return $result;
+}
+
+$action = trim($_GET['action'] ?? $_POST['action'] ?? '');
+
+// ── list ──────────────────────────────────────────────────────────────────────
+if ($action === 'list') {
+ echo json_encode(['ok' => true, 'profiles' => _rp_read_all()]);
+ exit;
+}
+
+// ── save (create or update) ───────────────────────────────────────────────────
+if ($_SERVER['REQUEST_METHOD'] === 'POST' && $action === 'save') {
+ $name = trim($_POST['name'] ?? '');
+ if (!$name || !preg_match('/^[a-zA-Z0-9_\-]+$/', $name)) {
+ echo json_encode(['ok' => false, 'error' => 'Invalid profile name — use letters, numbers, hyphens, underscores']); exit;
+ }
+
+ $raw = vv_read_conf_raw('master.conf');
+ $changes = [];
+
+ foreach (RP_ARRAYS as $varName => $fieldKey) {
+ $value = trim($_POST[$fieldKey] ?? '');
+ // Find and parse current array body
+ if (!preg_match('/declare\s+-A\s+' . preg_quote($varName, '/') . '\s*=\s*\((.*?)\)/s', $raw, $m)) continue;
+ $entries = _rp_parse_assoc($m[1]);
+ $entries[$name] = $value;
+ $changes[] = [
+ 'file' => 'master.conf',
+ 'key' => $varName,
+ 'type' => 'assoc_array',
+ 'value' => _rp_build_assoc($entries),
+ ];
+ }
+
+ if (!$changes) { echo json_encode(['ok' => false, 'error' => 'No profile arrays found in master.conf']); exit; }
+
+ $results = vv_conf_write_changes($changes);
+ $ok = !in_array(false, $results, true);
+ if ($ok) { vv_push_master_conf(); vv_push_setup_state(); }
+ echo json_encode(['ok' => $ok, 'results' => $results]);
+ exit;
+}
+
+// ── delete ────────────────────────────────────────────────────────────────────
+if ($_SERVER['REQUEST_METHOD'] === 'POST' && $action === 'delete') {
+ $name = trim($_POST['name'] ?? '');
+ if (!$name || !preg_match('/^[a-zA-Z0-9_\-]+$/', $name)) {
+ echo json_encode(['ok' => false, 'error' => 'Invalid profile name']); exit;
+ }
+
+ $raw = vv_read_conf_raw('master.conf');
+ $changes = [];
+
+ foreach (RP_ARRAYS as $varName => $fieldKey) {
+ if (!preg_match('/declare\s+-A\s+' . preg_quote($varName, '/') . '\s*=\s*\((.*?)\)/s', $raw, $m)) continue;
+ $entries = _rp_parse_assoc($m[1]);
+ if (!array_key_exists($name, $entries)) continue;
+ unset($entries[$name]);
+ $changes[] = [
+ 'file' => 'master.conf',
+ 'key' => $varName,
+ 'type' => 'assoc_array',
+ 'value' => _rp_build_assoc($entries),
+ ];
+ }
+
+ if (!$changes) { echo json_encode(['ok' => false, 'error' => 'Profile not found']); exit; }
+
+ $results = vv_conf_write_changes($changes);
+ $ok = !in_array(false, $results, true);
+ if ($ok) { vv_push_master_conf(); vv_push_setup_state(); }
+ echo json_encode(['ok' => $ok]);
+ exit;
+}
+
+echo json_encode(['ok' => false, 'error' => 'Unknown action']);
diff --git a/Plugin/unraid/include/monitor.php b/Plugin/unraid/include/monitor.php
index d00e5d1..de80e6f 100644
--- a/Plugin/unraid/include/monitor.php
+++ b/Plugin/unraid/include/monitor.php
@@ -367,10 +367,30 @@ function vv_rsync_status(): array {
];
}
+ // Profile activity — last 7 days, aggregated per profile
+ $bwLog = DATA_DIR . '/bandwidth_history.db';
+ $cutoff7 = date('Y-m-d', strtotime('-7 days'));
+ $profiles = [];
+ if (file_exists($bwLog)) {
+ foreach (file($bwLog, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES) ?: [] as $line) {
+ $p = explode('|', $line);
+ if (count($p) < 4 || ($p[0] ?? '') < $cutoff7) continue;
+ $name = $p[2] ?? '';
+ if (!$name) continue;
+ if (!isset($profiles[$name])) $profiles[$name] = ['runs' => 0, 'dur' => 0, 'bytes' => 0];
+ $profiles[$name]['runs']++;
+ $profiles[$name]['dur'] += (int)($p[3] ?? 0);
+ $profiles[$name]['bytes'] += (int)($p[5] ?? 0);
+ }
+ }
+ arsort($profiles); // sort by run count descending
+ $bwSummary = array_slice($profiles, 0, 6, true);
+
return [
- 'enabled' => $enabled,
- 'windows' => $windows,
- 'active' => $active,
- 'last_sync' => $lastSync,
+ 'enabled' => $enabled,
+ 'windows' => $windows,
+ 'active' => $active,
+ 'last_sync' => $lastSync,
+ 'bw_summary' => $bwSummary,
];
}
diff --git a/Plugin/unraid/include/partnership.php b/Plugin/unraid/include/partnership.php
index ccfdc30..fa0984e 100644
--- a/Plugin/unraid/include/partnership.php
+++ b/Plugin/unraid/include/partnership.php
@@ -30,6 +30,27 @@ function vv_pt_config(): array {
}
// ── Mirror sync health — the partnership's actual job (rsync orchestrators) ──────
+
+function vv_pt_sync_summary(string $logFile): string {
+ if (!file_exists($logFile)) return '';
+ $tail = array_filter(array_slice(file($logFile, FILE_IGNORE_NEW_LINES), -30), 'strlen');
+ foreach (array_reverse(array_values($tail)) as $raw) {
+ // Strip emoji / Unicode decoration for regex matching
+ $line = trim(preg_replace('/[\x{1F000}-\x{1FFFF}\x{2600}-\x{27BF}\x{FE0F}]/u', '', $raw));
+ $line = preg_replace('/\s+/', ' ', $line);
+ // "Critical sync complete — HOST1 — 1m39s — 2 share(s)"
+ if (preg_match('/Critical sync complete\s*—\s*\S+\s*—\s*([\w]+)\s*—\s*(.+)/i', $line, $m))
+ return trim($m[2]) . ' in ' . $m[1];
+ // "Status: all complete — 0 share(s) synced, 11 job(s) run"
+ if (preg_match('/Status:\s*all complete\s*—\s*(.+)/i', $line, $m))
+ return trim($m[1]);
+ // "Status: X failure(s)" or "Failures: N"
+ if (preg_match('/Failures:\s*(\d+)/i', $line, $m) && (int)$m[1] > 0)
+ return (int)$m[1] . ' failure(s)';
+ }
+ return '';
+}
+
function vv_pt_sync(): array {
$jobs = [
'critical' => 'Orchestrators/critical_sync_maintenance',
@@ -39,14 +60,17 @@ function vv_pt_sync(): array {
$out = ['jobs' => []];
foreach ($jobs as $key => $base) {
$statFile = LOG_DIR . '/' . $base . '.json';
+ $logFile = LOG_DIR . '/' . $base . '.log';
$s = file_exists($statFile) ? json_decode(@file_get_contents($statFile), true) : null;
$out['jobs'][$key] = is_array($s) ? [
- 'status' => $s['status'] ?? 'unknown',
- 'start' => isset($s['start']) ? (int)$s['start'] : null,
- 'end' => isset($s['end']) ? (int)$s['end'] : null,
+ 'status' => $s['status'] ?? 'unknown',
+ 'start' => isset($s['start']) ? (int)$s['start'] : null,
+ 'end' => isset($s['end']) ? (int)$s['end'] : null,
+ 'summary' => vv_pt_sync_summary($logFile),
] : null;
}
$v = vv_conf_vars();
+ $out['interval_min'] = (int)($v['PARTNERSHIP_SYNC_INTERVAL'] ?? 30);
// Rsync gate flags (master.conf) — global Tier 1 + per-tier Tier 2.
$out['gates'] = [
'global' => ['var' => 'RSYNC_ENABLED', 'on' => ($v['RSYNC_ENABLED'] ?? 'true') === 'true'],
diff --git a/Plugin/unraid/pages/monitor.php b/Plugin/unraid/pages/monitor.php
index 0f85fea..bcaf999 100644
--- a/Plugin/unraid/pages/monitor.php
+++ b/Plugin/unraid/pages/monitor.php
@@ -1,4 +1,10 @@
+
@@ -1318,74 +1324,143 @@ function vvPollMonitor() {
// ── Rsync ────────────────────────────────────────────────────────────────
(function() {
- const rs = d.rsync ?? {};
- const enabled = rs.enabled ?? true;
- const windows = rs.windows ?? {};
- const active = rs.active ?? [];
- const lastSync = rs.last_sync ?? {};
- const now = Math.floor(Date.now() / 1000);
+ const rs = d.rsync ?? {};
+ const enabled = rs.enabled ?? true;
+ const windows = rs.windows ?? {};
+ const active = rs.active ?? [];
+ const lastSync = rs.last_sync ?? {};
+ const bwSummary = rs.bw_summary ?? {};
+ const now = Math.floor(Date.now() / 1000);
const el = document.getElementById('vv-rsync-body');
if (!el) return;
- const gColor = enabled ? '#4caf50' : '#555';
- let html = `● ${enabled ? 'enabled' : 'disabled'}
`;
+ function _dur(s) {
+ if (!s) return '—';
+ if (s < 60) return s + 's';
+ if (s < 3600) return Math.floor(s/60) + 'm' + (s%60 ? String(s%60).padStart(2,'0')+'s' : '');
+ return Math.floor(s/3600) + 'h' + Math.floor((s%3600)/60) + 'm';
+ }
+ function _ago(diff) {
+ if (diff < 60) return diff + 's';
+ if (diff < 3600) return Math.floor(diff/60) + 'm';
+ if (diff < 86400) return Math.floor(diff/3600) + 'h';
+ return Math.floor(diff/86400) + 'd';
+ }
+ function _fmtBytes(b) {
+ if (!b) return '';
+ if (b >= 1073741824) return (b/1073741824).toFixed(1) + 'G';
+ if (b >= 1048576) return (b/1048576).toFixed(0) + 'M';
+ return (b/1024).toFixed(0) + 'K';
+ }
- // Window badges: C D I W
- html += '';
+ const gCol = enabled ? '#4caf50' : '#555';
+
+ // ── Header: gate dot + window badges ──────────────────────────────────
+ let html = `
+
● ${enabled ? 'ENABLED' : 'DISABLED'}
+
`;
[['C','critical'],['D','daily'],['I','intermediate'],['W','weekly']].forEach(([s,k]) => {
- const on = windows[k] ?? false;
- html += `${s} `;
+ const on = windows[k] ?? false;
+ const run = active.some(a => a.profile?.includes(k));
+ const bg = run ? '#1a2a0a' : on ? '#0f1a0f' : '#111';
+ const brd = run ? '#3a6a1a' : on ? '#1a3a1a' : '#222';
+ const col = run ? '#8bc34a' : on ? '#4caf50' : '#333';
+ html += `${s} `;
});
- html += '
';
+ html += `
`;
- // Active sessions
+ // ── Profile activity bars (7 days) ────────────────────────────────────
+ const profEntries = Object.entries(bwSummary);
+ if (profEntries.length) {
+ const maxRuns = Math.max(...profEntries.map(([,v]) => v.runs), 1);
+ const colors = ['#4caf50','#4a9eff','#ffb74d','#ce93d8','#ef5350','#4dd0e1'];
+ html += `
+
7-day profile activity
`;
+ profEntries.forEach(([name, v], i) => {
+ const pct = Math.round(v.runs / maxRuns * 100);
+ const col = colors[i % colors.length];
+ const bytes = v.bytes > 0 ? _fmtBytes(v.bytes) : '';
+ const short = name.length > 18 ? name.slice(0,16) + '…' : name;
+ html += `
+
+ ${short}
+
+ ${v.runs}×${bytes ? ' · ' + bytes : ''}
+
+
+
`;
+ });
+ html += `
`;
+ }
+
+ // ── Active syncs ───────────────────────────────────────────────────────
if (active.length) {
active.forEach(a => {
const sec = a.elapsed ?? 0;
- const dur = sec < 60 ? sec+'s' : sec < 3600 ? Math.floor(sec/60)+'m'+String(sec%60).padStart(2,'0')+'s' : Math.floor(sec/3600)+'h'+Math.floor((sec%3600)/60)+'m';
- html += `
-
⟳ ${a.profile}
-
${dur}
+ html += `
+
+ ⟳ ${a.profile}
+ ${_dur(sec)}
+
+
`;
});
- html += `
`;
}
- // Last sync per window
- const syncRows = [['critical','critical'],['daily','daily'],['intermediate','interm'],['weekly','weekly']];
- let hasSyncs = false;
- let syncHtml = '';
- syncRows.forEach(([key, label]) => {
- const s = lastSync[key];
- if (!s || !s.ts) return;
- hasSyncs = true;
- const diff = now - s.ts;
- const ago = diff < 60 ? diff+'s' : diff < 3600 ? Math.floor(diff/60)+'m' : diff < 86400 ? Math.floor(diff/3600)+'h' : Math.floor(diff/86400)+'d';
- const ok = s.status === 'ok' || s.status === 'success';
- const warn = s.status === 'warn';
- const run = s.status === 'running';
- const col = ok ? '#4caf50' : warn ? '#ff9800' : run ? '#4fc3f7' : '#f44336';
- const icon = ok ? '✓' : warn ? '!' : run ? '●' : '✗';
- syncHtml += `
-
${label}
-
${ago}
-
${icon}
+ // ── Tier last-run rows with recency bars ───────────────────────────────
+ const tierMeta = {
+ critical: {label:'Critical', cadenceSec: 30*60},
+ daily: {label:'Daily', cadenceSec: 24*3600},
+ intermediate: {label:'Interm', cadenceSec: 4*3600},
+ weekly: {label:'Weekly', cadenceSec: 7*86400},
+ };
+ html += `
`;
+ Object.entries(tierMeta).forEach(([key, meta]) => {
+ const s = lastSync[key];
+ const ts = s?.ts ?? 0;
+ const ok = s?.status === 'ok' || s?.status === 'success';
+ const warn = s?.status === 'warn';
+ const run = s?.status === 'running';
+ const err = s && !ok && !warn && !run && ts > 0;
+ const col = run ? '#4a9eff' : ok ? '#4caf50' : warn ? '#ff9800' : err ? '#f44336' : '#2a2a2a';
+ const icon = run ? '⟳' : ok ? '✓' : warn ? '!' : err ? '✗' : '—';
+ const diff = ts ? now - ts : null;
+ const pct = ts ? Math.min(100, Math.round((now - ts) / meta.cadenceSec * 100)) : 0;
+ const barC = pct >= 100 ? '#3a1a1a' : pct >= 75 ? '#2a1a0a' : '#0a1a0a';
+ const fillC= pct >= 100 ? '#f44336' : pct >= 75 ? '#ff9800' : '#2a6a2a';
+
+ html += `
+
+ ${meta.label}
+
+ ${s?.duration ? _dur(s.duration) : ''}
+
+ ${diff !== null ? _ago(diff) : '—'}
+ ${icon}
+
+
`;
});
- if (hasSyncs) {
- html += `
Last sync
`;
- html += syncHtml;
- }
+ html += `
`;
el.innerHTML = html;
const card = document.getElementById('vv-rsync-card');
if (card) {
card.classList.remove('vv-accent-ok','vv-accent-warn','vv-accent-err');
- if (active.length) card.classList.add('vv-accent-ok');
+ if (!enabled) card.classList.add('vv-accent-err');
+ else if (active.length) card.classList.add('vv-accent-ok');
}
})();
diff --git a/Plugin/unraid/pages/partnership.php b/Plugin/unraid/pages/partnership.php
index a1c30c3..72ec10c 100644
--- a/Plugin/unraid/pages/partnership.php
+++ b/Plugin/unraid/pages/partnership.php
@@ -66,6 +66,16 @@ textarea.vv-set-input { resize:vertical; white-space:pre; }
Loading…
+
+
+
+
Host Settings
+ Save
+
+
+
+
@@ -74,16 +84,19 @@ textarea.vv-set-input { resize:vertical; white-space:pre; }
Loading…
-
-
-
Mirror Sync
-
Loading…
-
+
+
-
-
-
Actions
-
Loading…
+
+
+
+
Mirror Sync
+
Loading…
+
+
@@ -303,6 +316,165 @@ function vvPtToggleSync(el, varName, enabled) {
.catch(e => { alert('Error: ' + e); el.style.pointerEvents = ''; el.style.opacity = ''; });
}
+// ── Host Settings ─────────────────────────────────────────────────────────────
+
+function _renderHostSettings(nodes, isOwner) {
+ let html = '
';
+ for (const n of nodes) {
+ const rdonly = isOwner ? '' : 'readonly style="opacity:.5;cursor:default;"';
+ html += `
`;
+ }
+ html += '
';
+ if (!isOwner) html += '
Editing requires host1.
';
+ return html;
+}
+
+function vvPtHostChanged(el) {
+ el.classList.toggle('changed', el.value !== el.dataset.orig);
+ const any = document.querySelector('.vv-pt-host-inp.changed');
+ document.getElementById('vv-pt-hosts-save').style.display = any ? '' : 'none';
+}
+
+function vvPtSaveHosts(btn) {
+ const changedEls = document.querySelectorAll('.vv-pt-host-inp.changed');
+ if (!changedEls.length) return;
+ const changes = [];
+ changedEls.forEach(el => changes.push({key: el.dataset.key, file: el.dataset.file, type: el.dataset.type, value: el.value}));
+ if (!confirm(`Save ${changes.length} host setting(s)?`)) return;
+ btn.disabled = true; btn.textContent = '⟳ Saving…';
+ fetch('/plugins/varaverk/api/confform.php', {
+ method: 'POST',
+ headers: {'Content-Type': 'application/x-www-form-urlencoded'},
+ body: new URLSearchParams({
+ csrf_token: typeof csrf_token !== 'undefined' ? csrf_token : '',
+ id: '__settings__',
+ changes: JSON.stringify(changes)
+ })
+ })
+ .then(r => r.json())
+ .then(d => {
+ btn.disabled = false;
+ if (d.ok) {
+ changedEls.forEach(el => { el.dataset.orig = el.value; el.classList.remove('changed'); });
+ btn.textContent = '✓ Saved';
+ setTimeout(() => { btn.textContent = 'Save'; btn.style.display = 'none'; }, 2000);
+ } else {
+ btn.textContent = 'Save';
+ alert('Save failed: ' + (d.error ?? 'Unknown error'));
+ }
+ })
+ .catch(e => { btn.disabled = false; btn.textContent = 'Save'; alert('Error: ' + e); });
+}
+
+// ── Array settings cards ───────────────────────────────────────────────────────
+
+let _vvArrFiles = null;
+
+function _vvInitArrayCards() {
+ if (_vvArrFiles !== null) return;
+ _vvArrFiles = false; // loading sentinel
+ fetch('/plugins/varaverk/api/partnership_settings.php?_=' + Date.now())
+ .then(r => r.json())
+ .then(d => {
+ _vvArrFiles = (d.ok && d.files) ? d.files : [];
+ _vvRenderArrayCards();
+ })
+ .catch(() => { _vvArrFiles = []; document.getElementById('vv-pt-arrays-wrap').style.display = 'none'; });
+}
+
+function _vvRenderArrayCards() {
+ const files = _vvArrFiles;
+ if (!files || !files.length) { document.getElementById('vv-pt-arrays-wrap').style.display = 'none'; return; }
+
+ let html = '';
+ let hasArrays = false;
+ for (const f of files) {
+ for (const g of f.groups) {
+ for (const fld of g.fields) {
+ if (fld.type === 'scalar') continue;
+ hasArrays = true;
+ const rows = Math.min(16, (fld.value.match(/\n/g) || []).length + 2);
+ const attrs = `class="vv-set-input" data-key="${_vvSetEsc(fld.key)}" `
+ + `data-file="${_vvSetEsc(fld.file)}" data-type="${_vvSetEsc(fld.type)}" `
+ + `data-orig="${_vvSetEsc(fld.value)}" oninput="vvPtArrChanged(this)"`;
+ html += `
+
+
${_vvSetEsc(fld.key)}
+
+ ${fld.desc ? `${_vvSetEsc(fld.desc)} ` : ''}
+ ▸
+
+
+
+
+
+
`;
+ }
+ }
+ }
+
+ const wrap = document.getElementById('vv-pt-arrays-wrap');
+ if (!hasArrays) { wrap.style.display = 'none'; return; }
+
+ wrap.style.display = '';
+ wrap.innerHTML = `
+
+
Array Settings
+ Save Changes
+
+ ${html}
+
`;
+}
+
+function vvPtArrChanged(el) {
+ el.classList.toggle('changed', el.value !== el.dataset.orig);
+ const any = document.querySelector('#vv-pt-arrays-wrap .vv-set-input.changed');
+ const saveBtn = document.getElementById('vv-pt-arrays-save');
+ if (saveBtn) saveBtn.style.display = any ? '' : 'none';
+}
+
+function vvPtSaveArrays(btn) {
+ const changedEls = document.querySelectorAll('#vv-pt-arrays-wrap .vv-set-input.changed');
+ if (!changedEls.length) return;
+ const changes = [];
+ changedEls.forEach(el => changes.push({key: el.dataset.key, file: el.dataset.file, type: el.dataset.type, value: el.value}));
+ if (!confirm(`Save ${changes.length} changed setting(s)?`)) return;
+ btn.disabled = true; btn.textContent = '⟳ Saving…';
+ fetch('/plugins/varaverk/api/confform.php', {
+ method: 'POST',
+ headers: {'Content-Type': 'application/x-www-form-urlencoded'},
+ body: new URLSearchParams({
+ csrf_token: typeof csrf_token !== 'undefined' ? csrf_token : '',
+ id: '__settings__',
+ changes: JSON.stringify(changes)
+ })
+ })
+ .then(r => r.json())
+ .then(d => {
+ btn.disabled = false;
+ if (d.ok) {
+ changedEls.forEach(el => { el.dataset.orig = el.value; el.classList.remove('changed'); });
+ btn.textContent = '✓ Saved';
+ _vvArrFiles = null; // invalidate so next init re-fetches
+ setTimeout(() => _vvInitArrayCards(), 600);
+ } else {
+ btn.textContent = 'Save Changes';
+ alert('Save failed: ' + (d.error ?? 'Unknown error'));
+ }
+ })
+ .catch(e => { btn.disabled = false; btn.textContent = 'Save Changes'; alert('Error: ' + e); });
+}
+
// ── Settings panel ─────────────────────────────────────────────────────────────
let _vvSetLoaded = false;
@@ -343,33 +515,26 @@ function vvPtLoadSettings() {
function vvPtRenderSettings(files) {
let html = '';
for (const f of files) {
- // One block per file; partnership scope yields a single section per file, so flatten.
const label = f.file === 'master.conf' ? 'master.conf — shared' : f.file + ' — this host';
+ const scalarFields = f.groups.flatMap(g => g.fields.filter(fld => fld.type === 'scalar'));
+ if (!scalarFields.length) continue;
html += `
${_vvSetEsc(label)} ▾
`;
- for (const g of f.groups) {
- for (const fld of g.fields) {
- const attrs = `class="vv-set-input" data-key="${_vvSetEsc(fld.key)}" `
- + `data-file="${_vvSetEsc(fld.file)}" data-type="${_vvSetEsc(fld.type)}" `
- + `data-orig="${_vvSetEsc(fld.value)}" oninput="vvPtSetChanged(this)"`;
- html += `
-
${_vvSetEsc(fld.key)}
`;
- if (fld.desc) html += `
${_vvSetEsc(fld.desc)}
`;
- if (fld.type === 'scalar') {
- html += `
`;
- } else {
- const rows = Math.min(16, (fld.value.match(/\n/g) || []).length + 2);
- html += `
`;
- }
- html += `
`;
- }
+ for (const fld of scalarFields) {
+ const attrs = `class="vv-set-input" data-key="${_vvSetEsc(fld.key)}" `
+ + `data-file="${_vvSetEsc(fld.file)}" data-type="${_vvSetEsc(fld.type)}" `
+ + `data-orig="${_vvSetEsc(fld.value)}" oninput="vvPtSetChanged(this)"`;
+ html += `
+
${_vvSetEsc(fld.key)}
`;
+ if (fld.desc) html += `
${_vvSetEsc(fld.desc)}
`;
+ html += `
`;
}
html += `
`;
}
- return html;
+ return html || '
No scalar settings available.
';
}
function vvPtToggleNode(hdr) {
@@ -491,47 +656,98 @@ function _syncToggle(gate, dimmed) {
}
function _renderSync(sync) {
- const jobs = sync.jobs || {};
- const gates = sync.gates || {};
+ const jobs = sync.jobs || {};
+ const gates = sync.gates || {};
+ const intervalMin = sync.interval_min || 30;
+
const sCol = s => s === 'ok' ? '#4caf50' : s === 'running' ? '#4a9eff'
: s === 'warn' ? '#ff9800' : s === 'error' ? '#f44336' : '#555';
- const sLbl = s => s === 'ok' ? '✓ ok' : s === 'running' ? '⟳ running'
- : s === 'warn' ? '⚠ warn' : s === 'error' ? '✗ error' : '— never';
+ const sLbl = s => s === 'ok' ? '✓' : s === 'running' ? '⟳' : s === 'warn' ? '⚠' : s === 'error' ? '✗' : '—';
+
+ function _dur(sec) {
+ if (!sec || sec < 0) return '';
+ if (sec < 60) return sec + 's';
+ if (sec < 3600) return Math.floor(sec / 60) + 'm ' + (sec % 60) + 's';
+ return Math.floor(sec / 3600) + 'h ' + Math.floor((sec % 3600) / 60) + 'm';
+ }
+
+ function _countdown(ts) {
+ if (!ts) return null;
+ const sec = ts - Math.floor(Date.now() / 1000);
+ if (sec <= 0) return 'due';
+ if (sec < 60) return 'in ' + sec + 's';
+ if (sec < 3600) return 'in ' + Math.floor(sec / 60) + 'm';
+ return 'in ' + Math.floor(sec / 3600) + 'h ' + Math.floor((sec % 3600) / 60) + 'm';
+ }
const globalOff = gates.global && !gates.global.on;
- // Master toggle row — controls all mirroring (RSYNC_ENABLED, Tier 1).
- let html = '';
+ // Wrap everything in a flex-column so the bottom section anchors naturally
+ let html = '
';
+
+ // Master toggle row
if (gates.global) {
- html += `
+ html += `
All mirroring
${_syncToggle(gates.global, false)}
`;
}
- // Per-tier rows: status + last-run + per-tier toggle (dimmed when global is off).
- const labels = {critical: 'Critical (30 min)', daily: 'Daily', weekly: 'Weekly'};
+ // Per-tier rows
+ const tierMeta = {
+ critical: {label: 'Critical', sched: 'every 30 min', desc: 'appdata · partnership · play state'},
+ daily: {label: 'Daily', sched: 'nightly', desc: 'media shares · arr sync · docker'},
+ weekly: {label: 'Weekly', sched: 'weekly', desc: 'bulk shares · updates · cleanup'},
+ };
+
+ // Critical next-run estimate
+ const critJob = jobs.critical;
+ const critNextTs = critJob && critJob.end && critJob.status !== 'running'
+ ? critJob.end + intervalMin * 60 : null;
+
+ html += '
';
+
for (const key of ['critical', 'daily', 'weekly']) {
- const j = jobs[key];
+ const j = jobs[key];
+ const meta = tierMeta[key];
const status = j ? j.status : null;
const when = j && j.start
- ? (j.status === 'running' ? 'started ' + _relTime(j.start)
- : _relTime(j.end || j.start))
- : '—';
- html += `
-
${labels[key]}
-
- ${sLbl(status)}
- ${when}
- ${_syncToggle(gates[key], globalOff)}
-
+ ? (j.status === 'running' ? 'started ' + _relTime(j.start) : _relTime(j.end || j.start))
+ : null;
+ const dur = j && j.start && j.end && j.status !== 'running' ? _dur(j.end - j.start) : '';
+ const summary = j && j.summary ? j.summary : '';
+
+ html += `
+
+ ${meta.label}
+ ${meta.sched}
+
+
+ ${sLbl(status)}
+ ${dur ? `${dur} ` : ''}
+ ${when ? `${when} ` : ''}
+ ${_syncToggle(gates[key], globalOff)}
+
+
+
${meta.desc}
+ ${summary ? `
${_vvSetEsc(summary)}
` : ''}
`;
}
+ html += '
'; // flex:1
+
+ // Bottom info strip
+ const nextLabel = critNextTs ? _countdown(critNextTs) : null;
+ html += `
+ interval: ${intervalMin}min
+ ${nextLabel ? `next critical ${nextLabel} ` : ''}
+
`;
+
if (globalOff) {
html += `
⚠ All mirroring is off — per-tier switches have no effect until re-enabled.
`;
}
+ html += '
'; // outer flex column
return html;
}
@@ -721,214 +937,280 @@ function _renderActions(nodes, cfg) {
let html = '';
+ // ── No partner ─────────────────────────────────────────────────────────────
if (!hasPartner && isOwner) {
- html += `
- No partner configured — edit master.conf (Scheduler tab) and set HOST2.
+ html += `
+
No partner host configured
+
Set HOST2 in master.conf (Host Settings above) to enable partnership.
`;
}
// ── Per-remote host section ────────────────────────────────────────────────
if (isOwner && hasPartner) {
- remotes.forEach(remote => {
- const phase = remote.onboard_phase ?? 0;
+ remotes.forEach((remote, idx) => {
+ const phase = remote.onboard_phase ?? 0;
const isOnboarding = !!_vvOnboarding[remote.id];
- const dotCol = remote.ts_online === null ? '#555' : remote.ts_online ? '#4caf50' : '#f44336';
- const dotLbl = remote.ts_online === null ? 'unknown' : remote.ts_online ? 'online' : 'offline';
- const pt = remote.partnership || {};
- const ptState = (pt.state || '').toUpperCase();
- const ptDate = pt.updated
+ const isDeleting = !!_vvDeleteKeys[remote.id];
+ const dotCol = remote.ts_online === null ? '#444' : remote.ts_online ? '#4caf50' : '#f44336';
+ const dotLbl = remote.ts_online === null ? 'unknown' : remote.ts_online ? 'online' : 'offline';
+ const pt = remote.partnership || {};
+ const ptState = (pt.state || '').toUpperCase();
+ const ptDate = pt.updated
? new Date(parseInt(pt.updated) * 1000).toLocaleDateString([], {month:'short',day:'numeric',year:'numeric'})
: null;
- const sys = remote.system || {};
- const sysUptime = sys.uptime_sec ? _uptime(sys.uptime_sec) : null;
- const sysVer = sys.unraid_version || null;
+ const sys = remote.system || {};
+ const sysUptime = sys.uptime_sec ? _uptime(sys.uptime_sec) : null;
+ const sysVer = sys.unraid_version || null;
- html += `
`;
+ // Left border colour by phase
+ const borderCol = phase === 2 ? '#2a5a2a'
+ : phase === 1 ? '#4a3000'
+ : isOnboarding ? '#0e2a4a' : '#222';
- // Header: host + online dot + test connection
- html += `
-
- ${remote.id}
- ${remote.hostname}
-
+ if (idx > 0) html += '
';
+
+ html += `
`;
+
+ // ── Card header ─────────────────────────────────────────────────────────
+ html += `
-
+ ${remote.id}
+ ${remote.hostname}
+ ● ${dotLbl}
+
+
+
⇄ Test
- ● ${dotLbl}
+ style="font-size:9px;padding:2px 7px;" title="SSH echo round-trip">⇄ ping
`;
- const isDeleting = !!_vvDeleteKeys[remote.id];
+ html += `
`;
- // ── Phase 0: not provisioned ─────────────────────────────────────────
+ // ── Phase 0: not provisioned ───────────────────────────────────────────
if (phase === 0) {
if (isOnboarding) {
- html += `
-
-
Step 1
-
Install SSH key on ${remote.hostname}
+ html += `
+
+
+ Step 1
+ Install SSH key on ${remote.hostname}
+
+
+
Enter ${remote.hostname} root password when prompted · tab auto-updates on completion
-
-
🖥 Open Terminal
-
${termCmd}
+
+
+ Step 2
+ Push conf
+ if key already installed separately
+
+
▶ Push Conf
-
Enter ${remote.hostname} root password when prompted · tab auto-updates on completion
-
- Step 2
- Push conf
- if key already installed separately
-
-
▶ Push Conf
-
-
✕ Close
+
+ ✕ Close
`;
} else {
- html += `
-
○ Not provisioned
-
▶ Onboard
-
🗑 Delete Keys
+ html += `
+
+
Not provisioned
+
SSH keys not installed · conf not pushed
+
+
+ ▶ Onboard
+ 🗑 Keys
+
`;
}
// ── Phase 1: SSH done, awaiting HOST2 ─────────────────────────────────
} else if (phase === 1) {
- html += `
- ⏳ SSH ready
- ${remote.ts_ip ? `${remote.ts_ip} ` : ''}
-
-
- Waiting for ${remote.id} to install Varaverk and complete its onboard. Phase 2 triggers automatically.
-
-
-
- ▶ Run Phase 2 Manually
-
-
✕ Cancel
-
🗑 Delete Keys
+ html += `
+
+
+ SSH ready
+ ${remote.ts_ip ? `${remote.ts_ip} ` : ''}
+
+
+ Waiting for ${remote.id} to install Varaverk and complete onboard. Phase 2 triggers automatically.
+
+
+
+ ▶ Phase 2
+ ✕ Cancel
+ 🗑 Keys
+
`;
// ── Phase 2: fully onboarded ───────────────────────────────────────────
} else {
const ptColor = ptState === 'ACTIVE' ? '#4caf50' : ptState === 'INACTIVE' ? '#f44336' : '#555';
- html += `
- ✅ Active
- ${ptDate ? `since ${ptDate} ` : ''}
-
-
- ${remote.ts_ip ? `${remote.ts_ip} ` : ''}
- ${sysUptime ? `up ${sysUptime} ` : ''}
- ${sysVer ? `unRAID ${sysVer} ` : ''}
- ${ptState ? `${ptState} ` : ''}
-
-
-
↻ Re-run Phase 2
-
🗑 Delete Keys
+ html += `
+
+
+ Active
+ ${ptState ? `${ptState} ` : ''}
+ ${ptDate ? `since ${ptDate} ` : ''}
+
+
+ ${remote.ts_ip ? `${remote.ts_ip} ` : ''}
+ ${sysVer ? `unRAID ${sysVer} ` : ''}
+ ${sysUptime ? `up ${sysUptime} ` : ''}
+
+
+
+ ↻ Re-run
+ 🗑 Keys
+
`;
}
- // ── Delete Keys panel (any phase) ──────────────────────────────────────
+ // ── Delete Keys panel ──────────────────────────────────────────────────
if (isDeleting) {
- html += `
-
Remove SSH access
-
+ html += `
+
Remove SSH access
+
-
HOST1 → ${remote.id}
-
HOST1's key on ${remote.hostname} · deletes local key pair
+
HOST1 → ${remote.id}
+
HOST1's key on ${remote.hostname} · deletes local pair
✕ Remove HOST1 key
+ style="white-space:nowrap;font-size:10px;">✕ HOST1 key
-
${remote.id} → HOST1
-
${remote.hostname}'s key on HOST1 · removes from authorized_keys
+
${remote.id} → HOST1
+
${remote.hostname}'s key from authorized_keys
✕ Remove ${remote.id} key
+ style="white-space:nowrap;font-size:10px;">✕ ${remote.id} key
-
+
✓ Done
+ style="font-size:10px;opacity:.6;">✓ Done
`;
}
- html += `
`;
+ html += `
`; // padding + card
});
}
- // ── HOST1 local setup (when needed after phase 1+) ────────────────────────
+ // ── HOST1 local setup ──────────────────────────────────────────────────────
if (isOwner && selfNode && !selfNode.local_done && remotes.some(r => (r.onboard_phase ?? 0) >= 1)) {
- html += `
-
- ${selfNode.id} — ${selfNode.hostname}
(this server)
+ html += `
+
+
${selfNode.id} — ${selfNode.hostname}
+
FolderView3 integration · marks HOST1 ready
-
-
- ▶ Complete HOST1 Setup
-
+ ▶ Complete Setup
+
`;
+ }
+
+ // ── Mirror onboard (non-owner) ─────────────────────────────────────────────
+ if (!isOwner) {
+ html += `
+
+
+
Join partnership
+
SSH key setup + notify owner
+
+
▶ Onboard
`;
}
- // ── Bottom row ─────────────────────────────────────────────────────────────
- html += `
`;
- if (!isOwner) {
- html += `
- ▶ Onboard (Mirror)
- `;
- }
- html += `
- ▶ Offboard
-
`;
+ // ── Offboard + Transfer ────────────────────────────────────────────────────
+ html += `
`;
+
+ html += `
+
+
Offboard
+
End partnership · revoke SSH · restore WebUIs
+
+
+ ▶ Offboard
+
+
`;
if (cfg.enabled && isOwner) {
- const tok = (cfg.transfer_confirm || 'i-understand-this-transfers-ownership')
- .replace(/'/g, "\\'");
- html += `
-
- Transfer ownership to the mirror — promotes it to owner.
-
- ⇄ Transfer Ownership
-
-
-
- Or run manually: bash Partnership/partnership_transfer.sh --confirm=${cfg.transfer_confirm || 'i-understand-this-transfers-ownership'}
+ const tok = (cfg.transfer_confirm || 'i-understand-this-transfers-ownership').replace(/'/g, "\\'");
+ html += `
+
+
Transfer ownership
+
Promotes mirror to owner · requires sustained health checks
+
+ ⇄ Transfer
+
+
+
+ bash Partnership/partnership_transfer.sh --confirm=${cfg.transfer_confirm || 'i-understand-this-transfers-ownership'}
`;
}
+ html += '
'; // offboard+transfer section
+
return html;
}
// ── Main render ───────────────────────────────────────────────────────────────
function _render(data) {
- const cfg = data.config || {};
- const nodes = data.nodes || [];
+ const cfg = data.config || {};
+ const nodes = data.nodes || [];
+ const isOwner = nodes.some(n => n.is_me && n.is_owner);
// Config bar
document.getElementById('vv-pt-config-body').innerHTML = _renderConfig(cfg);
+ // Host settings — always shown upfront
+ const hostsCard = document.getElementById('vv-pt-hosts-card');
+ hostsCard.style.display = nodes.length ? '' : 'none';
+ document.getElementById('vv-pt-hosts-body').innerHTML = _renderHostSettings(nodes, isOwner);
+ document.getElementById('vv-pt-hosts-save').style.display = 'none';
+
// Offline countdown warning
_renderOfflineWarn(cfg);
@@ -946,6 +1228,12 @@ function _render(data) {
// Actions
document.getElementById('vv-pt-actions-body').innerHTML = _renderActions(nodes, cfg);
+ // Array settings cards above Actions — load once
+ _vvInitArrayCards();
+
+ // Settings panel — host1 only
+ document.getElementById('vv-pt-settings-card').style.display = isOwner ? '' : 'none';
+
// Highlight Onboard button when arriving from first-run wizard
if (new URLSearchParams(location.search).get('vv_onboard') === '1') {
const onboardBtn = document.querySelector('#vv-pt-actions-body .vv-pt-action-btn.run');
diff --git a/Plugin/unraid/pages/rsync.php b/Plugin/unraid/pages/rsync.php
new file mode 100644
index 0000000..94ba7f4
--- /dev/null
+++ b/Plugin/unraid/pages/rsync.php
@@ -0,0 +1,1366 @@
+
+
+
+ Rsync
+
+
+
+
+
+
+
+ Manual Rsync
+
+
+
▾ Show
+
+
+
+
+
+
+
+
+
Local source
+
+
+ ↑
+ ⟳ Browse
+
+
+
+
trailing / = sync contents · without / = sync the folder
+
+
+
+
+
+
+
Remote server
+
+ Loading hosts…
+
+
+
+
+
+
Remote path
+
+
+ ↑
+ ⟳ Browse
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ ▶ Run Rsync
+ ▾ Show Rsync
+ ■ Stop
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Rsync Reference
+ ⚙ Profile Editor
+
+
+
+
+
+ ▾ How to use this page
+
+
+
+
Status card — shows whether the global rsync gate is open. Red = all syncs blocked regardless of tier toggles.
+
Sync windows — per-tier enable/disable. A tier can be enabled here but still blocked by the global gate.
+
Settings → Global gate — RSYNC_ENABLED in master.conf. Master switch; turn off during rebuilds or maintenance.
+
Settings → Tier toggles — CRITICAL/DAILY/etc._RSYNC_ENABLED. Disable one tier without affecting others.
+
Bandwidth history — last 30 days of per-profile rsync runs. Bar = relative time spent. Click recent runs for detail.
+
Manual Rsync — one-off transfer to any partner host. Browse remote dirs via SSH. Runs in background, output streams below.
+
Profile Editor — create and edit named profiles used by appdata syncs. Each profile controls flags, BW limit, container stop lists, and excludes.
+
+
+
+
+
+
+
+ ▸ Sync windows
+
+
+
+
Critical (30 min) — appdata (arrs, auth, databases), partnership health check, play state sync. Designed to be fast — uses dirty-sync profiles where possible.
+
Intermediate (4 hr) — arr sync between the nightly runs. Catches new indexer hits without waiting until midnight.
+
Daily (nightly) — media shares, arr maintenance, media cleaner, docker updates, daily container restarts.
+
Weekly — bulk media shares, update checks, weekly restarts, SMART health digest.
+
Fallback — fires when a server comes back online after being the active fallback. Syncs any changes that happened during downtime.
+
Two-tier toggle — global gate (RSYNC_ENABLED) blocks everything. Per-tier flags (e.g. DAILY_RSYNC_ENABLED) disable just that window. Both must be ON for syncs to run.
+
+
+
+
+
+
+
+ ▸ Profile system
+
+
+
+
What is a profile? — A named set of rsync options applied to a specific appdata directory. Appdata shares match a profile by their directory basename (lowercased).
+
PROFILE_RSYNC_OPTS — Full rsync flag string for this profile. Does NOT inherit DEFAULT_RSYNC_OPTS — list every desired flag explicitly. Use $BW_LIMIT to reference the profile's bandwidth limit.
+
PROFILE_BW_LIMIT — Bandwidth cap in KB/s. Overrides the global BW_LIMIT for this profile.
+
PROFILE_RETRY_COUNT / SLEEP — How many times to retry on failure, and how long to wait between attempts.
+
PROFILE_CRITICAL_CONTAINER_NAMES — Containers stopped on BOTH hosts before rsync. Local stops first (flush DBs), then remote. Only running containers get restarted — stopped ones stay stopped.
+
PROFILE_DELAYED_CONTAINERS — Containers that need a delay after the others start (e.g. NextCloud waits for Postgres).
+
PROFILE_CONTAINER_DELAY — Seconds to wait before starting delayed containers.
+
PROFILE_EXCLUDE_DIRS — Space-separated patterns excluded from the rsync. Supports globs (e.g. logs *.tmp *.db-wal).
+
PROFILE_REMOTE_RESTART_CONTAINERS — Used by dirty-sync profiles. Restarts these containers on the remote after sync so it picks up the new data.
+
Clean vs dirty sync — Clean: stop containers, --delete, full mirror. Dirty: containers keep running, WAL files excluded, --inplace --no-whole-file for safe partial writes. Use dirty for frequent critical syncs.
+
App layer — Scripts add their appdata directory to HOST*_CRITICAL_SYNC_SHARES or HOST*_DAILY_SYNC_SHARES using /path/to/dir|profile-name syntax. Creating the profile here makes it available for that reference.
+
+
+
+
+
+
+
+ ▸ Common flag combinations
+
+
+
+
Clean sync (weekly appdata) -av --human-readable --bwlimit=$BW_LIMIT --delete Full mirror — removes files on destination that no longer exist at source. Safe when containers are stopped.
+
Dirty sync (every 30 min) -av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file Containers stay running. --inplace writes directly to destination file (avoids temp-file rename issues). --no-whole-file uses delta transfer. Exclude WAL files to avoid corrupt state.
+
Database sync (arrs) -av --info=progress2 --human-readable --bwlimit=$BW_LIMIT --delete --inplace Stop containers first. --info=progress2 gives cleaner progress output in logs. Lower BW limit to not starve other syncs running concurrently.
+
Large media (weekly) -av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file High BW limit, inplace for large files, delete to keep mirror clean. No container stops — media files are read-only.
+
--delete — remove files at destination that no longer exist at source. Essential for a true mirror.
+
--inplace — write file data directly into destination file instead of creating a temp copy first. Faster for large files; required for some databases.
+
--no-whole-file — force delta transfer even over fast LAN. Reduces data sent for files that change incrementally.
+
--checksum — compare files by checksum rather than mtime+size. Slower but catches any corruption. Use for archive syncs, not frequent syncs.
+
+
+
+
+
+
+
+
+
← Rsync Info
+
Profile Editor
+
+
+ Loading…
+
+ + New
+
+
+
+
+
+ Select a profile above to edit, or click + New to create one.
+
+
+
+
+
+
+
+
+
diff --git a/Plugin/unraid/pages/scheduler.php b/Plugin/unraid/pages/scheduler.php
index 98d59db..704b2be 100644
--- a/Plugin/unraid/pages/scheduler.php
+++ b/Plugin/unraid/pages/scheduler.php
@@ -466,16 +466,6 @@ $runningScripts = array_unique($runningScripts);
Child (orch OFF) — enter a cron to run it standalone
Rsync badge — toggle writes TIER_RSYNC_ENABLED directly to master.conf
Rsync (orch OFF) — fill location + standalone cron + Save; both required for independent firing
-
Editor Keyboard Shortcuts
-
Ctrl+S — save · Ctrl+Z / Ctrl+Y — undo / redo (200 steps)
-
Ctrl+/ — toggle line comment · Tab / Shift+Tab — indent / dedent
-
Ctrl+F — find · Ctrl+H — find & replace · Ctrl+G — go to line
-
Ctrl+D — select next occurrence · Ctrl+L — select line · Ctrl+Shift+K — delete line
-
Alt+↑/↓ — move line · Alt+Shift+↓ — duplicate line below
-
Home — smart (first non-space → col 0)
-
( { [ " ` — auto-close pair; wraps selection; Backspace deletes both when between them
-
A− / A+ in status bar — font size (9–22px, persists across sessions)
-
Indent guides — faint vertical lines every 2 spaces in the background
@@ -909,6 +899,33 @@ Still the same two servers, two households, the same media stack running itself.
×
+
+
+
+
+
+ Ctrl+S — save
+ Ctrl+Z / Ctrl+Y — undo / redo (200 steps)
+ Ctrl+/ — toggle line comment
+ Tab / Shift+Tab — indent / dedent
+ Ctrl+F — find
+ Ctrl+H — find & replace
+ Ctrl+G — go to line
+ Ctrl+D — select next occurrence
+ Ctrl+L — select line
+ Ctrl+Shift+K — delete line
+ Alt+↑/↓ — move line
+ Alt+Shift+↓ — duplicate line below
+ Home — smart (first non-space → col 0)
+ ( { [ " ` — auto-close pair; wraps selection; Backspace deletes both
+ A− / A+ in status bar — font size (9–22px, persists)
+
+
+
+
diff --git a/Watchdogs/docker_watchdog.sh b/Watchdogs/docker_watchdog.sh
index 8a1b9c3..93b527b 100755
--- a/Watchdogs/docker_watchdog.sh
+++ b/Watchdogs/docker_watchdog.sh
@@ -130,11 +130,10 @@
# STATE FILES
# ==============================================================================================
#
-# WATCHDOG_STATE_FILE — strike counts, daemon flags (STATE_DIR — survives reboots)
+# WATCHDOG_STATE_FILE — strike counts, daemon flags (STATE_DIR — survives reboots)
# DOCKER_WATCHDOG_FAILED_FILE — container skip list (STATE_DIR — survives reboots)
-# DOCKER_WATCHDOG_MANUAL_STOP_FILE — intentionally stopped containers (STATE_DIR — survives reboots)
-# WATCHDOG_CONTAINER_RESTART_LOG — restart history for loop detection (DATA_DIR)
-# RW_STATE_FILE — read-only: resource_watchdog RAM emergency flag
+# WATCHDOG_CONTAINER_RESTART_LOG — restart history for loop detection (DATA_DIR)
+# RW_STATE_FILE — read-only: resource_watchdog RAM emergency flag
#
# ==============================================================================================
# CONFIGURATION
@@ -150,6 +149,11 @@
# HTTP health check endpoints. Format: "ContainerName:http://host:port"
# Aliased by detect_hosts() → WATCHDOG_CONTAINER_URLS
#
+# HOST*_WATCHDOG_CONTAINER_API_CHECKS
+# API liveness checks — deeper than HTTP. Format: "ContainerName:URL|APIKey"
+# Use endpoints that require a live DB round-trip (e.g. Emby /System/Info).
+# Aliased by detect_hosts() → WATCHDOG_CONTAINER_API_CHECKS
+#
# HOST*_WATCHDOG_REQUIRED_CONTAINERS
# Containers that must always be running. Aliased by detect_hosts() →
# WATCHDOG_REQUIRED_CONTAINERS
@@ -258,7 +262,7 @@ validate_unraid_cmd "/usr/local/emhttp/plugins/dynamix/scripts/notify" "
# Ensure state files exist
touch "$WATCHDOG_STATE_FILE" "$WATCHDOG_CONTAINER_RESTART_LOG" \
- "$DOCKER_WATCHDOG_FAILED_FILE" "$DOCKER_WATCHDOG_MANUAL_STOP_FILE" 2>/dev/null
+ "$DOCKER_WATCHDOG_FAILED_FILE" 2>/dev/null
# Timeout for all docker commands — configurable via WATCHDOG_DAEMON_TIMEOUT in master.conf
DOCKER_TIMEOUT="${WATCHDOG_DAEMON_TIMEOUT:-20}"
@@ -274,8 +278,6 @@ if [[ "$SHOW_STATUS" == true ]]; then
echo "$ICON_CONTAINERS Required: ${WATCHDOG_REQUIRED_CONTAINERS[*]:-none}"
echo "$ICON_WATCHDOG Scan all: $WATCHDOG_SCAN_ALL"
echo "$ICON_WATCHDOG Ignore: ${WATCHDOG_SCAN_IGNORE[*]:-none}"
- _ms_list=$(cat "$DOCKER_WATCHDOG_MANUAL_STOP_FILE" 2>/dev/null | tr '\n' ' ' | xargs)
- echo "$ICON_WATCHDOG Manual-stop: ${_ms_list:-none}"
echo "$ICON_WATCHDOG Schedule: every 15 min (cron via watchdog_orchestrator)"
echo "$ICON_WATCHDOG Startup grace: ${WATCHDOG_STARTUP_GRACE}s"
echo "$ICON_WATCHDOG Restart limit: $WATCHDOG_CONTAINER_RESTART_LIMIT in ${WATCHDOG_CONTAINER_RESTART_WINDOW}h"
@@ -330,28 +332,6 @@ remove_from_skip_list() {
warn "$1 recovered — removed from skip list ✅"
}
-# Check if container was intentionally stopped (exit 0/143 — clean/SIGTERM)
-is_manually_stopped() {
- grep -q "^${1}$" "$DOCKER_WATCHDOG_MANUAL_STOP_FILE" 2>/dev/null
-}
-
-# Mark container as intentionally stopped — auto-cleared when seen running again
-add_to_manual_stop() {
- local container="$1" exit_code="$2"
- if ! is_manually_stopped "$container"; then
- echo "$container" >> "$DOCKER_WATCHDOG_MANUAL_STOP_FILE"
- log "$container — stopped cleanly (exit $exit_code) — skipping until restarted"
- fi
-}
-
-# Remove container from manual-stop list — called when container is seen running again
-remove_from_manual_stop() {
- if is_manually_stopped "$1"; then
- sed -i "/^${1}$/d" "$DOCKER_WATCHDOG_MANUAL_STOP_FILE" 2>/dev/null
- log "$1 — running again — removed from manual-stop list ✅"
- fi
-}
-
# Log a restart event to the rolling restart history file
log_restart() {
local container="$1"
@@ -656,10 +636,6 @@ CYCLE_START=$(date +%s)
_skip_contents=$(cat "$DOCKER_WATCHDOG_FAILED_FILE" 2>/dev/null | tr '\n' ' ' | xargs)
[[ -n "$_skip_contents" ]] && warn "$ICON_SKIP Skip list active: $_skip_contents — manual intervention needed"
- local _manual_stop_contents
- _manual_stop_contents=$(cat "$DOCKER_WATCHDOG_MANUAL_STOP_FILE" 2>/dev/null | tr '\n' ' ' | xargs)
- [[ -n "$_manual_stop_contents" ]] && log "$ICON_SKIP Manual-stop list: $_manual_stop_contents"
-
# ── Docker daemon health check — first check every run ──────────────────────────────────
# If daemon is hung all container operations will fail — check first, skip run if down
if ! check_docker_daemon; then
@@ -716,36 +692,26 @@ CYCLE_START=$(date +%s)
fi
if [[ "$STATUS" == "true" ]]; then
- # Running — clear any strikes and manual-stop flag
- remove_from_manual_stop "$container"
+ # Running — clear any strikes
set_strikes "$container" 0 "$WATCHDOG_STATE_FILE"
log "$ICON_RUNNING $container — running ✅"
- elif is_manually_stopped "$container"; then
- log "$container — manually stopped — skipping"
else
- # Check if this was a clean/intentional stop before striking
- EXIT_CODE=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
- '{{.State.ExitCode}}' "$container" 2>/dev/null || echo "-1")
- if [[ "$EXIT_CODE" == "0" || "$EXIT_CODE" == "143" ]]; then
- add_to_manual_stop "$container" "$EXIT_CODE"
- else
- STRIKES=$(get_strikes "$container" "$WATCHDOG_STATE_FILE")
- STRIKES=$(( STRIKES + 1 ))
- set_strikes "$container" "$STRIKES" "$WATCHDOG_STATE_FILE"
- warn "$container — not running (strike $STRIKES/$SYS_WATCHDOG_STRIKE_LIMIT)"
- ((T1_WARNINGS++))
+ STRIKES=$(get_strikes "$container" "$WATCHDOG_STATE_FILE")
+ STRIKES=$(( STRIKES + 1 ))
+ set_strikes "$container" "$STRIKES" "$WATCHDOG_STATE_FILE"
+ warn "$container — not running (strike $STRIKES/$SYS_WATCHDOG_STRIKE_LIMIT)"
+ ((T1_WARNINGS++))
- if [[ "$STRIKES" -ge "$SYS_WATCHDOG_STRIKE_LIMIT" ]]; then
- result=0
- safe_restart "$container" "required container down" || result=$?
- case $result in
- 0) set_strikes "$container" 0 "$WATCHDOG_STATE_FILE"
- ((T1_RESTARTS++))
- queue_notify "$container was down and restarted on $(hostname)" "warning" ;;
- 2) : ;; # Added to skip list — already notified
- *) queue_notify "$container failed to restart on $(hostname)" "warning" ;;
- esac
- fi
+ if [[ "$STRIKES" -ge "$SYS_WATCHDOG_STRIKE_LIMIT" ]]; then
+ result=0
+ safe_restart "$container" "required container down" || result=$?
+ case $result in
+ 0) set_strikes "$container" 0 "$WATCHDOG_STATE_FILE"
+ ((T1_RESTARTS++))
+ queue_notify "$container was down and restarted on $(hostname)" "warning" ;;
+ 2) : ;; # Added to skip list — already notified
+ *) queue_notify "$container failed to restart on $(hostname)" "warning" ;;
+ esac
fi
fi
done
@@ -843,6 +809,54 @@ CYCLE_START=$(date +%s)
done
fi
+ # ── API liveness checks ───────────────────────────────────────────────────────────────────
+ # Catches containers that serve HTTP 200 but are internally frozen (DB lock, deadlocked
+ # thread, etc.). Endpoint must require a live DB round-trip to respond successfully.
+ # Format per entry: "URL|APIKey"
+ if [[ ${#WATCHDOG_CONTAINER_API_CHECKS[@]} -gt 0 ]]; then
+ for container in "${!WATCHDOG_CONTAINER_API_CHECKS[@]}"; do
+ IFS='|' read -r _api_url _api_key <<< "${WATCHDOG_CONTAINER_API_CHECKS[$container]}"
+
+ # Skip entirely if key is absent or a placeholder — check is optional protection
+ if [[ -z "$_api_key" || "$_api_key" == "YOUR_API_KEY"* || "$_api_key" == "placeholder"* ]]; then
+ log "$container — API check skipped (no key configured)"
+ continue
+ fi
+
+ _api_http=$(curl -s --max-time "$CURL_TIMEOUT" \
+ -H "X-Emby-Token: ${_api_key}" \
+ -o /tmp/_varaverk_api_check \
+ -w "%{http_code}" \
+ "$_api_url" 2>/dev/null)
+ _resp=$(cat /tmp/_varaverk_api_check 2>/dev/null)
+
+ # 401/403 = wrong key — skip silently, don't penalise the container
+ if [[ "$_api_http" == "401" || "$_api_http" == "403" ]]; then
+ log "$container — API check skipped (HTTP $_api_http — key may be wrong or revoked)"
+ continue
+ fi
+
+ if echo "$_resp" | jq -e '.ServerName // .Id // .Version' >/dev/null 2>&1; then
+ set_strikes "${container}_api" 0 "$WATCHDOG_STATE_FILE"
+ else
+ API_STRIKES=$(get_strikes "${container}_api" "$WATCHDOG_STATE_FILE")
+ API_STRIKES=$(( API_STRIKES + 1 ))
+ set_strikes "${container}_api" "$API_STRIKES" "$WATCHDOG_STATE_FILE"
+ warn "$container — API unresponsive at $_api_url (strike $API_STRIKES/$RESP_FAIL_LIMIT)"
+ ((T1_WARNINGS++))
+ if [[ "$API_STRIKES" -ge "$RESP_FAIL_LIMIT" ]]; then
+ result=0
+ safe_restart "$container" "API unresponsive at $_api_url" || result=$?
+ if [[ $result -eq 0 ]]; then
+ set_strikes "${container}_api" 0 "$WATCHDOG_STATE_FILE"
+ ((T1_RESTARTS++))
+ queue_notify "$container API unresponsive at $_api_url on $(hostname) — restarted" "warning"
+ fi
+ fi
+ fi
+ done
+ fi
+
# ==========================================================================================
# ── TIER 2 — Global Health Scan ───────────────────────────────────────────────────────────
# ==========================================================================================
@@ -850,13 +864,6 @@ CYCLE_START=$(date +%s)
ALL_CONTAINERS=$(timeout "$DOCKER_TIMEOUT" docker ps --format "{{.Names}}" 2>/dev/null)
- # ── Auto-clear manual-stop list ───────────────────────────────────────────────────────
- # Any container now running was started intentionally — remove from manual-stop list
- while IFS= read -r container; do
- [[ -z "$container" ]] && continue
- remove_from_manual_stop "$container"
- done <<< "$ALL_CONTAINERS"
-
# ── Unhealthy containers ──────────────────────────────────────────────────────────────
if [[ "$WATCHDOG_RESTART_UNHEALTHY" == "true" ]]; then
UNHEALTHY=$(timeout "$DOCKER_TIMEOUT" docker ps \
@@ -961,13 +968,13 @@ CYCLE_START=$(date +%s)
fi
# ── Unexpected exits ──────────────────────────────────────────────────────────────────
- # Exit 0/143 = clean/SIGTERM — intentional stop, add to manual-stop list and skip.
- # All other non-zero exits = crash — restart via safe_restart.
+ # Only non-zero exit codes — exit 0 is a clean stop, not a crash
# Skips containers already covered by WATCHDOG_REQUIRED_CONTAINERS (handled in Tier 1)
if [[ "$WATCHDOG_RESTART_CRASHED" == "true" ]]; then
- EXITED=$(timeout "$DOCKER_TIMEOUT" docker ps -a \
+ CRASHED=$(timeout "$DOCKER_TIMEOUT" docker ps -a \
--filter status=exited \
- --format "{{.Names}}|{{.Status}}" 2>/dev/null)
+ --format "{{.Names}}|{{.Status}}" 2>/dev/null | \
+ grep -v "Exited (0)")
while IFS='|' read -r container status; do
[[ -z "$container" ]] && continue
[[ -n "${IGNORE_MAP[$container]:-}" ]] && continue
@@ -978,20 +985,6 @@ CYCLE_START=$(date +%s)
[[ "$container" == "$req" ]] && already_required=true && break
done
[[ "$already_required" == true ]] && continue
-
- # Extract exit code from status string e.g. "Exited (143) 2 hours ago"
- EXIT_CODE="-1"
- [[ "$status" =~ Exited\ \(([0-9]+)\) ]] && EXIT_CODE="${BASH_REMATCH[1]}"
-
- # Clean/intentional stop — add to manual-stop list and skip
- if [[ "$EXIT_CODE" == "0" || "$EXIT_CODE" == "143" ]]; then
- add_to_manual_stop "$container" "$EXIT_CODE"
- continue
- fi
-
- # Already known to be manually stopped from a prior cycle
- is_manually_stopped "$container" && continue
-
error "$container — $status (unexpected exit)"
((T2_WARNINGS++))
result=0
@@ -1000,7 +993,7 @@ CYCLE_START=$(date +%s)
((T2_RESTARTS++))
queue_notify "$container crashed on $(hostname) ($status) — restarted" "warning"
fi
- done <<< "$EXITED"
+ done <<< "$CRASHED"
fi
fi # WATCHDOG_SCAN_ALL
diff --git a/common.sh b/common.sh
index 8841067..e8a3f7a 100755
--- a/common.sh
+++ b/common.sh
@@ -600,6 +600,7 @@ detect_hosts() {
_alias_assoc "WATCHDOG_CONTAINERS"
_alias_assoc "WATCHDOG_CONTAINER_URLS"
+ _alias_assoc "WATCHDOG_CONTAINER_API_CHECKS"
_alias_assoc "WATCHDOG_DEPENDENCIES"
_alias_assoc "WATCHDOG_APPDATA_SIZES"