From de6fcc399725c7490979ed69814fd21048e24522 Mon Sep 17 00:00:00 2001 From: Gmer4Lfe Date: Wed, 3 Jun 2026 15:58:50 -0400 Subject: [PATCH] Add Rsync page, upgrade monitor rsync card, partnership overhaul, API key periodic check, docker watchdog manual-stop detection --- Media/play_state_sync.sh | 35 +- Plugin/unraid/Varaverk.page | 4 +- Plugin/unraid/api/manual_sync.php | 241 +++++ Plugin/unraid/api/rsync.php | 47 + Plugin/unraid/api/rsync_profiles.php | 132 +++ Plugin/unraid/include/monitor.php | 28 +- Plugin/unraid/include/partnership.php | 30 +- Plugin/unraid/pages/monitor.php | 165 ++- Plugin/unraid/pages/partnership.php | 640 ++++++++---- Plugin/unraid/pages/rsync.php | 1366 +++++++++++++++++++++++++ Plugin/unraid/pages/scheduler.php | 37 +- Watchdogs/docker_watchdog.sh | 163 ++- common.sh | 1 + 13 files changed, 2546 insertions(+), 343 deletions(-) create mode 100644 Plugin/unraid/api/manual_sync.php create mode 100644 Plugin/unraid/api/rsync.php create mode 100644 Plugin/unraid/api/rsync_profiles.php create mode 100644 Plugin/unraid/pages/rsync.php diff --git a/Media/play_state_sync.sh b/Media/play_state_sync.sh index caf6b63..37141da 100755 --- a/Media/play_state_sync.sh +++ b/Media/play_state_sync.sh @@ -435,29 +435,19 @@ for lname in "${!USER_MAP[@]}"; do if [[ -z "$_iid" ]]; then _ptype="${_pkey%%:*}" _pval="${_pkey##*:}" - _search_field="" - case "$_ptype" in imdb) _search_field="imdb.${_pval}" ;; tmdb) _search_field="tmdb.${_pval##movie:}" ;; tvdb) - # AnyProviderIdEquals is unreliable for TVDB in Jellyfin 10.x — - # returns the entire library. Search by season+episode instead, - # then validate by ProviderIds.Tvdb. - # pkey format: tvdb:ep:TVDB_EP_ID:sSEASONePEP - _tvdb_ep_id="${_pkey#tvdb:ep:}" - _tvdb_ep_id="${_tvdb_ep_id%%:*}" - if [[ "$_pval" =~ ^s([0-9]+)e([0-9]+)$ ]]; then - _iid=$(_api_get "${SRV_URL[$_si]}" "${SRV_KEY[$_si]}" \ - "Items?IncludeItemTypes=Episode&ParentIndexNumber=${BASH_REMATCH[1]}&IndexNumber=${BASH_REMATCH[2]}&Recursive=true&Fields=ProviderIds&Limit=500" 2>/dev/null \ - | jq -r --arg tvdb "$_tvdb_ep_id" \ - '.Items[] | select(.ProviderIds.Tvdb == $tvdb) | .Id' 2>/dev/null | head -1) - fi + # _pkey format: tvdb:ep:{tvdb_id}:s{season}e{ep} + # ##*: gives "s7e2" (wrong); strip prefix then first : + _tvdb_num="${_pkey#tvdb:ep:}"; _tvdb_num="${_tvdb_num%%:*}" + _search_field="tvdb.${_tvdb_num}" ;; - mb) : ;; # skip music if not found + mb) _search_field="" ;; # skip music if not found esac - if [[ -z "$_iid" && -n "$_search_field" ]]; then + if [[ -n "$_search_field" ]]; then _iid=$(_api_get "${SRV_URL[$_si]}" "${SRV_KEY[$_si]}" \ "Items?AnyProviderIdEquals=${_search_field}&Recursive=true&Fields=ProviderIds&Limit=1" 2>/dev/null \ | jq -r '.Items[0].Id // empty' 2>/dev/null) @@ -477,8 +467,17 @@ for lname in "${!USER_MAP[@]}"; do if [[ "$_auth_played" == "true" ]]; then # Mark as played with date _date_param="" - [[ "$_auth_lplayed" != "null" && -n "$_auth_lplayed" ]] && \ - _date_param="?DatePlayed=${_auth_lplayed//[: ]/%3A}" + if [[ "$_auth_lplayed" != "null" && -n "$_auth_lplayed" ]]; then + # Emby's PlayedItems endpoint rejects 7-digit fractional seconds (.0000000) + # with HTTP 500; strip to whole seconds before encoding + if [[ "$_auth_lplayed" == *.* ]]; then + _lp="${_auth_lplayed%.*}" + [[ "$_auth_lplayed" == *Z ]] && _lp+="Z" + else + _lp="$_auth_lplayed" + fi + _date_param="?DatePlayed=${_lp//[: ]/%3A}" + fi _http=$(_api_post "${SRV_URL[$_si]}" "${SRV_KEY[$_si]}" \ "Users/${_uid}/PlayedItems/${_iid}${_date_param}") if [[ "$_http" == "200" || "$_http" == "201" ]]; then diff --git a/Plugin/unraid/Varaverk.page b/Plugin/unraid/Varaverk.page index 82e978e..fa7afd8 100644 --- a/Plugin/unraid/Varaverk.page +++ b/Plugin/unraid/Varaverk.page @@ -24,9 +24,9 @@ unset($_master, $_h1m, $_host1_blank, $_my_hostid, $_conf_missing); // Determine active tab $tab = $_GET['tab'] ?? 'monitor'; -$validTabs = ['monitor', 'scheduler', 'docker', 'watchdog', 'partnership', 'fallback', 'arrs']; +$validTabs = ['monitor', 'scheduler', 'docker', 'watchdog', 'partnership', 'fallback', 'arrs', 'rsync']; if (!in_array($tab, $validTabs)) $tab = 'monitor'; -$tabLabels = ['monitor' => 'Monitor', 'scheduler' => 'Scheduler', 'docker' => 'Docker', 'watchdog' => 'Watchdog', 'partnership' => 'Partnership', 'fallback' => 'FallBack', 'arrs' => 'Arrs']; +$tabLabels = ['monitor' => 'Monitor', 'scheduler' => 'Scheduler', 'docker' => 'Docker', 'watchdog' => 'Watchdog', 'partnership' => 'Partnership', 'fallback' => 'FallBack', 'arrs' => 'Arrs', 'rsync' => 'Rsync']; ?> diff --git a/Plugin/unraid/api/manual_sync.php b/Plugin/unraid/api/manual_sync.php new file mode 100644 index 0000000..cfcf878 --- /dev/null +++ b/Plugin/unraid/api/manual_sync.php @@ -0,0 +1,241 @@ + $host, 'ssh_key' => $sshKey]; +} + +function _ms_target(string $slot, string $sshKey): array { + $vars = vv_conf_vars(); + $hostname = $vars[strtoupper($slot)] ?? ''; + if (!$hostname) return ['ok' => false, 'error' => 'Unknown host: ' . $slot]; + if (!$sshKey || !file_exists($sshKey)) + return ['ok' => false, 'error' => 'SSH key not configured on this host']; + $ip = vv_resolve_tailscale_ip($hostname); + if (!$ip) return ['ok' => false, 'error' => 'Cannot reach ' . $hostname . ' via Tailscale']; + return ['ok' => true, 'ip' => $ip, 'hostname' => $hostname]; +} + +$action = trim($_GET['action'] ?? $_POST['action'] ?? ''); + +// ── hosts ───────────────────────────────────────────────────────────────────── +if ($action === 'hosts') { + ['host' => $current, 'ssh_key' => $sshKey] = _ms_caller(); + $all = vv_arr_known_hosts(); + $peers = vv_pt_ts_peers(); + $out = []; + foreach ($all as $slot => $hostname) { + if ($slot === $current) continue; + $label = strtolower($hostname); + $ts = $peers[$label] ?? []; + $out[] = [ + 'slot' => $slot, + 'id' => strtoupper($slot), + 'hostname' => $hostname, + 'online' => $ts['online'] ?? null, + 'ip' => $ts['ip'] ?? null, + ]; + } + echo json_encode(['ok' => true, 'hosts' => $out, 'has_key' => !empty($sshKey) && file_exists($sshKey), 'ssh_key' => $sshKey]); + exit; +} + +// ── browse remote (SSH) ─────────────────────────────────────────────────────── +if ($action === 'browse') { + $slot = trim($_GET['host'] ?? ''); + $path = trim($_GET['path'] ?? '/mnt/user'); + if (!preg_match('#^/[^\0]*$#', $path) || str_contains($path, '..')) { + echo json_encode(['ok' => false, 'error' => 'Invalid path']); exit; + } + ['ssh_key' => $sshKey] = _ms_caller(); + $t = _ms_target($slot, $sshKey); + if (!$t['ok']) { echo json_encode($t); exit; } + + $clean = rtrim($path, '/') ?: '/'; + $out = vv_pt_ssh($t['ip'], $sshKey, + 'find ' . escapeshellarg($clean) . ' -maxdepth 1 -mindepth 1 -type d 2>/dev/null', + 8); + $dirs = array_values(array_filter(array_map('trim', explode("\n", $out)))); + sort($dirs); + $dirs = array_slice($dirs, 0, 200); + // If empty, do a quick SSH echo to distinguish "no dirs" from "SSH failed" + if (empty($dirs)) { + $ping = trim(vv_pt_ssh($t['ip'], $sshKey, 'echo ok', 4)); + if ($ping !== 'ok') { + echo json_encode(['ok' => false, 'error' => 'SSH connection failed to ' . $t['hostname']]); + exit; + } + } + $parent = ($clean !== '/') ? (dirname($clean) ?: '/') : null; + echo json_encode(['ok' => true, 'path' => $clean, 'dirs' => $dirs, 'parent' => $parent]); + exit; +} + +// ── browse local ────────────────────────────────────────────────────────────── +if ($action === 'browse_local') { + $path = trim($_GET['path'] ?? '/mnt/user'); + if (!preg_match('#^/[^\0]*$#', $path) || str_contains($path, '..')) { + echo json_encode(['ok' => false, 'error' => 'Invalid path']); exit; + } + $clean = rtrim($path, '/') ?: '/'; + if (!is_dir($clean)) { + echo json_encode(['ok' => false, 'error' => 'Not a directory: ' . $clean]); exit; + } + $out = shell_exec('find ' . escapeshellarg($clean) . ' -maxdepth 1 -mindepth 1 -type d 2>/dev/null | sort | head -200') ?: ''; + $dirs = array_values(array_filter(array_map('trim', explode("\n", $out)))); + $parent = ($clean !== '/') ? (dirname($clean) ?: '/') : null; + echo json_encode(['ok' => true, 'path' => $clean, 'dirs' => $dirs, 'parent' => $parent]); + exit; +} + +// ── run (background, returns token) ────────────────────────────────────────── +if ($_SERVER['REQUEST_METHOD'] === 'POST' && $action === 'run') { + $local = trim($_POST['local'] ?? ''); + $slot = trim($_POST['host'] ?? ''); + $remotePath = trim($_POST['remote_path'] ?? ''); + $user = preg_replace('/[^a-z0-9_.-]/i', '', trim($_POST['user'] ?? 'root')) ?: 'root'; + $bwLimit = max(0, (int)($_POST['bw_limit'] ?? 0)); + $useKey = ($_POST['use_key'] ?? '1') !== '0'; + $rawFlags = trim($_POST['flags'] ?? ''); + $flags = $rawFlags !== '' ? preg_replace('/[`$!|&;><(){}\[\]\\\\]/', '', $rawFlags) : '-av --stats'; + + foreach (['local' => $local, 'host' => $slot, 'remote_path' => $remotePath] as $f => $v) { + if (!$v) { echo json_encode(['ok' => false, 'error' => 'Missing: ' . $f]); exit; } + } + foreach ([$local, $remotePath] as $p) { + if (!str_starts_with($p, '/') || str_contains($p, '..') || preg_match('/[\x00\n\r`$]/', $p)) { + echo json_encode(['ok' => false, 'error' => 'Invalid path: ' . $p]); exit; + } + } + + // ── Safeguards ──────────────────────────────────────────────────────────── + + // Block syncing from/to dangerous system paths + $blocked = ['/', '/proc', '/sys', '/dev', '/run', '/etc', '/bin', '/sbin', + '/usr', '/lib', '/lib64', '/boot/EFI', '/tmp']; + foreach ($blocked as $b) { + if (rtrim($local, '/') === $b) { + echo json_encode(['ok' => false, 'error' => 'Refusing to sync from system path: ' . $b]); exit; + } + if (rtrim($remotePath, '/') === $b) { + echo json_encode(['ok' => false, 'error' => 'Refusing to sync to system path: ' . $b]); exit; + } + } + + // Local source must exist + if (!file_exists($local)) { + echo json_encode(['ok' => false, 'error' => 'Local source does not exist: ' . $local]); exit; + } + + // Prevent concurrent manual syncs + foreach (glob('/tmp/vv_ms_*.pid') ?: [] as $pf) { + $pid = (int)trim(@file_get_contents($pf) ?: '0'); + if ($pid > 0 && file_exists('/proc/' . $pid)) { + echo json_encode(['ok' => false, 'error' => 'Another manual sync is already running — stop it first.']); exit; + } + @unlink($pf); // stale + } + + ['ssh_key' => $sshKey] = _ms_caller(); + $t = _ms_target($slot, $sshKey); + if (!$t['ok']) { echo json_encode($t); exit; } + + // Remote destination must exist + $destCheck = trim(vv_pt_ssh($t['ip'], $sshKey, + 'test -d ' . escapeshellarg($remotePath) . ' && echo ok || echo missing', 5)); + if ($destCheck === 'missing') { + echo json_encode(['ok' => false, 'error' => 'Remote destination does not exist: ' . $remotePath]); exit; + } + if ($destCheck !== 'ok') { + // SSH check inconclusive — log warning but proceed; rsync will fail cleanly if needed + // (e.g. Tailscale not yet up, rsync error will surface in output) + } + + // ── Build and launch ────────────────────────────────────────────────────── + + $token = bin2hex(random_bytes(8)); + $logFile = '/tmp/vv_ms_' . $token . '.log'; + $pidFile = '/tmp/vv_ms_' . $token . '.pid'; + + if ($useKey && $sshKey && file_exists($sshKey)) { + $sshOpts = 'ssh -i ' . escapeshellarg($sshKey) . ' -o StrictHostKeyChecking=no -o BatchMode=yes -o ConnectTimeout=10'; + } else { + $sshOpts = 'ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10'; + } + if ($bwLimit) $flags .= ' --bwlimit=' . (int)$bwLimit; + + $src = escapeshellarg(rtrim($local, '/') . '/'); + $dst = escapeshellarg($user . '@' . $t['ip'] . ':' . rtrim($remotePath, '/') . '/'); + + // Start rsync in background, capture its PID for stop support + $inner = "rsync $flags -e " . escapeshellarg($sshOpts) . " $src $dst >> " . escapeshellarg($logFile) . " 2>&1 &" + . " RSYNC_PID=\$!;" + . " echo \$RSYNC_PID > " . escapeshellarg($pidFile) . ";" + . " wait \$RSYNC_PID;" + . " echo __DONE__ >> " . escapeshellarg($logFile) . ";" + . " rm -f " . escapeshellarg($pidFile); + shell_exec('nohup bash -c ' . escapeshellarg($inner) . ' &>/dev/null &'); + + echo json_encode(['ok' => true, 'token' => $token]); + exit; +} + +// ── stop ────────────────────────────────────────────────────────────────────── +if ($_SERVER['REQUEST_METHOD'] === 'POST' && $action === 'stop') { + $token = preg_replace('/[^a-f0-9]/', '', trim($_POST['token'] ?? '')); + if (!$token || strlen($token) !== 16) { + echo json_encode(['ok' => false, 'error' => 'Invalid token']); exit; + } + $pidFile = '/tmp/vv_ms_' . $token . '.pid'; + $logFile = '/tmp/vv_ms_' . $token . '.log'; + + $pid = (int)trim(@file_get_contents($pidFile) ?: '0'); + if ($pid > 0 && file_exists('/proc/' . $pid)) { + shell_exec('kill -TERM ' . $pid . ' 2>/dev/null'); + usleep(400000); // 400ms grace + if (file_exists('/proc/' . $pid)) shell_exec('kill -KILL ' . $pid . ' 2>/dev/null'); + } + @unlink($pidFile); + @file_put_contents($logFile, "\n\n--- Cancelled by user ---\n__DONE__\n", FILE_APPEND); + echo json_encode(['ok' => true]); + exit; +} + +// ── poll ────────────────────────────────────────────────────────────────────── +if ($action === 'poll') { + $token = preg_replace('/[^a-f0-9]/', '', trim($_GET['token'] ?? '')); + if (!$token || strlen($token) !== 16) { + echo json_encode(['ok' => false, 'error' => 'Invalid token']); exit; + } + $logFile = '/tmp/vv_ms_' . $token . '.log'; + $pidFile = '/tmp/vv_ms_' . $token . '.pid'; + if (!file_exists($logFile)) { + echo json_encode(['ok' => true, 'output' => '', 'done' => false, 'started' => false]); exit; + } + $content = file_get_contents($logFile) ?: ''; + $done = str_contains($content, '__DONE__'); + if ($done) { + $content = str_replace(['__DONE__', "\n\n\n"], ['', "\n\n"], $content); + @unlink($logFile); + @unlink($pidFile); + } + // Check if rsync process is actually alive (catches crashes without __DONE__) + $pid = (int)trim(@file_get_contents($pidFile) ?: '0'); + if (!$done && $pid > 0 && !file_exists('/proc/' . $pid)) { + $done = true; + $content .= "\n\n--- Process ended unexpectedly ---"; + @unlink($pidFile); + } + echo json_encode(['ok' => true, 'output' => $content, 'done' => $done, 'started' => true]); + exit; +} + +echo json_encode(['ok' => false, 'error' => 'Unknown action']); diff --git a/Plugin/unraid/api/rsync.php b/Plugin/unraid/api/rsync.php new file mode 100644 index 0000000..bcfc10a --- /dev/null +++ b/Plugin/unraid/api/rsync.php @@ -0,0 +1,47 @@ + $p[0], + 'time' => $p[1], + 'profile' => $p[2], + 'duration' => (int)$p[3], + 'status' => trim($p[4]), + 'bytes' => isset($p[5]) ? (int)$p[5] : 0, + ]; + } +} + +echo json_encode([ + 'enabled' => $base['enabled'], + 'windows' => $base['windows'], + 'active' => $base['active'], + 'last_sync' => $base['last_sync'], + 'bw_history' => $history, + 'bw_warn_gb' => $warnGb, + 'settings' => [ + 'bw_limit' => (int)($vars['BW_LIMIT'] ?? 0), + 'retry_count' => (int)($vars['RETRY_COUNT'] ?? 3), + 'sleep' => (int)($vars['SLEEP'] ?? 300), + 'bw_warn_gb' => (float)($vars['BANDWIDTH_WARN_GB'] ?? 50), + ], + 'ts' => time(), +]); diff --git a/Plugin/unraid/api/rsync_profiles.php b/Plugin/unraid/api/rsync_profiles.php new file mode 100644 index 0000000..7616a5c --- /dev/null +++ b/Plugin/unraid/api/rsync_profiles.php @@ -0,0 +1,132 @@ + 'rsync_opts', + 'PROFILE_BW_LIMIT' => 'bw_limit', + 'PROFILE_RETRY_COUNT' => 'retry_count', + 'PROFILE_SLEEP' => 'sleep', + 'PROFILE_CRITICAL_CONTAINER_NAMES' => 'critical_containers', + 'PROFILE_DELAYED_CONTAINERS' => 'delayed_containers', + 'PROFILE_CONTAINER_DELAY' => 'container_delay', + 'PROFILE_EXCLUDE_DIRS' => 'exclude_dirs', + 'PROFILE_REMOTE_RESTART_CONTAINERS' => 'remote_restart', +]; + +// Parse [key]="value" or [key]=bare entries from an assoc_array body +function _rp_parse_assoc(string $body): array { + $result = []; + preg_match_all('/\[([^\]]+)\]\s*=\s*(?:"([^"]*)"|([^\s#\n]*))/', $body, $m, PREG_SET_ORDER); + foreach ($m as $match) { + $key = $match[1]; + $val = $match[2] !== '' ? $match[2] : ($match[3] ?? ''); + $result[$key] = $val; + } + return $result; +} + +// Rebuild assoc_array body from entries map +function _rp_build_assoc(array $entries): string { + $lines = []; + foreach ($entries as $k => $v) { + // Quote if empty, has spaces or special shell chars + if ($v === '' || preg_match('/[\s\$\!\[\]\(\)\|\'`\\\\]/', $v)) { + $lines[] = ' [' . $k . ']="' . str_replace('"', '\\"', $v) . '"'; + } else { + $lines[] = ' [' . $k . ']=' . $v; + } + } + return implode("\n", $lines); +} + +// Read all PROFILE_* arrays from master.conf → [profile_name => [field => value]] +function _rp_read_all(): array { + $raw = vv_read_conf_raw('master.conf'); + $result = []; + foreach (RP_ARRAYS as $varName => $fieldKey) { + if (!preg_match('/declare\s+-A\s+' . preg_quote($varName, '/') . '\s*=\s*\((.*?)\)/s', $raw, $m)) continue; + foreach (_rp_parse_assoc($m[1]) as $profile => $value) { + $result[$profile][$fieldKey] = $value; + } + } + return $result; +} + +$action = trim($_GET['action'] ?? $_POST['action'] ?? ''); + +// ── list ────────────────────────────────────────────────────────────────────── +if ($action === 'list') { + echo json_encode(['ok' => true, 'profiles' => _rp_read_all()]); + exit; +} + +// ── save (create or update) ─────────────────────────────────────────────────── +if ($_SERVER['REQUEST_METHOD'] === 'POST' && $action === 'save') { + $name = trim($_POST['name'] ?? ''); + if (!$name || !preg_match('/^[a-zA-Z0-9_\-]+$/', $name)) { + echo json_encode(['ok' => false, 'error' => 'Invalid profile name — use letters, numbers, hyphens, underscores']); exit; + } + + $raw = vv_read_conf_raw('master.conf'); + $changes = []; + + foreach (RP_ARRAYS as $varName => $fieldKey) { + $value = trim($_POST[$fieldKey] ?? ''); + // Find and parse current array body + if (!preg_match('/declare\s+-A\s+' . preg_quote($varName, '/') . '\s*=\s*\((.*?)\)/s', $raw, $m)) continue; + $entries = _rp_parse_assoc($m[1]); + $entries[$name] = $value; + $changes[] = [ + 'file' => 'master.conf', + 'key' => $varName, + 'type' => 'assoc_array', + 'value' => _rp_build_assoc($entries), + ]; + } + + if (!$changes) { echo json_encode(['ok' => false, 'error' => 'No profile arrays found in master.conf']); exit; } + + $results = vv_conf_write_changes($changes); + $ok = !in_array(false, $results, true); + if ($ok) { vv_push_master_conf(); vv_push_setup_state(); } + echo json_encode(['ok' => $ok, 'results' => $results]); + exit; +} + +// ── delete ──────────────────────────────────────────────────────────────────── +if ($_SERVER['REQUEST_METHOD'] === 'POST' && $action === 'delete') { + $name = trim($_POST['name'] ?? ''); + if (!$name || !preg_match('/^[a-zA-Z0-9_\-]+$/', $name)) { + echo json_encode(['ok' => false, 'error' => 'Invalid profile name']); exit; + } + + $raw = vv_read_conf_raw('master.conf'); + $changes = []; + + foreach (RP_ARRAYS as $varName => $fieldKey) { + if (!preg_match('/declare\s+-A\s+' . preg_quote($varName, '/') . '\s*=\s*\((.*?)\)/s', $raw, $m)) continue; + $entries = _rp_parse_assoc($m[1]); + if (!array_key_exists($name, $entries)) continue; + unset($entries[$name]); + $changes[] = [ + 'file' => 'master.conf', + 'key' => $varName, + 'type' => 'assoc_array', + 'value' => _rp_build_assoc($entries), + ]; + } + + if (!$changes) { echo json_encode(['ok' => false, 'error' => 'Profile not found']); exit; } + + $results = vv_conf_write_changes($changes); + $ok = !in_array(false, $results, true); + if ($ok) { vv_push_master_conf(); vv_push_setup_state(); } + echo json_encode(['ok' => $ok]); + exit; +} + +echo json_encode(['ok' => false, 'error' => 'Unknown action']); diff --git a/Plugin/unraid/include/monitor.php b/Plugin/unraid/include/monitor.php index d00e5d1..de80e6f 100644 --- a/Plugin/unraid/include/monitor.php +++ b/Plugin/unraid/include/monitor.php @@ -367,10 +367,30 @@ function vv_rsync_status(): array { ]; } + // Profile activity — last 7 days, aggregated per profile + $bwLog = DATA_DIR . '/bandwidth_history.db'; + $cutoff7 = date('Y-m-d', strtotime('-7 days')); + $profiles = []; + if (file_exists($bwLog)) { + foreach (file($bwLog, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES) ?: [] as $line) { + $p = explode('|', $line); + if (count($p) < 4 || ($p[0] ?? '') < $cutoff7) continue; + $name = $p[2] ?? ''; + if (!$name) continue; + if (!isset($profiles[$name])) $profiles[$name] = ['runs' => 0, 'dur' => 0, 'bytes' => 0]; + $profiles[$name]['runs']++; + $profiles[$name]['dur'] += (int)($p[3] ?? 0); + $profiles[$name]['bytes'] += (int)($p[5] ?? 0); + } + } + arsort($profiles); // sort by run count descending + $bwSummary = array_slice($profiles, 0, 6, true); + return [ - 'enabled' => $enabled, - 'windows' => $windows, - 'active' => $active, - 'last_sync' => $lastSync, + 'enabled' => $enabled, + 'windows' => $windows, + 'active' => $active, + 'last_sync' => $lastSync, + 'bw_summary' => $bwSummary, ]; } diff --git a/Plugin/unraid/include/partnership.php b/Plugin/unraid/include/partnership.php index ccfdc30..fa0984e 100644 --- a/Plugin/unraid/include/partnership.php +++ b/Plugin/unraid/include/partnership.php @@ -30,6 +30,27 @@ function vv_pt_config(): array { } // ── Mirror sync health — the partnership's actual job (rsync orchestrators) ────── + +function vv_pt_sync_summary(string $logFile): string { + if (!file_exists($logFile)) return ''; + $tail = array_filter(array_slice(file($logFile, FILE_IGNORE_NEW_LINES), -30), 'strlen'); + foreach (array_reverse(array_values($tail)) as $raw) { + // Strip emoji / Unicode decoration for regex matching + $line = trim(preg_replace('/[\x{1F000}-\x{1FFFF}\x{2600}-\x{27BF}\x{FE0F}]/u', '', $raw)); + $line = preg_replace('/\s+/', ' ', $line); + // "Critical sync complete — HOST1 — 1m39s — 2 share(s)" + if (preg_match('/Critical sync complete\s*—\s*\S+\s*—\s*([\w]+)\s*—\s*(.+)/i', $line, $m)) + return trim($m[2]) . ' in ' . $m[1]; + // "Status: all complete — 0 share(s) synced, 11 job(s) run" + if (preg_match('/Status:\s*all complete\s*—\s*(.+)/i', $line, $m)) + return trim($m[1]); + // "Status: X failure(s)" or "Failures: N" + if (preg_match('/Failures:\s*(\d+)/i', $line, $m) && (int)$m[1] > 0) + return (int)$m[1] . ' failure(s)'; + } + return ''; +} + function vv_pt_sync(): array { $jobs = [ 'critical' => 'Orchestrators/critical_sync_maintenance', @@ -39,14 +60,17 @@ function vv_pt_sync(): array { $out = ['jobs' => []]; foreach ($jobs as $key => $base) { $statFile = LOG_DIR . '/' . $base . '.json'; + $logFile = LOG_DIR . '/' . $base . '.log'; $s = file_exists($statFile) ? json_decode(@file_get_contents($statFile), true) : null; $out['jobs'][$key] = is_array($s) ? [ - 'status' => $s['status'] ?? 'unknown', - 'start' => isset($s['start']) ? (int)$s['start'] : null, - 'end' => isset($s['end']) ? (int)$s['end'] : null, + 'status' => $s['status'] ?? 'unknown', + 'start' => isset($s['start']) ? (int)$s['start'] : null, + 'end' => isset($s['end']) ? (int)$s['end'] : null, + 'summary' => vv_pt_sync_summary($logFile), ] : null; } $v = vv_conf_vars(); + $out['interval_min'] = (int)($v['PARTNERSHIP_SYNC_INTERVAL'] ?? 30); // Rsync gate flags (master.conf) — global Tier 1 + per-tier Tier 2. $out['gates'] = [ 'global' => ['var' => 'RSYNC_ENABLED', 'on' => ($v['RSYNC_ENABLED'] ?? 'true') === 'true'], diff --git a/Plugin/unraid/pages/monitor.php b/Plugin/unraid/pages/monitor.php index 0f85fea..bcaf999 100644 --- a/Plugin/unraid/pages/monitor.php +++ b/Plugin/unraid/pages/monitor.php @@ -1,4 +1,10 @@ + @@ -1318,74 +1324,143 @@ function vvPollMonitor() { // ── Rsync ──────────────────────────────────────────────────────────────── (function() { - const rs = d.rsync ?? {}; - const enabled = rs.enabled ?? true; - const windows = rs.windows ?? {}; - const active = rs.active ?? []; - const lastSync = rs.last_sync ?? {}; - const now = Math.floor(Date.now() / 1000); + const rs = d.rsync ?? {}; + const enabled = rs.enabled ?? true; + const windows = rs.windows ?? {}; + const active = rs.active ?? []; + const lastSync = rs.last_sync ?? {}; + const bwSummary = rs.bw_summary ?? {}; + const now = Math.floor(Date.now() / 1000); const el = document.getElementById('vv-rsync-body'); if (!el) return; - const gColor = enabled ? '#4caf50' : '#555'; - let html = `
● ${enabled ? 'enabled' : 'disabled'}
`; + function _dur(s) { + if (!s) return '—'; + if (s < 60) return s + 's'; + if (s < 3600) return Math.floor(s/60) + 'm' + (s%60 ? String(s%60).padStart(2,'0')+'s' : ''); + return Math.floor(s/3600) + 'h' + Math.floor((s%3600)/60) + 'm'; + } + function _ago(diff) { + if (diff < 60) return diff + 's'; + if (diff < 3600) return Math.floor(diff/60) + 'm'; + if (diff < 86400) return Math.floor(diff/3600) + 'h'; + return Math.floor(diff/86400) + 'd'; + } + function _fmtBytes(b) { + if (!b) return ''; + if (b >= 1073741824) return (b/1073741824).toFixed(1) + 'G'; + if (b >= 1048576) return (b/1048576).toFixed(0) + 'M'; + return (b/1024).toFixed(0) + 'K'; + } - // Window badges: C D I W - html += '
'; + const gCol = enabled ? '#4caf50' : '#555'; + + // ── Header: gate dot + window badges ────────────────────────────────── + let html = `
+ ● ${enabled ? 'ENABLED' : 'DISABLED'} +
`; [['C','critical'],['D','daily'],['I','intermediate'],['W','weekly']].forEach(([s,k]) => { - const on = windows[k] ?? false; - html += `${s}`; + const on = windows[k] ?? false; + const run = active.some(a => a.profile?.includes(k)); + const bg = run ? '#1a2a0a' : on ? '#0f1a0f' : '#111'; + const brd = run ? '#3a6a1a' : on ? '#1a3a1a' : '#222'; + const col = run ? '#8bc34a' : on ? '#4caf50' : '#333'; + html += `${s}`; }); - html += '
'; + html += `
`; - // Active sessions + // ── Profile activity bars (7 days) ──────────────────────────────────── + const profEntries = Object.entries(bwSummary); + if (profEntries.length) { + const maxRuns = Math.max(...profEntries.map(([,v]) => v.runs), 1); + const colors = ['#4caf50','#4a9eff','#ffb74d','#ce93d8','#ef5350','#4dd0e1']; + html += `
+
7-day profile activity
`; + profEntries.forEach(([name, v], i) => { + const pct = Math.round(v.runs / maxRuns * 100); + const col = colors[i % colors.length]; + const bytes = v.bytes > 0 ? _fmtBytes(v.bytes) : ''; + const short = name.length > 18 ? name.slice(0,16) + '…' : name; + html += `
+
+ ${short} + + ${v.runs}×${bytes ? ' · ' + bytes : ''} +
+
+
+
+
`; + }); + html += `
`; + } + + // ── Active syncs ─────────────────────────────────────────────────────── if (active.length) { active.forEach(a => { const sec = a.elapsed ?? 0; - const dur = sec < 60 ? sec+'s' : sec < 3600 ? Math.floor(sec/60)+'m'+String(sec%60).padStart(2,'0')+'s' : Math.floor(sec/3600)+'h'+Math.floor((sec%3600)/60)+'m'; - html += `
- ⟳ ${a.profile} - ${dur} + html += `
+
+ ⟳ ${a.profile} + ${_dur(sec)} +
+
+
+
`; }); - html += `
`; } - // Last sync per window - const syncRows = [['critical','critical'],['daily','daily'],['intermediate','interm'],['weekly','weekly']]; - let hasSyncs = false; - let syncHtml = ''; - syncRows.forEach(([key, label]) => { - const s = lastSync[key]; - if (!s || !s.ts) return; - hasSyncs = true; - const diff = now - s.ts; - const ago = diff < 60 ? diff+'s' : diff < 3600 ? Math.floor(diff/60)+'m' : diff < 86400 ? Math.floor(diff/3600)+'h' : Math.floor(diff/86400)+'d'; - const ok = s.status === 'ok' || s.status === 'success'; - const warn = s.status === 'warn'; - const run = s.status === 'running'; - const col = ok ? '#4caf50' : warn ? '#ff9800' : run ? '#4fc3f7' : '#f44336'; - const icon = ok ? '✓' : warn ? '!' : run ? '●' : '✗'; - syncHtml += `
- ${label} - ${ago} - ${icon} + // ── Tier last-run rows with recency bars ─────────────────────────────── + const tierMeta = { + critical: {label:'Critical', cadenceSec: 30*60}, + daily: {label:'Daily', cadenceSec: 24*3600}, + intermediate: {label:'Interm', cadenceSec: 4*3600}, + weekly: {label:'Weekly', cadenceSec: 7*86400}, + }; + html += `
`; + Object.entries(tierMeta).forEach(([key, meta]) => { + const s = lastSync[key]; + const ts = s?.ts ?? 0; + const ok = s?.status === 'ok' || s?.status === 'success'; + const warn = s?.status === 'warn'; + const run = s?.status === 'running'; + const err = s && !ok && !warn && !run && ts > 0; + const col = run ? '#4a9eff' : ok ? '#4caf50' : warn ? '#ff9800' : err ? '#f44336' : '#2a2a2a'; + const icon = run ? '⟳' : ok ? '✓' : warn ? '!' : err ? '✗' : '—'; + const diff = ts ? now - ts : null; + const pct = ts ? Math.min(100, Math.round((now - ts) / meta.cadenceSec * 100)) : 0; + const barC = pct >= 100 ? '#3a1a1a' : pct >= 75 ? '#2a1a0a' : '#0a1a0a'; + const fillC= pct >= 100 ? '#f44336' : pct >= 75 ? '#ff9800' : '#2a6a2a'; + + html += `
+
+ ${meta.label} + + ${s?.duration ? _dur(s.duration) : ''} + + ${diff !== null ? _ago(diff) : '—'} + ${icon} +
+
+
+
`; }); - if (hasSyncs) { - html += `
Last sync
`; - html += syncHtml; - } + html += `
`; el.innerHTML = html; const card = document.getElementById('vv-rsync-card'); if (card) { card.classList.remove('vv-accent-ok','vv-accent-warn','vv-accent-err'); - if (active.length) card.classList.add('vv-accent-ok'); + if (!enabled) card.classList.add('vv-accent-err'); + else if (active.length) card.classList.add('vv-accent-ok'); } })(); diff --git a/Plugin/unraid/pages/partnership.php b/Plugin/unraid/pages/partnership.php index a1c30c3..72ec10c 100644 --- a/Plugin/unraid/pages/partnership.php +++ b/Plugin/unraid/pages/partnership.php @@ -66,6 +66,16 @@ textarea.vv-set-input { resize:vertical; white-space:pre; }
Loading…
+ + + @@ -74,16 +84,19 @@ textarea.vv-set-input { resize:vertical; white-space:pre; }
Loading…
- -
-

Mirror Sync

-
Loading…
-
+ + - -
-

Actions

-
Loading…
+ +
+
+

Mirror Sync

+
Loading…
+
+
+

Actions

+
Loading…
+
@@ -303,6 +316,165 @@ function vvPtToggleSync(el, varName, enabled) { .catch(e => { alert('Error: ' + e); el.style.pointerEvents = ''; el.style.opacity = ''; }); } +// ── Host Settings ───────────────────────────────────────────────────────────── + +function _renderHostSettings(nodes, isOwner) { + let html = '
'; + for (const n of nodes) { + const rdonly = isOwner ? '' : 'readonly style="opacity:.5;cursor:default;"'; + html += `
+
${_vvSetEsc(n.id)}
+ +
`; + } + html += '
'; + if (!isOwner) html += '
Editing requires host1.
'; + return html; +} + +function vvPtHostChanged(el) { + el.classList.toggle('changed', el.value !== el.dataset.orig); + const any = document.querySelector('.vv-pt-host-inp.changed'); + document.getElementById('vv-pt-hosts-save').style.display = any ? '' : 'none'; +} + +function vvPtSaveHosts(btn) { + const changedEls = document.querySelectorAll('.vv-pt-host-inp.changed'); + if (!changedEls.length) return; + const changes = []; + changedEls.forEach(el => changes.push({key: el.dataset.key, file: el.dataset.file, type: el.dataset.type, value: el.value})); + if (!confirm(`Save ${changes.length} host setting(s)?`)) return; + btn.disabled = true; btn.textContent = '⟳ Saving…'; + fetch('/plugins/varaverk/api/confform.php', { + method: 'POST', + headers: {'Content-Type': 'application/x-www-form-urlencoded'}, + body: new URLSearchParams({ + csrf_token: typeof csrf_token !== 'undefined' ? csrf_token : '', + id: '__settings__', + changes: JSON.stringify(changes) + }) + }) + .then(r => r.json()) + .then(d => { + btn.disabled = false; + if (d.ok) { + changedEls.forEach(el => { el.dataset.orig = el.value; el.classList.remove('changed'); }); + btn.textContent = '✓ Saved'; + setTimeout(() => { btn.textContent = 'Save'; btn.style.display = 'none'; }, 2000); + } else { + btn.textContent = 'Save'; + alert('Save failed: ' + (d.error ?? 'Unknown error')); + } + }) + .catch(e => { btn.disabled = false; btn.textContent = 'Save'; alert('Error: ' + e); }); +} + +// ── Array settings cards ─────────────────────────────────────────────────────── + +let _vvArrFiles = null; + +function _vvInitArrayCards() { + if (_vvArrFiles !== null) return; + _vvArrFiles = false; // loading sentinel + fetch('/plugins/varaverk/api/partnership_settings.php?_=' + Date.now()) + .then(r => r.json()) + .then(d => { + _vvArrFiles = (d.ok && d.files) ? d.files : []; + _vvRenderArrayCards(); + }) + .catch(() => { _vvArrFiles = []; document.getElementById('vv-pt-arrays-wrap').style.display = 'none'; }); +} + +function _vvRenderArrayCards() { + const files = _vvArrFiles; + if (!files || !files.length) { document.getElementById('vv-pt-arrays-wrap').style.display = 'none'; return; } + + let html = ''; + let hasArrays = false; + for (const f of files) { + for (const g of f.groups) { + for (const fld of g.fields) { + if (fld.type === 'scalar') continue; + hasArrays = true; + const rows = Math.min(16, (fld.value.match(/\n/g) || []).length + 2); + const attrs = `class="vv-set-input" data-key="${_vvSetEsc(fld.key)}" ` + + `data-file="${_vvSetEsc(fld.file)}" data-type="${_vvSetEsc(fld.type)}" ` + + `data-orig="${_vvSetEsc(fld.value)}" oninput="vvPtArrChanged(this)"`; + html += `
+
+ ${_vvSetEsc(fld.key)} +
+ ${fld.desc ? `${_vvSetEsc(fld.desc)}` : ''} + ▸ +
+
+ +
`; + } + } + } + + const wrap = document.getElementById('vv-pt-arrays-wrap'); + if (!hasArrays) { wrap.style.display = 'none'; return; } + + wrap.style.display = ''; + wrap.innerHTML = `
+
+

Array Settings

+ +
+ ${html} +
`; +} + +function vvPtArrChanged(el) { + el.classList.toggle('changed', el.value !== el.dataset.orig); + const any = document.querySelector('#vv-pt-arrays-wrap .vv-set-input.changed'); + const saveBtn = document.getElementById('vv-pt-arrays-save'); + if (saveBtn) saveBtn.style.display = any ? '' : 'none'; +} + +function vvPtSaveArrays(btn) { + const changedEls = document.querySelectorAll('#vv-pt-arrays-wrap .vv-set-input.changed'); + if (!changedEls.length) return; + const changes = []; + changedEls.forEach(el => changes.push({key: el.dataset.key, file: el.dataset.file, type: el.dataset.type, value: el.value})); + if (!confirm(`Save ${changes.length} changed setting(s)?`)) return; + btn.disabled = true; btn.textContent = '⟳ Saving…'; + fetch('/plugins/varaverk/api/confform.php', { + method: 'POST', + headers: {'Content-Type': 'application/x-www-form-urlencoded'}, + body: new URLSearchParams({ + csrf_token: typeof csrf_token !== 'undefined' ? csrf_token : '', + id: '__settings__', + changes: JSON.stringify(changes) + }) + }) + .then(r => r.json()) + .then(d => { + btn.disabled = false; + if (d.ok) { + changedEls.forEach(el => { el.dataset.orig = el.value; el.classList.remove('changed'); }); + btn.textContent = '✓ Saved'; + _vvArrFiles = null; // invalidate so next init re-fetches + setTimeout(() => _vvInitArrayCards(), 600); + } else { + btn.textContent = 'Save Changes'; + alert('Save failed: ' + (d.error ?? 'Unknown error')); + } + }) + .catch(e => { btn.disabled = false; btn.textContent = 'Save Changes'; alert('Error: ' + e); }); +} + // ── Settings panel ───────────────────────────────────────────────────────────── let _vvSetLoaded = false; @@ -343,33 +515,26 @@ function vvPtLoadSettings() { function vvPtRenderSettings(files) { let html = ''; for (const f of files) { - // One block per file; partnership scope yields a single section per file, so flatten. const label = f.file === 'master.conf' ? 'master.conf — shared' : f.file + ' — this host'; + const scalarFields = f.groups.flatMap(g => g.fields.filter(fld => fld.type === 'scalar')); + if (!scalarFields.length) continue; html += `
${_vvSetEsc(label)}▾
`; - for (const g of f.groups) { - for (const fld of g.fields) { - const attrs = `class="vv-set-input" data-key="${_vvSetEsc(fld.key)}" ` - + `data-file="${_vvSetEsc(fld.file)}" data-type="${_vvSetEsc(fld.type)}" ` - + `data-orig="${_vvSetEsc(fld.value)}" oninput="vvPtSetChanged(this)"`; - html += `
-
${_vvSetEsc(fld.key)}
`; - if (fld.desc) html += `
${_vvSetEsc(fld.desc)}
`; - if (fld.type === 'scalar') { - html += ``; - } else { - const rows = Math.min(16, (fld.value.match(/\n/g) || []).length + 2); - html += ``; - } - html += `
`; - } + for (const fld of scalarFields) { + const attrs = `class="vv-set-input" data-key="${_vvSetEsc(fld.key)}" ` + + `data-file="${_vvSetEsc(fld.file)}" data-type="${_vvSetEsc(fld.type)}" ` + + `data-orig="${_vvSetEsc(fld.value)}" oninput="vvPtSetChanged(this)"`; + html += `
+
${_vvSetEsc(fld.key)}
`; + if (fld.desc) html += `
${_vvSetEsc(fld.desc)}
`; + html += `
`; } html += `
`; } - return html; + return html || '
No scalar settings available.
'; } function vvPtToggleNode(hdr) { @@ -491,47 +656,98 @@ function _syncToggle(gate, dimmed) { } function _renderSync(sync) { - const jobs = sync.jobs || {}; - const gates = sync.gates || {}; + const jobs = sync.jobs || {}; + const gates = sync.gates || {}; + const intervalMin = sync.interval_min || 30; + const sCol = s => s === 'ok' ? '#4caf50' : s === 'running' ? '#4a9eff' : s === 'warn' ? '#ff9800' : s === 'error' ? '#f44336' : '#555'; - const sLbl = s => s === 'ok' ? '✓ ok' : s === 'running' ? '⟳ running' - : s === 'warn' ? '⚠ warn' : s === 'error' ? '✗ error' : '— never'; + const sLbl = s => s === 'ok' ? '✓' : s === 'running' ? '⟳' : s === 'warn' ? '⚠' : s === 'error' ? '✗' : '—'; + + function _dur(sec) { + if (!sec || sec < 0) return ''; + if (sec < 60) return sec + 's'; + if (sec < 3600) return Math.floor(sec / 60) + 'm ' + (sec % 60) + 's'; + return Math.floor(sec / 3600) + 'h ' + Math.floor((sec % 3600) / 60) + 'm'; + } + + function _countdown(ts) { + if (!ts) return null; + const sec = ts - Math.floor(Date.now() / 1000); + if (sec <= 0) return 'due'; + if (sec < 60) return 'in ' + sec + 's'; + if (sec < 3600) return 'in ' + Math.floor(sec / 60) + 'm'; + return 'in ' + Math.floor(sec / 3600) + 'h ' + Math.floor((sec % 3600) / 60) + 'm'; + } const globalOff = gates.global && !gates.global.on; - // Master toggle row — controls all mirroring (RSYNC_ENABLED, Tier 1). - let html = ''; + // Wrap everything in a flex-column so the bottom section anchors naturally + let html = '
'; + + // Master toggle row if (gates.global) { - html += `
+ html += `
All mirroring ${_syncToggle(gates.global, false)}
`; } - // Per-tier rows: status + last-run + per-tier toggle (dimmed when global is off). - const labels = {critical: 'Critical (30 min)', daily: 'Daily', weekly: 'Weekly'}; + // Per-tier rows + const tierMeta = { + critical: {label: 'Critical', sched: 'every 30 min', desc: 'appdata · partnership · play state'}, + daily: {label: 'Daily', sched: 'nightly', desc: 'media shares · arr sync · docker'}, + weekly: {label: 'Weekly', sched: 'weekly', desc: 'bulk shares · updates · cleanup'}, + }; + + // Critical next-run estimate + const critJob = jobs.critical; + const critNextTs = critJob && critJob.end && critJob.status !== 'running' + ? critJob.end + intervalMin * 60 : null; + + html += '
'; + for (const key of ['critical', 'daily', 'weekly']) { - const j = jobs[key]; + const j = jobs[key]; + const meta = tierMeta[key]; const status = j ? j.status : null; const when = j && j.start - ? (j.status === 'running' ? 'started ' + _relTime(j.start) - : _relTime(j.end || j.start)) - : '—'; - html += `
- ${labels[key]} - - ${sLbl(status)} - ${when} - ${_syncToggle(gates[key], globalOff)} - + ? (j.status === 'running' ? 'started ' + _relTime(j.start) : _relTime(j.end || j.start)) + : null; + const dur = j && j.start && j.end && j.status !== 'running' ? _dur(j.end - j.start) : ''; + const summary = j && j.summary ? j.summary : ''; + + html += `
+
+ ${meta.label} + ${meta.sched} + + + ${sLbl(status)} + ${dur ? `${dur}` : ''} + ${when ? `${when}` : ''} + ${_syncToggle(gates[key], globalOff)} + +
+
${meta.desc}
+ ${summary ? `
${_vvSetEsc(summary)}
` : ''}
`; } + html += '
'; // flex:1 + + // Bottom info strip + const nextLabel = critNextTs ? _countdown(critNextTs) : null; + html += `
+ interval: ${intervalMin}min + ${nextLabel ? `next critical ${nextLabel}` : ''} +
`; + if (globalOff) { html += `
⚠ All mirroring is off — per-tier switches have no effect until re-enabled.
`; } + html += '
'; // outer flex column return html; } @@ -721,214 +937,280 @@ function _renderActions(nodes, cfg) { let html = ''; + // ── No partner ───────────────────────────────────────────────────────────── if (!hasPartner && isOwner) { - html += `
- No partner configured — edit master.conf (Scheduler tab) and set HOST2. + html += `
+
No partner host configured
+
Set HOST2 in master.conf (Host Settings above) to enable partnership.
`; } // ── Per-remote host section ──────────────────────────────────────────────── if (isOwner && hasPartner) { - remotes.forEach(remote => { - const phase = remote.onboard_phase ?? 0; + remotes.forEach((remote, idx) => { + const phase = remote.onboard_phase ?? 0; const isOnboarding = !!_vvOnboarding[remote.id]; - const dotCol = remote.ts_online === null ? '#555' : remote.ts_online ? '#4caf50' : '#f44336'; - const dotLbl = remote.ts_online === null ? 'unknown' : remote.ts_online ? 'online' : 'offline'; - const pt = remote.partnership || {}; - const ptState = (pt.state || '').toUpperCase(); - const ptDate = pt.updated + const isDeleting = !!_vvDeleteKeys[remote.id]; + const dotCol = remote.ts_online === null ? '#444' : remote.ts_online ? '#4caf50' : '#f44336'; + const dotLbl = remote.ts_online === null ? 'unknown' : remote.ts_online ? 'online' : 'offline'; + const pt = remote.partnership || {}; + const ptState = (pt.state || '').toUpperCase(); + const ptDate = pt.updated ? new Date(parseInt(pt.updated) * 1000).toLocaleDateString([], {month:'short',day:'numeric',year:'numeric'}) : null; - const sys = remote.system || {}; - const sysUptime = sys.uptime_sec ? _uptime(sys.uptime_sec) : null; - const sysVer = sys.unraid_version || null; + const sys = remote.system || {}; + const sysUptime = sys.uptime_sec ? _uptime(sys.uptime_sec) : null; + const sysVer = sys.unraid_version || null; - html += `
`; + // Left border colour by phase + const borderCol = phase === 2 ? '#2a5a2a' + : phase === 1 ? '#4a3000' + : isOnboarding ? '#0e2a4a' : '#222'; - // Header: host + online dot + test connection - html += `
-
- ${remote.id} - ${remote.hostname} -
+ if (idx > 0) html += '
'; + + html += `
`; + + // ── Card header ───────────────────────────────────────────────────────── + html += `
- + ${remote.id} + ${remote.hostname} + ● ${dotLbl} +
+
+ - ● ${dotLbl} + style="font-size:9px;padding:2px 7px;" title="SSH echo round-trip">⇄ ping
`; - const isDeleting = !!_vvDeleteKeys[remote.id]; + html += `
`; - // ── Phase 0: not provisioned ───────────────────────────────────────── + // ── Phase 0: not provisioned ─────────────────────────────────────────── if (phase === 0) { if (isOnboarding) { - html += `
-
- Step 1 - Install SSH key on ${remote.hostname} + html += `
+
+
+ Step 1 + Install SSH key on ${remote.hostname} +
+
+ Open Terminal + ${termCmd} +
+
Enter ${remote.hostname} root password when prompted · tab auto-updates on completion
-
- 🖥 Open Terminal - ${termCmd} +
+
+ Step 2 + Push conf + if key already installed separately +
+
-
Enter ${remote.hostname} root password when prompted · tab auto-updates on completion
-
- Step 2 - Push conf - if key already installed separately -
-
-
- +
+
`; } else { - html += `
- ○ Not provisioned - - + html += `
+
+
Not provisioned
+
SSH keys not installed · conf not pushed
+
+
+ + +
`; } // ── Phase 1: SSH done, awaiting HOST2 ───────────────────────────────── } else if (phase === 1) { - html += `
- ⏳ SSH ready - ${remote.ts_ip ? `${remote.ts_ip}` : ''} -
-
- Waiting for ${remote.id} to install Varaverk and complete its onboard. Phase 2 triggers automatically. -
-
- - - + html += `
+
+
+ SSH ready + ${remote.ts_ip ? `${remote.ts_ip}` : ''} +
+
+ Waiting for ${remote.id} to install Varaverk and complete onboard.
Phase 2 triggers automatically. +
+
+
+ + + +
`; // ── Phase 2: fully onboarded ─────────────────────────────────────────── } else { const ptColor = ptState === 'ACTIVE' ? '#4caf50' : ptState === 'INACTIVE' ? '#f44336' : '#555'; - html += `
- ✅ Active - ${ptDate ? `since ${ptDate}` : ''} -
-
- ${remote.ts_ip ? `${remote.ts_ip}` : ''} - ${sysUptime ? `up ${sysUptime}` : ''} - ${sysVer ? `unRAID ${sysVer}` : ''} - ${ptState ? `${ptState}` : ''} -
-
- - + html += `
+
+
+ Active + ${ptState ? `${ptState}` : ''} + ${ptDate ? `since ${ptDate}` : ''} +
+
+ ${remote.ts_ip ? `${remote.ts_ip}` : ''} + ${sysVer ? `unRAID ${sysVer}` : ''} + ${sysUptime ? `up ${sysUptime}` : ''} +
+
+
+ + +
`; } - // ── Delete Keys panel (any phase) ────────────────────────────────────── + // ── Delete Keys panel ────────────────────────────────────────────────── if (isDeleting) { - html += `
-
Remove SSH access
-
+ html += `
+
Remove SSH access
+
-
HOST1 → ${remote.id}
-
HOST1's key on ${remote.hostname} · deletes local key pair
+
HOST1 → ${remote.id}
+
HOST1's key on ${remote.hostname} · deletes local pair
+ style="white-space:nowrap;font-size:10px;">✕ HOST1 key
-
${remote.id} → HOST1
-
${remote.hostname}'s key on HOST1 · removes from authorized_keys
+
${remote.id} → HOST1
+
${remote.hostname}'s key from authorized_keys
+ style="white-space:nowrap;font-size:10px;">✕ ${remote.id} key
-
+
+ style="font-size:10px;opacity:.6;">✓ Done
`; } - html += `
`; + html += `
`; // padding + card }); } - // ── HOST1 local setup (when needed after phase 1+) ──────────────────────── + // ── HOST1 local setup ────────────────────────────────────────────────────── if (isOwner && selfNode && !selfNode.local_done && remotes.some(r => (r.onboard_phase ?? 0) >= 1)) { - html += `
-
- ${selfNode.id} — ${selfNode.hostname} (this server) + html += `
+
+
${selfNode.id} — ${selfNode.hostname}
+
FolderView3 integration · marks HOST1 ready
-
- + +
`; + } + + // ── Mirror onboard (non-owner) ───────────────────────────────────────────── + if (!isOwner) { + html += `
+
+
+
Join partnership
+
SSH key setup + notify owner
+
+
`; } - // ── Bottom row ───────────────────────────────────────────────────────────── - html += `
`; - if (!isOwner) { - html += ``; - } - html += `
`; + // ── Offboard + Transfer ──────────────────────────────────────────────────── + html += `
`; + + html += `
+
+
Offboard
+
End partnership · revoke SSH · restore WebUIs
+
+ +
`; if (cfg.enabled && isOwner) { - const tok = (cfg.transfer_confirm || 'i-understand-this-transfers-ownership') - .replace(/'/g, "\\'"); - html += `
-
- Transfer ownership to the mirror — promotes it to owner. - -
-
- Or run manually: bash Partnership/partnership_transfer.sh --confirm=${cfg.transfer_confirm || 'i-understand-this-transfers-ownership'} + const tok = (cfg.transfer_confirm || 'i-understand-this-transfers-ownership').replace(/'/g, "\\'"); + html += `
+
+
Transfer ownership
+
Promotes mirror to owner · requires sustained health checks
+ +
+
+ bash Partnership/partnership_transfer.sh --confirm=${cfg.transfer_confirm || 'i-understand-this-transfers-ownership'}
`; } + html += '
'; // offboard+transfer section + return html; } // ── Main render ─────────────────────────────────────────────────────────────── function _render(data) { - const cfg = data.config || {}; - const nodes = data.nodes || []; + const cfg = data.config || {}; + const nodes = data.nodes || []; + const isOwner = nodes.some(n => n.is_me && n.is_owner); // Config bar document.getElementById('vv-pt-config-body').innerHTML = _renderConfig(cfg); + // Host settings — always shown upfront + const hostsCard = document.getElementById('vv-pt-hosts-card'); + hostsCard.style.display = nodes.length ? '' : 'none'; + document.getElementById('vv-pt-hosts-body').innerHTML = _renderHostSettings(nodes, isOwner); + document.getElementById('vv-pt-hosts-save').style.display = 'none'; + // Offline countdown warning _renderOfflineWarn(cfg); @@ -946,6 +1228,12 @@ function _render(data) { // Actions document.getElementById('vv-pt-actions-body').innerHTML = _renderActions(nodes, cfg); + // Array settings cards above Actions — load once + _vvInitArrayCards(); + + // Settings panel — host1 only + document.getElementById('vv-pt-settings-card').style.display = isOwner ? '' : 'none'; + // Highlight Onboard button when arriving from first-run wizard if (new URLSearchParams(location.search).get('vv_onboard') === '1') { const onboardBtn = document.querySelector('#vv-pt-actions-body .vv-pt-action-btn.run'); diff --git a/Plugin/unraid/pages/rsync.php b/Plugin/unraid/pages/rsync.php new file mode 100644 index 0000000..94ba7f4 --- /dev/null +++ b/Plugin/unraid/pages/rsync.php @@ -0,0 +1,1366 @@ + + +
+ Rsync + +
+ + +
+
+
+ Manual Rsync + +
+ +
+ +
+ +
+ + +
+ + +
+
+ Rsync Reference + +
+ + +
+
+ ▾ How to use this page +
+
+
+
Status card — shows whether the global rsync gate is open. Red = all syncs blocked regardless of tier toggles.
+
Sync windows — per-tier enable/disable. A tier can be enabled here but still blocked by the global gate.
+
Settings → Global gate — RSYNC_ENABLED in master.conf. Master switch; turn off during rebuilds or maintenance.
+
Settings → Tier toggles — CRITICAL/DAILY/etc._RSYNC_ENABLED. Disable one tier without affecting others.
+
Bandwidth history — last 30 days of per-profile rsync runs. Bar = relative time spent. Click recent runs for detail.
+
Manual Rsync — one-off transfer to any partner host. Browse remote dirs via SSH. Runs in background, output streams below.
+
Profile Editor — create and edit named profiles used by appdata syncs. Each profile controls flags, BW limit, container stop lists, and excludes.
+
+
+
+ + +
+
+ ▸ Sync windows +
+ +
+ + +
+
+ ▸ Profile system +
+ +
+ + +
+
+ ▸ Common flag combinations +
+ +
+
+ + + + +
+ + diff --git a/Plugin/unraid/pages/scheduler.php b/Plugin/unraid/pages/scheduler.php index 98d59db..704b2be 100644 --- a/Plugin/unraid/pages/scheduler.php +++ b/Plugin/unraid/pages/scheduler.php @@ -466,16 +466,6 @@ $runningScripts = array_unique($runningScripts);
  • Child (orch OFF) — enter a cron to run it standalone
  • Rsync badge — toggle writes TIER_RSYNC_ENABLED directly to master.conf
  • Rsync (orch OFF) — fill location + standalone cron + Save; both required for independent firing
    • -
  • Editor Keyboard Shortcuts
    • -
  • Ctrl+S — save · Ctrl+Z / Ctrl+Y — undo / redo (200 steps)
    • -
  • Ctrl+/ — toggle line comment · Tab / Shift+Tab — indent / dedent
    • -
  • Ctrl+F — find · Ctrl+H — find & replace · Ctrl+G — go to line
    • -
  • Ctrl+D — select next occurrence · Ctrl+L — select line · Ctrl+Shift+K — delete line
    • -
  • Alt+↑/↓ — move line · Alt+Shift+↓ — duplicate line below
    • -
  • Home — smart (first non-space → col 0)
    • -
  • ( { [ " ` — auto-close pair; wraps selection; Backspace deletes both when between them
    • -
  • A− / A+ in status bar — font size (9–22px, persists across sessions)
    • -
  • Indent guides — faint vertical lines every 2 spaces in the background
    • @@ -909,6 +899,33 @@ Still the same two servers, two households, the same media stack running itself. ×
    + +
    +
    + ▾ + Keyboard Shortcuts +
    +
    +
      +
    • Ctrl+S — save
      • +
    • Ctrl+Z / Ctrl+Y — undo / redo (200 steps)
      • +
    • Ctrl+/ — toggle line comment
      • +
    • Tab / Shift+Tab — indent / dedent
      • +
    • Ctrl+F — find
      • +
    • Ctrl+H — find & replace
      • +
    • Ctrl+G — go to line
      • +
    • Ctrl+D — select next occurrence
      • +
    • Ctrl+L — select line
      • +
    • Ctrl+Shift+K — delete line
      • +
    • Alt+↑/↓ — move line
      • +
    • Alt+Shift+↓ — duplicate line below
      • +
    • Home — smart (first non-space → col 0)
      • +
    • ( { [ " ` — auto-close pair; wraps selection; Backspace deletes both
      • +
    • A− / A+ in status bar — font size (9–22px, persists)
      • +
    +
    +
    +
    diff --git a/Watchdogs/docker_watchdog.sh b/Watchdogs/docker_watchdog.sh index 8a1b9c3..93b527b 100755 --- a/Watchdogs/docker_watchdog.sh +++ b/Watchdogs/docker_watchdog.sh @@ -130,11 +130,10 @@ # STATE FILES # ============================================================================================== # -# WATCHDOG_STATE_FILE — strike counts, daemon flags (STATE_DIR — survives reboots) +# WATCHDOG_STATE_FILE — strike counts, daemon flags (STATE_DIR — survives reboots) # DOCKER_WATCHDOG_FAILED_FILE — container skip list (STATE_DIR — survives reboots) -# DOCKER_WATCHDOG_MANUAL_STOP_FILE — intentionally stopped containers (STATE_DIR — survives reboots) -# WATCHDOG_CONTAINER_RESTART_LOG — restart history for loop detection (DATA_DIR) -# RW_STATE_FILE — read-only: resource_watchdog RAM emergency flag +# WATCHDOG_CONTAINER_RESTART_LOG — restart history for loop detection (DATA_DIR) +# RW_STATE_FILE — read-only: resource_watchdog RAM emergency flag # # ============================================================================================== # CONFIGURATION @@ -150,6 +149,11 @@ # HTTP health check endpoints. Format: "ContainerName:http://host:port" # Aliased by detect_hosts() → WATCHDOG_CONTAINER_URLS # +# HOST*_WATCHDOG_CONTAINER_API_CHECKS +# API liveness checks — deeper than HTTP. Format: "ContainerName:URL|APIKey" +# Use endpoints that require a live DB round-trip (e.g. Emby /System/Info). +# Aliased by detect_hosts() → WATCHDOG_CONTAINER_API_CHECKS +# # HOST*_WATCHDOG_REQUIRED_CONTAINERS # Containers that must always be running. Aliased by detect_hosts() → # WATCHDOG_REQUIRED_CONTAINERS @@ -258,7 +262,7 @@ validate_unraid_cmd "/usr/local/emhttp/plugins/dynamix/scripts/notify" " # Ensure state files exist touch "$WATCHDOG_STATE_FILE" "$WATCHDOG_CONTAINER_RESTART_LOG" \ - "$DOCKER_WATCHDOG_FAILED_FILE" "$DOCKER_WATCHDOG_MANUAL_STOP_FILE" 2>/dev/null + "$DOCKER_WATCHDOG_FAILED_FILE" 2>/dev/null # Timeout for all docker commands — configurable via WATCHDOG_DAEMON_TIMEOUT in master.conf DOCKER_TIMEOUT="${WATCHDOG_DAEMON_TIMEOUT:-20}" @@ -274,8 +278,6 @@ if [[ "$SHOW_STATUS" == true ]]; then echo "$ICON_CONTAINERS Required: ${WATCHDOG_REQUIRED_CONTAINERS[*]:-none}" echo "$ICON_WATCHDOG Scan all: $WATCHDOG_SCAN_ALL" echo "$ICON_WATCHDOG Ignore: ${WATCHDOG_SCAN_IGNORE[*]:-none}" - _ms_list=$(cat "$DOCKER_WATCHDOG_MANUAL_STOP_FILE" 2>/dev/null | tr '\n' ' ' | xargs) - echo "$ICON_WATCHDOG Manual-stop: ${_ms_list:-none}" echo "$ICON_WATCHDOG Schedule: every 15 min (cron via watchdog_orchestrator)" echo "$ICON_WATCHDOG Startup grace: ${WATCHDOG_STARTUP_GRACE}s" echo "$ICON_WATCHDOG Restart limit: $WATCHDOG_CONTAINER_RESTART_LIMIT in ${WATCHDOG_CONTAINER_RESTART_WINDOW}h" @@ -330,28 +332,6 @@ remove_from_skip_list() { warn "$1 recovered — removed from skip list ✅" } -# Check if container was intentionally stopped (exit 0/143 — clean/SIGTERM) -is_manually_stopped() { - grep -q "^${1}$" "$DOCKER_WATCHDOG_MANUAL_STOP_FILE" 2>/dev/null -} - -# Mark container as intentionally stopped — auto-cleared when seen running again -add_to_manual_stop() { - local container="$1" exit_code="$2" - if ! is_manually_stopped "$container"; then - echo "$container" >> "$DOCKER_WATCHDOG_MANUAL_STOP_FILE" - log "$container — stopped cleanly (exit $exit_code) — skipping until restarted" - fi -} - -# Remove container from manual-stop list — called when container is seen running again -remove_from_manual_stop() { - if is_manually_stopped "$1"; then - sed -i "/^${1}$/d" "$DOCKER_WATCHDOG_MANUAL_STOP_FILE" 2>/dev/null - log "$1 — running again — removed from manual-stop list ✅" - fi -} - # Log a restart event to the rolling restart history file log_restart() { local container="$1" @@ -656,10 +636,6 @@ CYCLE_START=$(date +%s) _skip_contents=$(cat "$DOCKER_WATCHDOG_FAILED_FILE" 2>/dev/null | tr '\n' ' ' | xargs) [[ -n "$_skip_contents" ]] && warn "$ICON_SKIP Skip list active: $_skip_contents — manual intervention needed" - local _manual_stop_contents - _manual_stop_contents=$(cat "$DOCKER_WATCHDOG_MANUAL_STOP_FILE" 2>/dev/null | tr '\n' ' ' | xargs) - [[ -n "$_manual_stop_contents" ]] && log "$ICON_SKIP Manual-stop list: $_manual_stop_contents" - # ── Docker daemon health check — first check every run ────────────────────────────────── # If daemon is hung all container operations will fail — check first, skip run if down if ! check_docker_daemon; then @@ -716,36 +692,26 @@ CYCLE_START=$(date +%s) fi if [[ "$STATUS" == "true" ]]; then - # Running — clear any strikes and manual-stop flag - remove_from_manual_stop "$container" + # Running — clear any strikes set_strikes "$container" 0 "$WATCHDOG_STATE_FILE" log "$ICON_RUNNING $container — running ✅" - elif is_manually_stopped "$container"; then - log "$container — manually stopped — skipping" else - # Check if this was a clean/intentional stop before striking - EXIT_CODE=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \ - '{{.State.ExitCode}}' "$container" 2>/dev/null || echo "-1") - if [[ "$EXIT_CODE" == "0" || "$EXIT_CODE" == "143" ]]; then - add_to_manual_stop "$container" "$EXIT_CODE" - else - STRIKES=$(get_strikes "$container" "$WATCHDOG_STATE_FILE") - STRIKES=$(( STRIKES + 1 )) - set_strikes "$container" "$STRIKES" "$WATCHDOG_STATE_FILE" - warn "$container — not running (strike $STRIKES/$SYS_WATCHDOG_STRIKE_LIMIT)" - ((T1_WARNINGS++)) + STRIKES=$(get_strikes "$container" "$WATCHDOG_STATE_FILE") + STRIKES=$(( STRIKES + 1 )) + set_strikes "$container" "$STRIKES" "$WATCHDOG_STATE_FILE" + warn "$container — not running (strike $STRIKES/$SYS_WATCHDOG_STRIKE_LIMIT)" + ((T1_WARNINGS++)) - if [[ "$STRIKES" -ge "$SYS_WATCHDOG_STRIKE_LIMIT" ]]; then - result=0 - safe_restart "$container" "required container down" || result=$? - case $result in - 0) set_strikes "$container" 0 "$WATCHDOG_STATE_FILE" - ((T1_RESTARTS++)) - queue_notify "$container was down and restarted on $(hostname)" "warning" ;; - 2) : ;; # Added to skip list — already notified - *) queue_notify "$container failed to restart on $(hostname)" "warning" ;; - esac - fi + if [[ "$STRIKES" -ge "$SYS_WATCHDOG_STRIKE_LIMIT" ]]; then + result=0 + safe_restart "$container" "required container down" || result=$? + case $result in + 0) set_strikes "$container" 0 "$WATCHDOG_STATE_FILE" + ((T1_RESTARTS++)) + queue_notify "$container was down and restarted on $(hostname)" "warning" ;; + 2) : ;; # Added to skip list — already notified + *) queue_notify "$container failed to restart on $(hostname)" "warning" ;; + esac fi fi done @@ -843,6 +809,54 @@ CYCLE_START=$(date +%s) done fi + # ── API liveness checks ─────────────────────────────────────────────────────────────────── + # Catches containers that serve HTTP 200 but are internally frozen (DB lock, deadlocked + # thread, etc.). Endpoint must require a live DB round-trip to respond successfully. + # Format per entry: "URL|APIKey" + if [[ ${#WATCHDOG_CONTAINER_API_CHECKS[@]} -gt 0 ]]; then + for container in "${!WATCHDOG_CONTAINER_API_CHECKS[@]}"; do + IFS='|' read -r _api_url _api_key <<< "${WATCHDOG_CONTAINER_API_CHECKS[$container]}" + + # Skip entirely if key is absent or a placeholder — check is optional protection + if [[ -z "$_api_key" || "$_api_key" == "YOUR_API_KEY"* || "$_api_key" == "placeholder"* ]]; then + log "$container — API check skipped (no key configured)" + continue + fi + + _api_http=$(curl -s --max-time "$CURL_TIMEOUT" \ + -H "X-Emby-Token: ${_api_key}" \ + -o /tmp/_varaverk_api_check \ + -w "%{http_code}" \ + "$_api_url" 2>/dev/null) + _resp=$(cat /tmp/_varaverk_api_check 2>/dev/null) + + # 401/403 = wrong key — skip silently, don't penalise the container + if [[ "$_api_http" == "401" || "$_api_http" == "403" ]]; then + log "$container — API check skipped (HTTP $_api_http — key may be wrong or revoked)" + continue + fi + + if echo "$_resp" | jq -e '.ServerName // .Id // .Version' >/dev/null 2>&1; then + set_strikes "${container}_api" 0 "$WATCHDOG_STATE_FILE" + else + API_STRIKES=$(get_strikes "${container}_api" "$WATCHDOG_STATE_FILE") + API_STRIKES=$(( API_STRIKES + 1 )) + set_strikes "${container}_api" "$API_STRIKES" "$WATCHDOG_STATE_FILE" + warn "$container — API unresponsive at $_api_url (strike $API_STRIKES/$RESP_FAIL_LIMIT)" + ((T1_WARNINGS++)) + if [[ "$API_STRIKES" -ge "$RESP_FAIL_LIMIT" ]]; then + result=0 + safe_restart "$container" "API unresponsive at $_api_url" || result=$? + if [[ $result -eq 0 ]]; then + set_strikes "${container}_api" 0 "$WATCHDOG_STATE_FILE" + ((T1_RESTARTS++)) + queue_notify "$container API unresponsive at $_api_url on $(hostname) — restarted" "warning" + fi + fi + fi + done + fi + # ========================================================================================== # ── TIER 2 — Global Health Scan ─────────────────────────────────────────────────────────── # ========================================================================================== @@ -850,13 +864,6 @@ CYCLE_START=$(date +%s) ALL_CONTAINERS=$(timeout "$DOCKER_TIMEOUT" docker ps --format "{{.Names}}" 2>/dev/null) - # ── Auto-clear manual-stop list ─────────────────────────────────────────────────────── - # Any container now running was started intentionally — remove from manual-stop list - while IFS= read -r container; do - [[ -z "$container" ]] && continue - remove_from_manual_stop "$container" - done <<< "$ALL_CONTAINERS" - # ── Unhealthy containers ────────────────────────────────────────────────────────────── if [[ "$WATCHDOG_RESTART_UNHEALTHY" == "true" ]]; then UNHEALTHY=$(timeout "$DOCKER_TIMEOUT" docker ps \ @@ -961,13 +968,13 @@ CYCLE_START=$(date +%s) fi # ── Unexpected exits ────────────────────────────────────────────────────────────────── - # Exit 0/143 = clean/SIGTERM — intentional stop, add to manual-stop list and skip. - # All other non-zero exits = crash — restart via safe_restart. + # Only non-zero exit codes — exit 0 is a clean stop, not a crash # Skips containers already covered by WATCHDOG_REQUIRED_CONTAINERS (handled in Tier 1) if [[ "$WATCHDOG_RESTART_CRASHED" == "true" ]]; then - EXITED=$(timeout "$DOCKER_TIMEOUT" docker ps -a \ + CRASHED=$(timeout "$DOCKER_TIMEOUT" docker ps -a \ --filter status=exited \ - --format "{{.Names}}|{{.Status}}" 2>/dev/null) + --format "{{.Names}}|{{.Status}}" 2>/dev/null | \ + grep -v "Exited (0)") while IFS='|' read -r container status; do [[ -z "$container" ]] && continue [[ -n "${IGNORE_MAP[$container]:-}" ]] && continue @@ -978,20 +985,6 @@ CYCLE_START=$(date +%s) [[ "$container" == "$req" ]] && already_required=true && break done [[ "$already_required" == true ]] && continue - - # Extract exit code from status string e.g. "Exited (143) 2 hours ago" - EXIT_CODE="-1" - [[ "$status" =~ Exited\ \(([0-9]+)\) ]] && EXIT_CODE="${BASH_REMATCH[1]}" - - # Clean/intentional stop — add to manual-stop list and skip - if [[ "$EXIT_CODE" == "0" || "$EXIT_CODE" == "143" ]]; then - add_to_manual_stop "$container" "$EXIT_CODE" - continue - fi - - # Already known to be manually stopped from a prior cycle - is_manually_stopped "$container" && continue - error "$container — $status (unexpected exit)" ((T2_WARNINGS++)) result=0 @@ -1000,7 +993,7 @@ CYCLE_START=$(date +%s) ((T2_RESTARTS++)) queue_notify "$container crashed on $(hostname) ($status) — restarted" "warning" fi - done <<< "$EXITED" + done <<< "$CRASHED" fi fi # WATCHDOG_SCAN_ALL diff --git a/common.sh b/common.sh index 8841067..e8a3f7a 100755 --- a/common.sh +++ b/common.sh @@ -600,6 +600,7 @@ detect_hosts() { _alias_assoc "WATCHDOG_CONTAINERS" _alias_assoc "WATCHDOG_CONTAINER_URLS" + _alias_assoc "WATCHDOG_CONTAINER_API_CHECKS" _alias_assoc "WATCHDOG_DEPENDENCIES" _alias_assoc "WATCHDOG_APPDATA_SIZES"