audit echo vs log across all scripts — outcomes always visible, verbose for per-item loops
This commit is contained in:
Gmer4Lfe
@@ -0,0 +1,413 @@
|
||||
<?php
|
||||
$_myHost = vv_detect_host();
|
||||
$_vars = vv_conf_vars();
|
||||
$_myId = strtoupper($_myHost);
|
||||
|
||||
// Storage
|
||||
$_confKey = $_myId . '_STORAGE_MODE_INTERNAL';
|
||||
$_confVal = $_vars[$_confKey] ?? null;
|
||||
|
||||
// Notifications
|
||||
$_notifyUnraid = ($_vars['NOTIFY_UNRAID'] ?? 'true') !== 'false';
|
||||
$_enableLogging = ($_vars['ENABLE_LOGGING'] ?? 'false') === 'true';
|
||||
$_discordKey = $_myId . '_DISCORD_WEBHOOK';
|
||||
$_discordHook = $_vars[$_discordKey] ?? '';
|
||||
|
||||
// API key
|
||||
$_apiKeyVar = $_myId . '_UNRAID_API_KEY';
|
||||
$_apiKey = $_vars[$_apiKeyVar] ?? '';
|
||||
$_apiPreview = $_apiKey ? substr($_apiKey, 0, 8) . '...' . substr($_apiKey, -4) : '';
|
||||
?>
|
||||
<style>
|
||||
.vv-set-card { background:#161616;border:1px solid #2a2a2a;border-radius:6px;padding:14px 16px;margin-bottom:14px; }
|
||||
.vv-set-hdr { font-size:11px;font-weight:700;color:#666;text-transform:uppercase;letter-spacing:.07em;margin-bottom:12px; }
|
||||
.vv-set-row { display:flex;justify-content:space-between;align-items:baseline;gap:8px;margin:4px 0; }
|
||||
.vv-set-lbl { font-size:11px;color:#444; }
|
||||
.vv-set-val { font-size:12px;color:#888;text-align:right;font-family:monospace; }
|
||||
.vv-set-badge { display:inline-block;font-size:10px;padding:2px 8px;border-radius:3px;font-weight:700;letter-spacing:.04em; }
|
||||
.vv-set-badge.internal { background:#0d1f0d;color:#4caf50;border:1px solid #1a3a1a; }
|
||||
.vv-set-badge.flash { background:#1a1200;color:#ffb74d;border:1px solid #3a2800; }
|
||||
.vv-set-badge.custom { background:#0a1a2a;color:#4a9eff;border:1px solid #1a3a5a; }
|
||||
.vv-set-sep { border:none;border-top:1px solid #1e1e1e;margin:10px 0; }
|
||||
.vv-set-btn { background:#1a1a1a;border:1px solid #333;color:#888;font-size:11px;padding:5px 14px;
|
||||
border-radius:3px;cursor:pointer;transition:border-color .15s,color .15s; }
|
||||
.vv-set-btn:hover { border-color:#555;color:#ccc; }
|
||||
.vv-set-btn:disabled { opacity:.4;cursor:default; }
|
||||
.vv-set-btn.primary { background:#1a2a1a;border-color:#2d4a2d;color:#4caf50; }
|
||||
.vv-set-btn.primary:hover { background:#223a22; }
|
||||
.vv-set-btn.warn { background:#1f1500;border-color:#3a2800;color:#ffb74d; }
|
||||
.vv-set-btn.warn:hover { background:#2a1e00; }
|
||||
.vv-set-out { background:#080808;border:1px solid #1a1a1a;border-radius:3px;padding:10px 12px;
|
||||
font-family:monospace;font-size:10px;color:#555;white-space:pre-wrap;word-break:break-all;
|
||||
max-height:320px;overflow-y:auto;margin-top:10px;display:none;
|
||||
scrollbar-width:none; }
|
||||
.vv-set-out::-webkit-scrollbar { display:none; }
|
||||
.vv-set-info { font-size:11px;color:#444;line-height:1.6;margin-bottom:10px; }
|
||||
.vv-set-warn-box { background:#1a1200;border:1px solid #3a2800;border-radius:4px;
|
||||
padding:8px 12px;font-size:11px;color:#ffb74d;margin-top:8px;display:none; }
|
||||
.vv-set-tog-wrap { display:flex;align-items:center;gap:10px;cursor:pointer;user-select:none; }
|
||||
.vv-set-tog-track{ width:32px;height:18px;border-radius:9px;background:#222;border:1px solid #333;
|
||||
position:relative;transition:background .15s,border-color .15s;flex-shrink:0; }
|
||||
.vv-set-tog-track.on { background:#1a3a1a;border-color:#2d5a2d; }
|
||||
.vv-set-tog-track::after { content:'';position:absolute;top:2px;left:2px;width:12px;height:12px;
|
||||
border-radius:50%;background:#555;transition:left .15s,background .15s; }
|
||||
.vv-set-tog-track.on::after { left:16px;background:#4caf50; }
|
||||
.vv-set-tog-lbl { font-size:12px;color:#888; }
|
||||
.vv-set-tog-sub { font-size:10px;color:#3a3a3a;margin-top:1px; }
|
||||
.vv-set-inp { background:#0d0d0d;border:1px solid #2a2a2a;border-radius:3px;color:#888;
|
||||
font-size:12px;padding:5px 8px;outline:none;width:100%;box-sizing:border-box;
|
||||
font-family:monospace; }
|
||||
.vv-set-inp:focus{ border-color:#444; }
|
||||
</style>
|
||||
|
||||
<div style="max-width:740px;margin:0 auto;">
|
||||
|
||||
<!-- Storage Location card -->
|
||||
<div class="vv-set-card" id="vv-stor-card">
|
||||
<div style="display:flex;justify-content:space-between;align-items:center;margin-bottom:12px;">
|
||||
<span class="vv-set-hdr" style="margin-bottom:0;">Storage Location</span>
|
||||
<span id="vv-stor-badge" class="vv-set-badge">…</span>
|
||||
</div>
|
||||
|
||||
<div id="vv-stor-info" style="font-size:11px;color:#3a3a3a;">Loading…</div>
|
||||
|
||||
<hr class="vv-set-sep">
|
||||
|
||||
<div class="vv-set-info">
|
||||
Varaverk stores all scripts, configuration, state files, and the git repository in
|
||||
<code style="font-size:10px;color:#4a7a4a;background:#080808;padding:1px 4px;border-radius:2px;">SCRIPTS_DIR</code>.
|
||||
<br>
|
||||
<strong style="color:#555;">Internal NVMe/SSD</strong> — direct access, zero write-wear concern, git pull/push from boot volume.<br>
|
||||
<strong style="color:#555;">USB Flash</strong> — preserves flash lifetime. Requires array to be started. git pull syncs
|
||||
<code style="font-size:10px;color:#4a7a4a;background:#080808;padding:1px 4px;border-radius:2px;">Plugin/</code>
|
||||
back to <code style="font-size:10px;color:#4a7a4a;background:#080808;padding:1px 4px;border-radius:2px;">/boot/</code>
|
||||
after each pull so the webUI always stays current.
|
||||
</div>
|
||||
|
||||
<div id="vv-stor-warn-flash" class="vv-set-warn-box">
|
||||
Array must be started for Varaverk to function in flash mode. The plugin tab is always accessible.
|
||||
</div>
|
||||
<div id="vv-stor-warn-noarray" class="vv-set-warn-box">
|
||||
Array is not started — cannot migrate to flash mode right now. Start the array first.
|
||||
</div>
|
||||
|
||||
<div style="display:flex;gap:8px;align-items:center;flex-wrap:wrap;margin-top:10px;">
|
||||
<button id="vv-stor-migrate-btn" class="vv-set-btn" onclick="vvStorMigrate()" style="display:none;"></button>
|
||||
<button class="vv-set-btn" onclick="vvStorLoad()" style="font-size:10px;padding:4px 10px;">↻ Refresh</button>
|
||||
<span id="vv-stor-fb" style="font-size:11px;color:#444;"></span>
|
||||
</div>
|
||||
|
||||
<pre class="vv-set-out" id="vv-stor-out"></pre>
|
||||
</div>
|
||||
|
||||
<!-- Notifications card -->
|
||||
<div class="vv-set-card">
|
||||
<div class="vv-set-hdr">Notifications</div>
|
||||
|
||||
<div style="display:flex;flex-direction:column;gap:10px;margin-bottom:14px;">
|
||||
<label class="vv-set-tog-wrap">
|
||||
<div class="vv-set-tog-track<?= $_notifyUnraid ? ' on' : '' ?>" id="vv-ntf-unraid"
|
||||
onclick="vvNtfToggle(this,'NOTIFY_UNRAID')"></div>
|
||||
<div>
|
||||
<div class="vv-set-tog-lbl">Unraid native notifications</div>
|
||||
<div class="vv-set-tog-sub">Appears in the Unraid bell icon — job completions, warnings, errors</div>
|
||||
</div>
|
||||
</label>
|
||||
<label class="vv-set-tog-wrap">
|
||||
<div class="vv-set-tog-track<?= $_enableLogging ? ' on' : '' ?>" id="vv-ntf-logging"
|
||||
onclick="vvNtfToggle(this,'ENABLE_LOGGING')"></div>
|
||||
<div>
|
||||
<div class="vv-set-tog-lbl">Verbose logging</div>
|
||||
<div class="vv-set-tog-sub">Adds detailed [LOG] lines to script output — useful for debugging scheduled runs</div>
|
||||
</div>
|
||||
</label>
|
||||
</div>
|
||||
|
||||
<hr class="vv-set-sep">
|
||||
|
||||
<div style="margin-bottom:6px;">
|
||||
<div style="font-size:11px;color:#555;margin-bottom:5px;">
|
||||
Discord webhook
|
||||
<span style="color:#2a2a2a;font-size:10px;margin-left:4px;"><?= htmlspecialchars($_discordKey) ?></span>
|
||||
</div>
|
||||
<div style="display:flex;gap:6px;align-items:center;">
|
||||
<input class="vv-set-inp" id="vv-ntf-discord" type="text"
|
||||
placeholder="https://discord.com/api/webhooks/…"
|
||||
value="<?= htmlspecialchars($_discordHook) ?>">
|
||||
<button class="vv-set-btn primary" onclick="vvNtfSaveWebhook()" id="vv-ntf-save"
|
||||
style="white-space:nowrap;flex-shrink:0;">Save</button>
|
||||
</div>
|
||||
<div style="font-size:10px;color:#2a2a2a;margin-top:4px;">Leave blank to disable Discord notifications for this host</div>
|
||||
</div>
|
||||
<span id="vv-ntf-fb" style="font-size:11px;"></span>
|
||||
</div>
|
||||
|
||||
<!-- Unraid API Key card -->
|
||||
<div class="vv-set-card">
|
||||
<div style="display:flex;justify-content:space-between;align-items:center;margin-bottom:12px;">
|
||||
<span class="vv-set-hdr" style="margin-bottom:0;">Unraid API Keys</span>
|
||||
<span id="vv-api-badge" class="vv-set-badge">…</span>
|
||||
</div>
|
||||
|
||||
<div id="vv-api-hosts" style="margin-bottom:12px;"></div>
|
||||
|
||||
<div style="font-size:11px;color:#444;margin-bottom:12px;line-height:1.6;">
|
||||
Each host registers
|
||||
<code style="font-size:10px;color:#4a7a4a;background:#080808;padding:1px 4px;border-radius:2px;">Varaverk_HOST1</code>
|
||||
in <em>every</em> machine's local Unraid registry — own machine plus all partners.
|
||||
The key values are stored in the private <code style="font-size:10px;color:#4a7a4a;background:#080808;padding:1px 4px;border-radius:2px;">host*.conf</code>
|
||||
so each host can query every other host's GraphQL API directly — real-time monitoring, no SSH.
|
||||
During offboard, partner keys are removed from each registry automatically.
|
||||
</div>
|
||||
|
||||
<div style="display:flex;gap:8px;align-items:center;flex-wrap:wrap;">
|
||||
<button class="vv-set-btn" onclick="vvApiCheck()" id="vv-api-check-btn">↻ Check status</button>
|
||||
<button class="vv-set-btn" onclick="vvApiRenew(false)" id="vv-api-local-btn">Renew local</button>
|
||||
<button class="vv-set-btn primary" onclick="vvApiRenew(true)" id="vv-api-all-btn">Setup all host keys</button>
|
||||
<span id="vv-api-fb" style="font-size:11px;color:#444;"></span>
|
||||
</div>
|
||||
<pre class="vv-set-out" id="vv-api-out"></pre>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
<script>
|
||||
var _vvStorData = null;
|
||||
|
||||
function vvStorLoad() {
|
||||
const info = document.getElementById('vv-stor-info');
|
||||
const badge = document.getElementById('vv-stor-badge');
|
||||
if (info) info.textContent = 'Loading…';
|
||||
|
||||
fetch('/plugins/varaverk/api/storage.php?action=status&_=' + Date.now())
|
||||
.then(r => r.json())
|
||||
.then(d => {
|
||||
_vvStorData = d;
|
||||
if (!d.ok) { if (info) info.textContent = 'Error loading status'; return; }
|
||||
_vvStorRender(d);
|
||||
})
|
||||
.catch(() => { if (info) info.textContent = 'Request failed'; });
|
||||
}
|
||||
|
||||
function _vvStorRender(d) {
|
||||
const badge = document.getElementById('vv-stor-badge');
|
||||
const info = document.getElementById('vv-stor-info');
|
||||
const btn = document.getElementById('vv-stor-migrate-btn');
|
||||
const wFlash = document.getElementById('vv-stor-warn-flash');
|
||||
const wArr = document.getElementById('vv-stor-warn-noarray');
|
||||
|
||||
// Badge
|
||||
if (badge) {
|
||||
badge.textContent = d.current_mode.toUpperCase();
|
||||
badge.className = 'vv-set-badge ' + d.current_mode;
|
||||
}
|
||||
|
||||
// Info grid
|
||||
const modeLabel = d.current_mode === 'internal' ? 'Internal NVMe/SSD' : d.current_mode === 'flash' ? 'USB Flash' : 'Custom';
|
||||
const detLabel = d.detected === 'internal' ? 'internal NVMe/SSD' : 'USB flash';
|
||||
const confBadge = d.conf_val === null ? '<span style="color:#3a3a3a;">not set</span>'
|
||||
: d.conf_val === 'true' ? '<span style="color:#4caf50;">true (internal)</span>'
|
||||
: '<span style="color:#ffb74d;">false (flash)</span>';
|
||||
const matchIcon = (d.current_mode === d.detected || d.current_mode === 'custom') ? '' :
|
||||
' <span style="color:#ef5350;">⚠ mismatch with conf</span>';
|
||||
|
||||
if (info) info.innerHTML = `
|
||||
<div style="display:grid;grid-template-columns:auto 1fr;gap:3px 16px;align-items:baseline;">
|
||||
<span style="color:#333;">Mode</span> <span style="color:#888;">${modeLabel}${matchIcon}</span>
|
||||
<span style="color:#333;">SCRIPTS_DIR</span> <code style="font-size:10px;color:#4a7a4a;">${d.current_dir}</code>
|
||||
<span style="color:#333;">Boot device</span> <span style="color:#555;">${d.boot_disk} <span style="color:#2a2a2a;">(${d.transport})</span></span>
|
||||
<span style="color:#333;">Auto-detect</span> <span style="color:#555;">${detLabel}</span>
|
||||
<span style="color:#333;">${d.conf_key}</span> <span>${confBadge}</span>
|
||||
</div>`;
|
||||
|
||||
// Warnings
|
||||
if (wFlash) wFlash.style.display = (d.current_mode !== 'flash' && d.detected === 'usb') ? '' : 'none';
|
||||
if (wArr) wArr.style.display = 'none';
|
||||
|
||||
// Migrate button
|
||||
if (btn) {
|
||||
if (d.current_mode === 'internal') {
|
||||
btn.textContent = 'Migrate to Flash (appdata)';
|
||||
btn.className = 'vv-set-btn warn';
|
||||
btn.dataset.to = 'flash';
|
||||
btn.style.display = '';
|
||||
} else if (d.current_mode === 'flash') {
|
||||
btn.textContent = 'Migrate to Internal (/boot)';
|
||||
btn.className = 'vv-set-btn primary';
|
||||
btn.dataset.to = 'internal';
|
||||
btn.style.display = '';
|
||||
} else {
|
||||
btn.style.display = 'none';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function vvStorMigrate() {
|
||||
const btn = document.getElementById('vv-stor-migrate-btn');
|
||||
const out = document.getElementById('vv-stor-out');
|
||||
const fb = document.getElementById('vv-stor-fb');
|
||||
const wArr = document.getElementById('vv-stor-warn-noarray');
|
||||
const to = btn?.dataset.to;
|
||||
if (!to) return;
|
||||
|
||||
// Flash guard: array must be started
|
||||
if (to === 'flash' && _vvStorData && !_vvStorData.array_started) {
|
||||
if (wArr) wArr.style.display = '';
|
||||
return;
|
||||
}
|
||||
|
||||
const label = to === 'flash' ? 'Flash (appdata)' : 'Internal (/boot)';
|
||||
if (!confirm(`Migrate Varaverk storage to ${label}?\n\nThis will:\n• Copy all scripts, conf, and git repo to the new location\n• Update varaverk.cfg and master.conf\n• Delete the old location\n\nThe page will need a reload after migration.`)) return;
|
||||
|
||||
btn.disabled = true;
|
||||
btn.textContent = 'Migrating…';
|
||||
if (fb) fb.textContent = '';
|
||||
if (out) { out.textContent = 'Starting migration…\n'; out.style.display = ''; }
|
||||
|
||||
const fd = new FormData();
|
||||
fd.append('action', 'migrate');
|
||||
fd.append('to', to);
|
||||
|
||||
fetch('/plugins/varaverk/api/storage.php', { method: 'POST', body: fd })
|
||||
.then(r => r.json())
|
||||
.then(d => {
|
||||
btn.disabled = false;
|
||||
if (out) {
|
||||
out.textContent = d.output || '(no output)';
|
||||
out.scrollTop = out.scrollHeight;
|
||||
}
|
||||
if (d.ok) {
|
||||
btn.textContent = '✓ Done — reload page';
|
||||
btn.className = 'vv-set-btn primary';
|
||||
btn.onclick = () => location.reload();
|
||||
if (fb) { fb.style.color = '#4caf50'; fb.textContent = 'Migration complete — reload to apply'; }
|
||||
} else {
|
||||
btn.textContent = btn.dataset.to === 'flash' ? 'Migrate to Flash (appdata)' : 'Migrate to Internal (/boot)';
|
||||
if (fb) { fb.style.color = '#ef5350'; fb.textContent = 'Migration failed — see output above'; }
|
||||
}
|
||||
})
|
||||
.catch(e => {
|
||||
btn.disabled = false;
|
||||
btn.textContent = btn.dataset.to === 'flash' ? 'Migrate to Flash (appdata)' : 'Migrate to Internal (/boot)';
|
||||
if (fb) { fb.style.color = '#ef5350'; fb.textContent = 'Request failed: ' + e; }
|
||||
});
|
||||
}
|
||||
|
||||
vvStorLoad();
|
||||
|
||||
// ── Notifications ─────────────────────────────────────────────────────────────
|
||||
function vvNtfToggle(track, key) {
|
||||
const on = !track.classList.contains('on');
|
||||
track.classList.toggle('on', on);
|
||||
const file = (key === 'NOTIFY_UNRAID' || key === 'ENABLE_LOGGING') ? 'master.conf'
|
||||
: '<?= htmlspecialchars($_myHost) ?>.conf';
|
||||
const fd = new FormData();
|
||||
fd.append('id', 'settings');
|
||||
fd.append('changes', JSON.stringify([{ file, key, value: on ? 'true' : 'false', type: 'scalar' }]));
|
||||
fetch('/plugins/varaverk/api/confform.php', { method: 'POST', body: fd })
|
||||
.then(r => r.json())
|
||||
.then(d => { if (!d.ok) track.classList.toggle('on', !on); })
|
||||
.catch(() => track.classList.toggle('on', !on));
|
||||
}
|
||||
|
||||
function vvNtfSaveWebhook() {
|
||||
const inp = document.getElementById('vv-ntf-discord');
|
||||
const fb = document.getElementById('vv-ntf-fb');
|
||||
const btn = document.getElementById('vv-ntf-save');
|
||||
btn.disabled = true; btn.textContent = 'Saving…'; fb.textContent = '';
|
||||
const fd = new FormData();
|
||||
fd.append('id', 'settings');
|
||||
fd.append('changes', JSON.stringify([{
|
||||
file: '<?= htmlspecialchars($_myHost) ?>.conf',
|
||||
key: '<?= htmlspecialchars($_discordKey) ?>',
|
||||
value: inp.value.trim(),
|
||||
type: 'scalar',
|
||||
}]));
|
||||
fetch('/plugins/varaverk/api/confform.php', { method: 'POST', body: fd })
|
||||
.then(r => r.json())
|
||||
.then(d => {
|
||||
btn.disabled = false; btn.textContent = 'Save';
|
||||
fb.style.color = d.ok ? '#4caf50' : '#ef5350';
|
||||
fb.textContent = d.ok ? 'Saved ✓' : (d.error || 'Failed');
|
||||
if (d.ok) setTimeout(() => { fb.textContent = ''; }, 3000);
|
||||
})
|
||||
.catch(() => { btn.disabled = false; btn.textContent = 'Save'; fb.style.color='#ef5350'; fb.textContent='Request failed'; });
|
||||
}
|
||||
|
||||
// ── Unraid API Keys ───────────────────────────────────────────────────────────
|
||||
function vvApiCheck() {
|
||||
const btn = document.getElementById('vv-api-check-btn');
|
||||
const badge = document.getElementById('vv-api-badge');
|
||||
const hosts = document.getElementById('vv-api-hosts');
|
||||
btn.disabled = true; btn.textContent = 'Checking…';
|
||||
fetch('/plugins/varaverk/api/storage.php?action=api_status&_=' + Date.now())
|
||||
.then(r => r.json())
|
||||
.then(d => {
|
||||
btn.disabled = false; btn.textContent = '↻ Check status';
|
||||
if (!d.ok) return;
|
||||
const allOk = d.hosts.every(h => h.api_ok);
|
||||
const anyMissing = d.hosts.some(h => !h.key_present);
|
||||
badge.textContent = allOk ? 'all ok' : anyMissing ? 'keys missing' : 'partial';
|
||||
badge.className = 'vv-set-badge ' + (allOk ? 'internal' : 'flash');
|
||||
|
||||
if (hosts) hosts.innerHTML = d.hosts.map(h => {
|
||||
const tag = h.is_local
|
||||
? `<span style="color:#2a3a2a;font-size:9px;margin-left:4px;">own</span>`
|
||||
: `<span style="color:#2a2a3a;font-size:9px;margin-left:4px;">${d.my_id}'s key on ${h.host_id}</span>`;
|
||||
const keyBit = h.key_present
|
||||
? `<code style="font-size:9px;color:#4a6a4a;">${h.key_preview}</code>`
|
||||
: `<span style="color:#8b2a2a;font-size:10px;">not set — click Setup</span>`;
|
||||
const dot = h.api_ok ? '#4caf50' : (h.key_present ? '#ffb74d' : '#444');
|
||||
// Key name in that machine's registry: always Varaverk_<MY_ID>
|
||||
const keyName = `Varaverk_${d.my_id}`;
|
||||
const registry = h.is_local ? h.host_id : h.host_id;
|
||||
return `<div style="display:flex;align-items:center;gap:8px;padding:4px 0;border-bottom:1px solid #111;">
|
||||
<span style="width:6px;height:6px;border-radius:50%;background:${dot};flex-shrink:0;display:inline-block;"></span>
|
||||
<span style="font-size:11px;color:#666;flex-shrink:0;min-width:50px;">${h.host_id}</span>
|
||||
<code style="font-size:9px;color:#2a3a2a;flex-shrink:0;">${keyName}</code>
|
||||
<span style="font-size:9px;color:#2a2a2a;flex-shrink:0;">on ${registry}</span>
|
||||
${tag}
|
||||
<span style="flex:1;text-align:right;">${keyBit}</span>
|
||||
</div>`;
|
||||
}).join('');
|
||||
})
|
||||
.catch(() => { btn.disabled = false; btn.textContent = '↻ Check status'; });
|
||||
}
|
||||
|
||||
function vvApiRenew(allHosts) {
|
||||
const localBtn = document.getElementById('vv-api-local-btn');
|
||||
const allBtn = document.getElementById('vv-api-all-btn');
|
||||
const out = document.getElementById('vv-api-out');
|
||||
const fb = document.getElementById('vv-api-fb');
|
||||
const badge = document.getElementById('vv-api-badge');
|
||||
const activeBtn = allHosts ? allBtn : localBtn;
|
||||
|
||||
[localBtn, allBtn].forEach(b => { if(b) b.disabled = true; });
|
||||
activeBtn.textContent = allHosts ? 'Setting up…' : 'Renewing…';
|
||||
fb.textContent = ''; out.textContent = ''; out.style.display = '';
|
||||
|
||||
const fd = new FormData();
|
||||
fd.append('action', 'setup_apikeys');
|
||||
fd.append('all_hosts', allHosts ? '1' : '0');
|
||||
|
||||
fetch('/plugins/varaverk/api/storage.php', { method: 'POST', body: fd })
|
||||
.then(r => r.json())
|
||||
.then(d => {
|
||||
localBtn.disabled = false; localBtn.textContent = 'Renew local';
|
||||
allBtn.disabled = false; allBtn.textContent = 'Setup all host keys';
|
||||
out.textContent = d.output || '(no output)';
|
||||
out.scrollTop = out.scrollHeight;
|
||||
fb.style.color = d.ok ? '#4caf50' : '#ef5350';
|
||||
fb.textContent = d.ok ? '✓ Done' : 'Failed — see output';
|
||||
if (d.ok) { badge.textContent = ''; vvApiCheck(); }
|
||||
})
|
||||
.catch(() => {
|
||||
localBtn.disabled = false; localBtn.textContent = 'Renew local';
|
||||
allBtn.disabled = false; allBtn.textContent = 'Setup all host keys';
|
||||
fb.style.color = '#ef5350'; fb.textContent = 'Request failed';
|
||||
});
|
||||
}
|
||||
|
||||
// Auto-check on load
|
||||
vvApiCheck();
|
||||
</script>
|
||||
@@ -0,0 +1,394 @@
|
||||
<?php
|
||||
$_myHost = vv_detect_host();
|
||||
$_vars = vv_conf_vars();
|
||||
$_myId = strtoupper($_myHost);
|
||||
|
||||
// Storage
|
||||
$_confKey = $_myId . '_STORAGE_MODE_INTERNAL';
|
||||
$_confVal = $_vars[$_confKey] ?? null;
|
||||
|
||||
// Notifications
|
||||
$_notifyUnraid = ($_vars['NOTIFY_UNRAID'] ?? 'true') !== 'false';
|
||||
$_enableLogging = ($_vars['ENABLE_LOGGING'] ?? 'false') === 'true';
|
||||
$_discordKey = $_myId . '_DISCORD_WEBHOOK';
|
||||
$_discordHook = $_vars[$_discordKey] ?? '';
|
||||
|
||||
?>
|
||||
<style>
|
||||
.vv-set-card { background:#161616;border:1px solid #2a2a2a;border-radius:6px;padding:14px 16px;margin-bottom:14px; }
|
||||
.vv-set-hdr { font-size:11px;font-weight:700;color:#666;text-transform:uppercase;letter-spacing:.07em;margin-bottom:12px; }
|
||||
.vv-set-row { display:flex;justify-content:space-between;align-items:baseline;gap:8px;margin:4px 0; }
|
||||
.vv-set-lbl { font-size:11px;color:#444; }
|
||||
.vv-set-val { font-size:12px;color:#888;text-align:right;font-family:monospace; }
|
||||
.vv-set-badge { display:inline-block;font-size:10px;padding:2px 8px;border-radius:3px;font-weight:700;letter-spacing:.04em; }
|
||||
.vv-set-badge.internal { background:#0d1f0d;color:#4caf50;border:1px solid #1a3a1a; }
|
||||
.vv-set-badge.flash { background:#1a1200;color:#ffb74d;border:1px solid #3a2800; }
|
||||
.vv-set-badge.custom { background:#0a1a2a;color:#4a9eff;border:1px solid #1a3a5a; }
|
||||
.vv-set-sep { border:none;border-top:1px solid #1e1e1e;margin:10px 0; }
|
||||
.vv-set-btn { background:#1a1a1a;border:1px solid #333;color:#888;font-size:11px;padding:5px 14px;
|
||||
border-radius:3px;cursor:pointer;transition:border-color .15s,color .15s; }
|
||||
.vv-set-btn:hover { border-color:#555;color:#ccc; }
|
||||
.vv-set-btn:disabled { opacity:.4;cursor:default; }
|
||||
.vv-set-btn.primary { background:#1a2a1a;border-color:#2d4a2d;color:#4caf50; }
|
||||
.vv-set-btn.primary:hover { background:#223a22; }
|
||||
.vv-set-btn.warn { background:#1f1500;border-color:#3a2800;color:#ffb74d; }
|
||||
.vv-set-btn.warn:hover { background:#2a1e00; }
|
||||
.vv-set-out { background:#080808;border:1px solid #1a1a1a;border-radius:3px;padding:10px 12px;
|
||||
font-family:monospace;font-size:10px;color:#555;white-space:pre-wrap;word-break:break-all;
|
||||
max-height:320px;overflow-y:auto;margin-top:10px;display:none;
|
||||
scrollbar-width:none; }
|
||||
.vv-set-out::-webkit-scrollbar { display:none; }
|
||||
.vv-set-info { font-size:11px;color:#444;line-height:1.6;margin-bottom:10px; }
|
||||
.vv-set-warn-box { background:#1a1200;border:1px solid #3a2800;border-radius:4px;
|
||||
padding:8px 12px;font-size:11px;color:#ffb74d;margin-top:8px;display:none; }
|
||||
.vv-set-tog-wrap { display:flex;align-items:center;gap:10px;cursor:pointer;user-select:none; }
|
||||
.vv-set-tog-track{ width:32px;height:18px;border-radius:9px;background:#222;border:1px solid #333;
|
||||
position:relative;transition:background .15s,border-color .15s;flex-shrink:0; }
|
||||
.vv-set-tog-track.on { background:#1a3a1a;border-color:#2d5a2d; }
|
||||
.vv-set-tog-track::after { content:'';position:absolute;top:2px;left:2px;width:12px;height:12px;
|
||||
border-radius:50%;background:#555;transition:left .15s,background .15s; }
|
||||
.vv-set-tog-track.on::after { left:16px;background:#4caf50; }
|
||||
.vv-set-tog-lbl { font-size:12px;color:#888; }
|
||||
.vv-set-tog-sub { font-size:10px;color:#3a3a3a;margin-top:1px; }
|
||||
.vv-set-inp { background:#0d0d0d;border:1px solid #2a2a2a;border-radius:3px;color:#888;
|
||||
font-size:12px;padding:5px 8px;outline:none;width:100%;box-sizing:border-box;
|
||||
font-family:monospace; }
|
||||
.vv-set-inp:focus{ border-color:#444; }
|
||||
</style>
|
||||
|
||||
<div style="max-width:740px;margin:0 auto;">
|
||||
|
||||
<!-- Storage Location card -->
|
||||
<div class="vv-set-card" id="vv-stor-card">
|
||||
<div style="display:flex;justify-content:space-between;align-items:center;margin-bottom:12px;">
|
||||
<span class="vv-set-hdr" style="margin-bottom:0;">Storage Location</span>
|
||||
<span id="vv-stor-badge" class="vv-set-badge">…</span>
|
||||
</div>
|
||||
|
||||
<div id="vv-stor-info" style="font-size:11px;color:#3a3a3a;">Loading…</div>
|
||||
|
||||
<hr class="vv-set-sep">
|
||||
|
||||
<div class="vv-set-info">
|
||||
Varaverk stores all scripts, configuration, state files, and the git repository in
|
||||
<code style="font-size:10px;color:#4a7a4a;background:#080808;padding:1px 4px;border-radius:2px;">SCRIPTS_DIR</code>.
|
||||
<br>
|
||||
<strong style="color:#555;">Internal NVMe/SSD</strong> — direct access, zero write-wear concern, git pull/push from boot volume.<br>
|
||||
<strong style="color:#555;">USB Flash</strong> — preserves flash lifetime. Requires array to be started. git pull syncs
|
||||
<code style="font-size:10px;color:#4a7a4a;background:#080808;padding:1px 4px;border-radius:2px;">Plugin/</code>
|
||||
back to <code style="font-size:10px;color:#4a7a4a;background:#080808;padding:1px 4px;border-radius:2px;">/boot/</code>
|
||||
after each pull so the webUI always stays current.
|
||||
</div>
|
||||
|
||||
<div id="vv-stor-warn-flash" class="vv-set-warn-box">
|
||||
Array must be started for Varaverk to function in flash mode. The plugin tab is always accessible.
|
||||
</div>
|
||||
<div id="vv-stor-warn-noarray" class="vv-set-warn-box">
|
||||
Array is not started — cannot migrate to flash mode right now. Start the array first.
|
||||
</div>
|
||||
|
||||
<div style="display:flex;gap:8px;align-items:center;flex-wrap:wrap;margin-top:10px;">
|
||||
<button id="vv-stor-migrate-btn" class="vv-set-btn" onclick="vvStorMigrate()" style="display:none;"></button>
|
||||
<button class="vv-set-btn" onclick="vvStorLoad()" style="font-size:10px;padding:4px 10px;">↻ Refresh</button>
|
||||
<span id="vv-stor-fb" style="font-size:11px;color:#444;"></span>
|
||||
</div>
|
||||
|
||||
<pre class="vv-set-out" id="vv-stor-out"></pre>
|
||||
</div>
|
||||
|
||||
<!-- Notifications card -->
|
||||
<div class="vv-set-card">
|
||||
<div class="vv-set-hdr">Notifications</div>
|
||||
|
||||
<div style="display:flex;flex-direction:column;gap:10px;margin-bottom:14px;">
|
||||
<label class="vv-set-tog-wrap">
|
||||
<div class="vv-set-tog-track<?= $_notifyUnraid ? ' on' : '' ?>" id="vv-ntf-unraid"
|
||||
onclick="vvNtfToggle(this,'NOTIFY_UNRAID')"></div>
|
||||
<div>
|
||||
<div class="vv-set-tog-lbl">Unraid native notifications</div>
|
||||
<div class="vv-set-tog-sub">Appears in the Unraid bell icon — job completions, warnings, errors</div>
|
||||
</div>
|
||||
</label>
|
||||
<label class="vv-set-tog-wrap">
|
||||
<div class="vv-set-tog-track<?= $_enableLogging ? ' on' : '' ?>" id="vv-ntf-logging"
|
||||
onclick="vvNtfToggle(this,'ENABLE_LOGGING')"></div>
|
||||
<div>
|
||||
<div class="vv-set-tog-lbl">Verbose logging</div>
|
||||
<div class="vv-set-tog-sub">Adds detailed [LOG] lines to script output — useful for debugging scheduled runs</div>
|
||||
</div>
|
||||
</label>
|
||||
</div>
|
||||
|
||||
<hr class="vv-set-sep">
|
||||
|
||||
<div style="margin-bottom:6px;">
|
||||
<div style="font-size:11px;color:#555;margin-bottom:5px;">
|
||||
Discord webhook
|
||||
<span style="color:#2a2a2a;font-size:10px;margin-left:4px;"><?= htmlspecialchars($_discordKey) ?></span>
|
||||
</div>
|
||||
<div style="display:flex;gap:6px;align-items:center;">
|
||||
<input class="vv-set-inp" id="vv-ntf-discord" type="text"
|
||||
placeholder="https://discord.com/api/webhooks/…"
|
||||
value="<?= htmlspecialchars($_discordHook) ?>">
|
||||
<button class="vv-set-btn primary" onclick="vvNtfSaveWebhook()" id="vv-ntf-save"
|
||||
style="white-space:nowrap;flex-shrink:0;">Save</button>
|
||||
</div>
|
||||
<div style="font-size:10px;color:#2a2a2a;margin-top:4px;">Leave blank to disable Discord notifications for this host</div>
|
||||
</div>
|
||||
<span id="vv-ntf-fb" style="font-size:11px;"></span>
|
||||
</div>
|
||||
|
||||
<!-- Unraid API Key card -->
|
||||
<div class="vv-set-card">
|
||||
<div style="display:flex;justify-content:space-between;align-items:center;margin-bottom:12px;">
|
||||
<span class="vv-set-hdr" style="margin-bottom:0;">Unraid API Keys</span>
|
||||
<span id="vv-api-badge" class="vv-set-badge">…</span>
|
||||
</div>
|
||||
|
||||
<div id="vv-api-hosts" style="margin-bottom:12px;"></div>
|
||||
|
||||
<div style="font-size:11px;color:#444;margin-bottom:12px;line-height:1.6;">
|
||||
Each host registers its key in the local Unraid registry and pushes it to all partners over SSH.
|
||||
Key values are stored in the private <code style="font-size:10px;color:#4a7a4a;background:#080808;padding:1px 4px;border-radius:2px;">host*.conf</code>
|
||||
so each host can query every other host's GraphQL API directly — real-time monitoring, no SSH.
|
||||
During offboard, partner keys are removed from each registry automatically.
|
||||
</div>
|
||||
|
||||
<div style="display:flex;gap:8px;align-items:center;flex-wrap:wrap;">
|
||||
<button class="vv-set-btn" onclick="vvApiCheck()" id="vv-api-check-btn">↻ Check status</button>
|
||||
<button class="vv-set-btn primary" onclick="vvApiRenew()" id="vv-api-renew-btn">Renew</button>
|
||||
<span id="vv-api-fb" style="font-size:11px;color:#444;"></span>
|
||||
</div>
|
||||
<pre class="vv-set-out" id="vv-api-out"></pre>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
<script>
|
||||
var _vvStorData = null;
|
||||
|
||||
function vvStorLoad() {
|
||||
const info = document.getElementById('vv-stor-info');
|
||||
const badge = document.getElementById('vv-stor-badge');
|
||||
if (info) info.textContent = 'Loading…';
|
||||
|
||||
fetch('/plugins/varaverk/api/storage.php?action=status&_=' + Date.now())
|
||||
.then(r => r.json())
|
||||
.then(d => {
|
||||
_vvStorData = d;
|
||||
if (!d.ok) { if (info) info.textContent = 'Error loading status'; return; }
|
||||
_vvStorRender(d);
|
||||
})
|
||||
.catch(() => { if (info) info.textContent = 'Request failed'; });
|
||||
}
|
||||
|
||||
function _vvStorRender(d) {
|
||||
const badge = document.getElementById('vv-stor-badge');
|
||||
const info = document.getElementById('vv-stor-info');
|
||||
const btn = document.getElementById('vv-stor-migrate-btn');
|
||||
const wFlash = document.getElementById('vv-stor-warn-flash');
|
||||
const wArr = document.getElementById('vv-stor-warn-noarray');
|
||||
|
||||
// Badge
|
||||
if (badge) {
|
||||
badge.textContent = d.current_mode.toUpperCase();
|
||||
badge.className = 'vv-set-badge ' + d.current_mode;
|
||||
}
|
||||
|
||||
// Info grid
|
||||
const modeLabel = d.current_mode === 'internal' ? 'Internal NVMe/SSD' : d.current_mode === 'flash' ? 'USB Flash' : 'Custom';
|
||||
const detLabel = d.detected === 'internal' ? 'internal NVMe/SSD' : 'USB flash';
|
||||
const confBadge = d.conf_val === null ? '<span style="color:#3a3a3a;">not set</span>'
|
||||
: d.conf_val === 'true' ? '<span style="color:#4caf50;">true (internal)</span>'
|
||||
: '<span style="color:#ffb74d;">false (flash)</span>';
|
||||
const matchIcon = (d.current_mode === d.detected || d.current_mode === 'custom') ? '' :
|
||||
' <span style="color:#ef5350;">⚠ mismatch with conf</span>';
|
||||
|
||||
if (info) info.innerHTML = `
|
||||
<div style="display:grid;grid-template-columns:auto 1fr;gap:3px 16px;align-items:baseline;">
|
||||
<span style="color:#333;">Mode</span> <span style="color:#888;">${modeLabel}${matchIcon}</span>
|
||||
<span style="color:#333;">SCRIPTS_DIR</span> <code style="font-size:10px;color:#4a7a4a;">${d.current_dir}</code>
|
||||
<span style="color:#333;">Boot device</span> <span style="color:#555;">${d.boot_disk} <span style="color:#2a2a2a;">(${d.transport})</span></span>
|
||||
<span style="color:#333;">Auto-detect</span> <span style="color:#555;">${detLabel}</span>
|
||||
<span style="color:#333;">${d.conf_key}</span> <span>${confBadge}</span>
|
||||
</div>`;
|
||||
|
||||
// Warnings
|
||||
if (wFlash) wFlash.style.display = (d.current_mode !== 'flash' && d.detected === 'usb') ? '' : 'none';
|
||||
if (wArr) wArr.style.display = 'none';
|
||||
|
||||
// Migrate button
|
||||
if (btn) {
|
||||
if (d.current_mode === 'internal') {
|
||||
btn.textContent = 'Migrate to Flash (appdata)';
|
||||
btn.className = 'vv-set-btn warn';
|
||||
btn.dataset.to = 'flash';
|
||||
btn.style.display = '';
|
||||
} else if (d.current_mode === 'flash') {
|
||||
btn.textContent = 'Migrate to Internal (/boot)';
|
||||
btn.className = 'vv-set-btn primary';
|
||||
btn.dataset.to = 'internal';
|
||||
btn.style.display = '';
|
||||
} else {
|
||||
btn.style.display = 'none';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function vvStorMigrate() {
|
||||
const btn = document.getElementById('vv-stor-migrate-btn');
|
||||
const out = document.getElementById('vv-stor-out');
|
||||
const fb = document.getElementById('vv-stor-fb');
|
||||
const wArr = document.getElementById('vv-stor-warn-noarray');
|
||||
const to = btn?.dataset.to;
|
||||
if (!to) return;
|
||||
|
||||
// Flash guard: array must be started
|
||||
if (to === 'flash' && _vvStorData && !_vvStorData.array_started) {
|
||||
if (wArr) wArr.style.display = '';
|
||||
return;
|
||||
}
|
||||
|
||||
const label = to === 'flash' ? 'Flash (appdata)' : 'Internal (/boot)';
|
||||
if (!confirm(`Migrate Varaverk storage to ${label}?\n\nThis will:\n• Copy all scripts, conf, and git repo to the new location\n• Update varaverk.cfg and master.conf\n• Delete the old location\n\nThe page will need a reload after migration.`)) return;
|
||||
|
||||
btn.disabled = true;
|
||||
btn.textContent = 'Migrating…';
|
||||
if (fb) fb.textContent = '';
|
||||
if (out) { out.textContent = 'Starting migration…\n'; out.style.display = ''; }
|
||||
|
||||
const fd = new FormData();
|
||||
fd.append('action', 'migrate');
|
||||
fd.append('to', to);
|
||||
|
||||
fetch('/plugins/varaverk/api/storage.php', { method: 'POST', body: fd })
|
||||
.then(r => r.json())
|
||||
.then(d => {
|
||||
btn.disabled = false;
|
||||
if (out) {
|
||||
out.textContent = d.output || '(no output)';
|
||||
out.scrollTop = out.scrollHeight;
|
||||
}
|
||||
if (d.ok) {
|
||||
btn.textContent = '✓ Done — reload page';
|
||||
btn.className = 'vv-set-btn primary';
|
||||
btn.onclick = () => location.reload();
|
||||
if (fb) { fb.style.color = '#4caf50'; fb.textContent = 'Migration complete — reload to apply'; }
|
||||
} else {
|
||||
btn.textContent = btn.dataset.to === 'flash' ? 'Migrate to Flash (appdata)' : 'Migrate to Internal (/boot)';
|
||||
if (fb) { fb.style.color = '#ef5350'; fb.textContent = 'Migration failed — see output above'; }
|
||||
}
|
||||
})
|
||||
.catch(e => {
|
||||
btn.disabled = false;
|
||||
btn.textContent = btn.dataset.to === 'flash' ? 'Migrate to Flash (appdata)' : 'Migrate to Internal (/boot)';
|
||||
if (fb) { fb.style.color = '#ef5350'; fb.textContent = 'Request failed: ' + e; }
|
||||
});
|
||||
}
|
||||
|
||||
vvStorLoad();
|
||||
|
||||
// ── Notifications ─────────────────────────────────────────────────────────────
|
||||
function vvNtfToggle(track, key) {
|
||||
const on = !track.classList.contains('on');
|
||||
track.classList.toggle('on', on);
|
||||
const file = (key === 'NOTIFY_UNRAID' || key === 'ENABLE_LOGGING') ? 'master.conf'
|
||||
: '<?= htmlspecialchars($_myHost) ?>.conf';
|
||||
const fd = new FormData();
|
||||
fd.append('id', 'settings');
|
||||
fd.append('changes', JSON.stringify([{ file, key, value: on ? 'true' : 'false', type: 'scalar' }]));
|
||||
fetch('/plugins/varaverk/api/confform.php', { method: 'POST', body: fd })
|
||||
.then(r => r.json())
|
||||
.then(d => { if (!d.ok) track.classList.toggle('on', !on); })
|
||||
.catch(() => track.classList.toggle('on', !on));
|
||||
}
|
||||
|
||||
function vvNtfSaveWebhook() {
|
||||
const inp = document.getElementById('vv-ntf-discord');
|
||||
const fb = document.getElementById('vv-ntf-fb');
|
||||
const btn = document.getElementById('vv-ntf-save');
|
||||
btn.disabled = true; btn.textContent = 'Saving…'; fb.textContent = '';
|
||||
const fd = new FormData();
|
||||
fd.append('id', 'settings');
|
||||
fd.append('changes', JSON.stringify([{
|
||||
file: '<?= htmlspecialchars($_myHost) ?>.conf',
|
||||
key: '<?= htmlspecialchars($_discordKey) ?>',
|
||||
value: inp.value.trim(),
|
||||
type: 'scalar',
|
||||
}]));
|
||||
fetch('/plugins/varaverk/api/confform.php', { method: 'POST', body: fd })
|
||||
.then(r => r.json())
|
||||
.then(d => {
|
||||
btn.disabled = false; btn.textContent = 'Save';
|
||||
fb.style.color = d.ok ? '#4caf50' : '#ef5350';
|
||||
fb.textContent = d.ok ? 'Saved ✓' : (d.error || 'Failed');
|
||||
if (d.ok) setTimeout(() => { fb.textContent = ''; }, 3000);
|
||||
})
|
||||
.catch(() => { btn.disabled = false; btn.textContent = 'Save'; fb.style.color='#ef5350'; fb.textContent='Request failed'; });
|
||||
}
|
||||
|
||||
// ── Unraid API Keys ───────────────────────────────────────────────────────────
|
||||
function vvApiCheck() {
|
||||
const btn = document.getElementById('vv-api-check-btn');
|
||||
const badge = document.getElementById('vv-api-badge');
|
||||
const hosts = document.getElementById('vv-api-hosts');
|
||||
btn.disabled = true; btn.textContent = 'Checking…';
|
||||
fetch('/plugins/varaverk/api/storage.php?action=api_status&_=' + Date.now())
|
||||
.then(r => r.json())
|
||||
.then(d => {
|
||||
btn.disabled = false; btn.textContent = '↻ Check status';
|
||||
if (!d.ok) return;
|
||||
const allOk = d.hosts.every(h => h.api_ok);
|
||||
const anyMissing = d.hosts.some(h => !h.key_present);
|
||||
badge.textContent = allOk ? 'all ok' : anyMissing ? 'keys missing' : 'partial';
|
||||
badge.className = 'vv-set-badge ' + (allOk ? 'internal' : 'flash');
|
||||
|
||||
if (hosts) hosts.innerHTML = d.hosts.map(h => {
|
||||
const tag = h.is_local
|
||||
? `<span style="color:#2a3a2a;font-size:9px;margin-left:4px;">own</span>`
|
||||
: `<span style="color:#2a2a3a;font-size:9px;margin-left:4px;">${d.my_id}'s key on ${h.host_id}</span>`;
|
||||
const keyBit = h.key_present
|
||||
? `<code style="font-size:9px;color:#4a6a4a;">${h.key_preview}</code>`
|
||||
: `<span style="color:#8b2a2a;font-size:10px;">not set — click Renew</span>`;
|
||||
const dot = h.api_ok ? '#4caf50' : (h.key_present ? '#ffb74d' : '#444');
|
||||
return `<div style="display:flex;align-items:center;gap:8px;padding:4px 0;border-bottom:1px solid #111;">
|
||||
<span style="width:6px;height:6px;border-radius:50%;background:${dot};flex-shrink:0;display:inline-block;"></span>
|
||||
<span style="font-size:11px;color:#666;flex-shrink:0;min-width:50px;">${h.host_id}</span>
|
||||
<code style="font-size:9px;color:#2a3a2a;flex-shrink:0;">${h.is_local ? d.key_name : h.key_var}</code>
|
||||
${tag}
|
||||
<span style="flex:1;text-align:right;">${keyBit}</span>
|
||||
</div>`;
|
||||
}).join('');
|
||||
})
|
||||
.catch(() => { btn.disabled = false; btn.textContent = '↻ Check status'; });
|
||||
}
|
||||
|
||||
function vvApiRenew() {
|
||||
const btn = document.getElementById('vv-api-renew-btn');
|
||||
const out = document.getElementById('vv-api-out');
|
||||
const fb = document.getElementById('vv-api-fb');
|
||||
const badge = document.getElementById('vv-api-badge');
|
||||
btn.disabled = true; btn.textContent = 'Renewing…';
|
||||
fb.textContent = ''; out.textContent = ''; out.style.display = '';
|
||||
|
||||
const fd = new FormData();
|
||||
fd.append('action', 'setup_apikeys');
|
||||
fetch('/plugins/varaverk/api/storage.php', { method: 'POST', body: fd })
|
||||
.then(r => r.json())
|
||||
.then(d => {
|
||||
btn.disabled = false; btn.textContent = 'Renew';
|
||||
out.textContent = d.output || '(no output)';
|
||||
out.scrollTop = out.scrollHeight;
|
||||
fb.style.color = d.ok ? '#4caf50' : '#ef5350';
|
||||
fb.textContent = d.ok ? '✓ Done' : 'Failed — see output';
|
||||
if (d.ok) { badge.textContent = ''; vvApiCheck(); }
|
||||
})
|
||||
.catch(() => {
|
||||
btn.disabled = false; btn.textContent = 'Renew';
|
||||
fb.style.color = '#ef5350'; fb.textContent = 'Request failed';
|
||||
});
|
||||
}
|
||||
|
||||
// Auto-check on load
|
||||
vvApiCheck();
|
||||
</script>
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,350 @@
|
||||
<?php
|
||||
// Partnership page data helpers
|
||||
|
||||
require_once __DIR__ . '/config.php';
|
||||
require_once __DIR__ . '/arrs.php'; // vv_arr_known_hosts(), vv_arr_scalar()
|
||||
require_once __DIR__ . '/common.php'; // vv_system_info(), vv_docker_containers(), vv_remote_hosts_stats(), vv_api_node_metrics()
|
||||
|
||||
// ── Config ────────────────────────────────────────────────────────────────────
|
||||
|
||||
function vv_pt_config(): array {
|
||||
$v = vv_conf_vars();
|
||||
$offlineDays = null;
|
||||
$odFile = '/boot/config/partnership_offline_days.db';
|
||||
if (file_exists($odFile)) {
|
||||
$raw = trim(@file_get_contents($odFile) ?: '');
|
||||
if (is_numeric($raw)) $offlineDays = (int)$raw;
|
||||
}
|
||||
return [
|
||||
'enabled' => ($v['PARTNERSHIP_ENABLED'] ?? 'false') === 'true',
|
||||
'owner_host' => $v['PARTNERSHIP_OWNER_HOST'] ?? '',
|
||||
'sync_min' => (int)($v['PARTNERSHIP_SYNC_INTERVAL'] ?? 15),
|
||||
'grace_hours' => (int)($v['PARTNERSHIP_GRACE_HOURS'] ?? 6),
|
||||
'offline_threshold' => (int)($v['PARTNERSHIP_OFFLINE_THRESHOLD'] ?? 30),
|
||||
'offline_days' => $offlineDays,
|
||||
'remove_tailscale' => ($v['PARTNERSHIP_REMOVE_TAILSCALE'] ?? 'true') === 'true',
|
||||
'folderview3' => ($v['PARTNERSHIP_FOLDERVIEW3'] ?? 'false') === 'true',
|
||||
'tailscale_configured' => !empty($v['TAILSCALE_API_KEY']) && !empty($v['TAILSCALE_TAILNET']),
|
||||
'transfer_confirm' => $v['PARTNERSHIP_TRANSFER_CONFIRM'] ?? 'i-understand-this-transfers-ownership',
|
||||
];
|
||||
}
|
||||
|
||||
// ── Mirror sync health — the partnership's actual job (rsync orchestrators) ──────
|
||||
|
||||
function vv_pt_sync_summary(string $logFile): string {
|
||||
if (!file_exists($logFile)) return '';
|
||||
$tail = array_filter(array_slice(file($logFile, FILE_IGNORE_NEW_LINES), -30), 'strlen');
|
||||
foreach (array_reverse(array_values($tail)) as $raw) {
|
||||
// Strip emoji / Unicode decoration for regex matching
|
||||
$line = trim(preg_replace('/[\x{1F000}-\x{1FFFF}\x{2600}-\x{27BF}\x{FE0F}]/u', '', $raw));
|
||||
$line = preg_replace('/\s+/', ' ', $line);
|
||||
// "Critical sync complete — HOST1 — 1m39s — 2 share(s)"
|
||||
if (preg_match('/Critical sync complete\s*—\s*\S+\s*—\s*([\w]+)\s*—\s*(.+)/i', $line, $m))
|
||||
return trim($m[2]) . ' in ' . $m[1];
|
||||
// "Status: all complete — 0 share(s) synced, 11 job(s) run"
|
||||
if (preg_match('/Status:\s*all complete\s*—\s*(.+)/i', $line, $m))
|
||||
return trim($m[1]);
|
||||
// "Status: X failure(s)" or "Failures: N"
|
||||
if (preg_match('/Failures:\s*(\d+)/i', $line, $m) && (int)$m[1] > 0)
|
||||
return (int)$m[1] . ' failure(s)';
|
||||
}
|
||||
return '';
|
||||
}
|
||||
|
||||
function vv_pt_sync(): array {
|
||||
$jobs = [
|
||||
'critical' => 'Orchestrators/critical_sync_maintenance',
|
||||
'daily' => 'Orchestrators/daily_sync_maintenance',
|
||||
'weekly' => 'Orchestrators/weekly_sync_maintenance',
|
||||
];
|
||||
$out = ['jobs' => []];
|
||||
foreach ($jobs as $key => $base) {
|
||||
$statFile = LOG_DIR . '/' . $base . '.json';
|
||||
$logFile = LOG_DIR . '/' . $base . '.log';
|
||||
$s = file_exists($statFile) ? json_decode(@file_get_contents($statFile), true) : null;
|
||||
$out['jobs'][$key] = is_array($s) ? [
|
||||
'status' => $s['status'] ?? 'unknown',
|
||||
'start' => isset($s['start']) ? (int)$s['start'] : null,
|
||||
'end' => isset($s['end']) ? (int)$s['end'] : null,
|
||||
'summary' => vv_pt_sync_summary($logFile),
|
||||
] : null;
|
||||
}
|
||||
$v = vv_conf_vars();
|
||||
$out['interval_min'] = (int)($v['PARTNERSHIP_SYNC_INTERVAL'] ?? 30);
|
||||
// Rsync gate flags (master.conf) — global Tier 1 + per-tier Tier 2.
|
||||
$out['gates'] = [
|
||||
'global' => ['var' => 'RSYNC_ENABLED', 'on' => ($v['RSYNC_ENABLED'] ?? 'true') === 'true'],
|
||||
'critical' => ['var' => 'CRITICAL_RSYNC_ENABLED', 'on' => ($v['CRITICAL_RSYNC_ENABLED'] ?? 'true') === 'true'],
|
||||
'daily' => ['var' => 'DAILY_RSYNC_ENABLED', 'on' => ($v['DAILY_RSYNC_ENABLED'] ?? 'true') === 'true'],
|
||||
'weekly' => ['var' => 'WEEKLY_RSYNC_ENABLED', 'on' => ($v['WEEKLY_RSYNC_ENABLED'] ?? 'true') === 'true'],
|
||||
];
|
||||
// Back-compat keys still used by the warning line.
|
||||
$out['rsync_enabled'] = $out['gates']['global']['on'];
|
||||
$out['critical_enabled'] = $out['gates']['critical']['on'];
|
||||
return $out;
|
||||
}
|
||||
|
||||
// ── State file parser ─────────────────────────────────────────────────────────
|
||||
|
||||
function vv_pt_read_db(string $path): array {
|
||||
if (!file_exists($path)) return [];
|
||||
$out = [];
|
||||
foreach (file($path, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES) as $line) {
|
||||
[$k, $v] = array_pad(explode('=', trim($line), 2), 2, '');
|
||||
if ($k) $out[trim($k)] = trim($v, '"\'');
|
||||
}
|
||||
return $out;
|
||||
}
|
||||
|
||||
// ── Tailscale peers ───────────────────────────────────────────────────────────
|
||||
|
||||
function vv_pt_ts_peers(): array {
|
||||
$raw = shell_exec('tailscale status --json 2>/dev/null') ?: '{}';
|
||||
$data = json_decode($raw, true) ?: [];
|
||||
$peers = [];
|
||||
|
||||
// Self
|
||||
$self = $data['Self'] ?? [];
|
||||
$selfLabel = strtolower(explode('.', $self['DNSName'] ?? '')[0]);
|
||||
if ($selfLabel) {
|
||||
$peers[$selfLabel] = [
|
||||
'online' => true,
|
||||
'active' => true,
|
||||
'ip' => $self['TailscaleIPs'][0] ?? null,
|
||||
];
|
||||
}
|
||||
|
||||
// Peers
|
||||
foreach ($data['Peer'] ?? [] as $peer) {
|
||||
$label = strtolower(explode('.', $peer['DNSName'] ?? '')[0]);
|
||||
if (!$label) continue;
|
||||
$peers[$label] = [
|
||||
'online' => (bool)($peer['Online'] ?? false),
|
||||
'active' => (bool)($peer['Active'] ?? false),
|
||||
'ip' => $peer['TailscaleIPs'][0] ?? null,
|
||||
];
|
||||
}
|
||||
return $peers;
|
||||
}
|
||||
|
||||
// ── SSH helper — run a single command on a remote host ────────────────────────
|
||||
|
||||
function vv_pt_ssh(string $ip, string $sshKey, string $cmd, int $timeout = 4): string {
|
||||
if (!$ip || !$sshKey || !file_exists($sshKey)) return '';
|
||||
$full = sprintf(
|
||||
'ssh -i %s -o ConnectTimeout=%d -o StrictHostKeyChecking=no -o BatchMode=yes root@%s %s 2>/dev/null',
|
||||
escapeshellarg($sshKey), $timeout, escapeshellarg($ip), escapeshellarg($cmd)
|
||||
);
|
||||
return shell_exec($full) ?: '';
|
||||
}
|
||||
|
||||
// ── System info ───────────────────────────────────────────────────────────────
|
||||
|
||||
// vv_system_info() (common.php) provides version, load_avg, array_state.
|
||||
// /proc/uptime is the reliable uptime source (API uptime is an ISO date string, not seconds).
|
||||
// vv_docker_containers() (common.php) provides the running container list.
|
||||
function vv_pt_local_system(): array {
|
||||
$info = vv_system_info();
|
||||
$uptimeSec = file_exists('/proc/uptime')
|
||||
? (int)explode(' ', file_get_contents('/proc/uptime'))[0] : 0;
|
||||
return [
|
||||
'unraid_version' => $info['version'] ?? '',
|
||||
'uptime_sec' => $uptimeSec,
|
||||
'load_avg' => isset($info['load_avg']) ? $info['load_avg'][0] : null,
|
||||
'containers' => count(vv_docker_containers()),
|
||||
];
|
||||
}
|
||||
|
||||
// SSH fallback for remote nodes — version/uptime/load/containers in one call.
|
||||
// API stats from vv_remote_hosts_stats() take priority when available; SSH fills gaps.
|
||||
function vv_pt_remote_system(string $ip, string $sshKey): array {
|
||||
$out = vv_pt_ssh($ip, $sshKey,
|
||||
'printf "%s\nUPTIME:%s\nLOAD:%s\nCONTAINERS:%s\n" ' .
|
||||
'"$(cat /etc/unraid-version 2>/dev/null)" ' .
|
||||
'"$(cat /proc/uptime 2>/dev/null)" ' .
|
||||
'"$(awk \'{print $1}\' /proc/loadavg 2>/dev/null)" ' .
|
||||
'"$(docker ps -q 2>/dev/null | wc -l)"');
|
||||
$ver = '';
|
||||
preg_match('/VERSION="([^"]+)"/', $out, $m); if ($m) $ver = $m[1];
|
||||
$uptime = 0;
|
||||
if (preg_match('/UPTIME:([\d.]+)/', $out, $m)) $uptime = (int)$m[1];
|
||||
$load = null;
|
||||
if (preg_match('/LOAD:([\d.]+)/', $out, $m)) $load = round((float)$m[1], 2);
|
||||
$containers = null;
|
||||
if (preg_match('/CONTAINERS:(\d+)/', $out, $m)) $containers = (int)$m[1];
|
||||
return [
|
||||
'unraid_version' => $ver,
|
||||
'uptime_sec' => $uptime,
|
||||
'load_avg' => $load,
|
||||
'containers' => $containers,
|
||||
];
|
||||
}
|
||||
|
||||
// ── Per-node data ─────────────────────────────────────────────────────────────
|
||||
|
||||
function vv_pt_nodes(): array {
|
||||
$currentHost = vv_detect_host();
|
||||
$hosts = vv_arr_known_hosts(); // ['host1' => 'hostname', ...]
|
||||
$vars = vv_conf_vars();
|
||||
$tsPeers = vv_pt_ts_peers();
|
||||
$ownerSlot = strtolower($vars['PARTNERSHIP_OWNER_HOST'] ?? '');
|
||||
$setupDb = vv_setup_state_read();
|
||||
|
||||
// Remote host stats (API + 30s /tmp cache) — includes version, uptime, cpu/ram/array/temp/vms
|
||||
$remoteStats = vv_remote_hosts_stats();
|
||||
|
||||
// SSH key for this host
|
||||
$myId = strtoupper($currentHost);
|
||||
$myRaw = vv_read_conf_raw($currentHost . '.conf');
|
||||
$mySshKey = vv_arr_scalar($myRaw, $myId . '_SSH_KEY');
|
||||
|
||||
$nodes = [];
|
||||
foreach ($hosts as $slot => $hostname) {
|
||||
$isMe = ($slot === $currentHost || $currentHost === 'unknown');
|
||||
$isOwner = (strtolower($ownerSlot) === $slot);
|
||||
|
||||
// Tailscale
|
||||
$tsLabel = strtolower($hostname);
|
||||
$ts = $tsPeers[$tsLabel] ?? ['online' => null, 'active' => false, 'ip' => null];
|
||||
|
||||
// Fallback state
|
||||
$fbState = 'UNKNOWN';
|
||||
$fbPath = '/boot/config/fallback_state.db';
|
||||
if ($isMe) {
|
||||
$fb = vv_pt_read_db($fbPath);
|
||||
$fbState = $fb['state'] ?? 'UNKNOWN';
|
||||
} elseif ($ts['online'] && $ts['ip'] && $mySshKey) {
|
||||
$out = vv_pt_ssh($ts['ip'], $mySshKey, 'cat /boot/config/fallback_state.db 2>/dev/null');
|
||||
if ($out) {
|
||||
$fb = [];
|
||||
foreach (explode("\n", $out) as $line) {
|
||||
[$k, $v] = array_pad(explode('=', trim($line), 2), 2, '');
|
||||
if ($k) $fb[trim($k)] = trim($v, '"\'');
|
||||
}
|
||||
$fbState = $fb['state'] ?? 'UNKNOWN';
|
||||
}
|
||||
}
|
||||
|
||||
// Partnership DB — local only (each server writes its own)
|
||||
$dbPath = "/boot/config/partnership_{$hostname}.db";
|
||||
$ptDb = vv_pt_read_db($dbPath);
|
||||
|
||||
// System info
|
||||
$system = $isMe
|
||||
? vv_pt_local_system()
|
||||
: ($ts['online'] && $ts['ip'] && $mySshKey ? vv_pt_remote_system($ts['ip'], $mySshKey) : []);
|
||||
|
||||
// Onboard phase from setup.db — null=self, 0=not started, 1=SSH+conf done, 2=fully onboarded
|
||||
$nodeIdUpper = strtoupper($slot);
|
||||
$onboardPhase = $isMe ? null
|
||||
: (($setupDb[$nodeIdUpper . '_PHASE2_DONE'] ?? '') === 'true' ? 2
|
||||
: (($setupDb[$nodeIdUpper . '_PHASE1_DONE'] ?? '') === 'true' ? 1 : 0));
|
||||
// key_ready: local key generated but not yet installed on HOST2 (SSH pending manual step)
|
||||
$keyReady = !$isMe && ($setupDb[$nodeIdUpper . '_KEY_READY'] ?? '') === 'true';
|
||||
// For self: local setup complete flag (set by partnership_manager --onboard --local-only)
|
||||
$localDone = $isMe && ($setupDb[$nodeIdUpper . '_LOCAL_DONE'] ?? '') === 'true';
|
||||
|
||||
// Unraid API key status — checks Unraid's key store directly so deletions are reflected.
|
||||
$apiKeySet = false;
|
||||
$apiKeyPreview = '';
|
||||
if ($isMe) {
|
||||
$apiOut = shell_exec('/usr/local/sbin/unraid-api apikey --name "Varaverk" --json </dev/null 2>/dev/null');
|
||||
$apiData = json_decode(trim($apiOut ?? ''), true);
|
||||
if (is_array($apiData) && !empty($apiData['key'])) {
|
||||
$apiKeySet = true;
|
||||
$apiKeyPreview = substr($apiData['key'], 0, 8) . '...' . substr($apiData['key'], -4);
|
||||
}
|
||||
}
|
||||
|
||||
// Live metrics: local uses vv_api_data() (cached); remote uses vv_remote_hosts_stats() (30s cache)
|
||||
if ($isMe) {
|
||||
$metrics = array_merge(
|
||||
vv_api_node_metrics(vv_api_data()),
|
||||
array_filter([
|
||||
'load_avg' => $system['load_avg'] ?? null,
|
||||
'containers' => $system['containers'] ?? null,
|
||||
], fn($v) => $v !== null)
|
||||
);
|
||||
} else {
|
||||
$rStat = $remoteStats[$nodeIdUpper] ?? [];
|
||||
// Merge API metrics from remote stats with SSH extras (load, containers)
|
||||
$metrics = array_filter([
|
||||
'cpu_pct' => $rStat['cpu_pct'] ?? null,
|
||||
'ram_used_gb' => $rStat['ram_used_gb'] ?? null,
|
||||
'ram_total_gb' => $rStat['ram_total_gb'] ?? null,
|
||||
'array_used_tb' => $rStat['array_used_tb'] ?? null,
|
||||
'array_total_tb' => $rStat['array_total_tb'] ?? null,
|
||||
'max_disk_temp' => $rStat['max_disk_temp'] ?? null,
|
||||
'vm_count' => $rStat['vm_count'] ?? null,
|
||||
'load_avg' => $system['load_avg'] ?? null,
|
||||
'containers' => $system['containers'] ?? null,
|
||||
], fn($v) => $v !== null);
|
||||
// Fill version/uptime from API stats if SSH didn't provide them
|
||||
if (empty($system['unraid_version']) && !empty($rStat['version'])) {
|
||||
$system['unraid_version'] = $rStat['version'];
|
||||
}
|
||||
if (empty($system['uptime_sec']) && !empty($rStat['uptime_sec'])) {
|
||||
$system['uptime_sec'] = $rStat['uptime_sec'];
|
||||
}
|
||||
}
|
||||
|
||||
$nodes[] = [
|
||||
'slot' => $slot,
|
||||
'id' => $nodeIdUpper,
|
||||
'hostname' => $hostname,
|
||||
'is_me' => $isMe,
|
||||
'is_owner' => $isOwner,
|
||||
'ts_online' => $ts['online'],
|
||||
'ts_active' => $ts['active'],
|
||||
'ts_ip' => $ts['ip'],
|
||||
'fallback' => $fbState,
|
||||
'partnership' => $ptDb,
|
||||
'system' => $system,
|
||||
'onboard_phase' => $onboardPhase,
|
||||
'key_ready' => $keyReady,
|
||||
'local_done' => $localDone,
|
||||
'api_key_set' => $apiKeySet,
|
||||
'api_key_preview' => $apiKeyPreview,
|
||||
'metrics' => $metrics,
|
||||
];
|
||||
}
|
||||
return $nodes;
|
||||
}
|
||||
|
||||
// ── Connectivity test — SSH echo with round-trip timing ─────────────────────────
|
||||
function vv_pt_ping(string $slot): array {
|
||||
$slot = strtolower($slot);
|
||||
$vars = vv_conf_vars();
|
||||
$hostname = $vars[strtoupper($slot)] ?? '';
|
||||
if (!$hostname) return ['ok' => false, 'error' => 'Unknown host slot'];
|
||||
|
||||
$currentHost = vv_detect_host();
|
||||
$myRaw = vv_read_conf_raw($currentHost . '.conf');
|
||||
$sshKey = vv_arr_scalar($myRaw, strtoupper($currentHost) . '_SSH_KEY');
|
||||
if (!$sshKey || !file_exists($sshKey)) {
|
||||
return ['ok' => false, 'error' => 'No SSH key configured on this host'];
|
||||
}
|
||||
|
||||
$ip = vv_resolve_tailscale_ip($hostname);
|
||||
if (!$ip) return ['ok' => false, 'error' => "Cannot resolve Tailscale IP for $hostname"];
|
||||
|
||||
$t0 = microtime(true);
|
||||
$out = vv_pt_ssh($ip, $sshKey, 'echo ok', 8);
|
||||
$ms = (int)round((microtime(true) - $t0) * 1000);
|
||||
|
||||
if (trim($out) === 'ok') {
|
||||
return ['ok' => true, 'latency_ms' => $ms, 'host' => $hostname, 'ip' => $ip];
|
||||
}
|
||||
return ['ok' => false, 'error' => "SSH to $hostname ($ip) failed or timed out", 'host' => $hostname];
|
||||
}
|
||||
|
||||
// ── Entry point ───────────────────────────────────────────────────────────────
|
||||
|
||||
function vv_partnership_all(): array {
|
||||
return [
|
||||
'config' => vv_pt_config(),
|
||||
'nodes' => vv_pt_nodes(),
|
||||
'sync' => vv_pt_sync(),
|
||||
'ts' => time(),
|
||||
];
|
||||
}
|
||||
@@ -0,0 +1,352 @@
|
||||
<?php
|
||||
// Partnership page data helpers
|
||||
|
||||
require_once __DIR__ . '/config.php';
|
||||
require_once __DIR__ . '/arrs.php'; // vv_arr_known_hosts(), vv_arr_scalar()
|
||||
require_once __DIR__ . '/common.php'; // vv_system_info(), vv_docker_containers(), vv_remote_hosts_stats(), vv_api_node_metrics()
|
||||
|
||||
// ── Config ────────────────────────────────────────────────────────────────────
|
||||
|
||||
function vv_pt_config(): array {
|
||||
$v = vv_conf_vars();
|
||||
$offlineDays = null;
|
||||
$odFile = '/boot/config/partnership_offline_days.db';
|
||||
if (file_exists($odFile)) {
|
||||
$raw = trim(@file_get_contents($odFile) ?: '');
|
||||
if (is_numeric($raw)) $offlineDays = (int)$raw;
|
||||
}
|
||||
return [
|
||||
'enabled' => ($v['PARTNERSHIP_ENABLED'] ?? 'false') === 'true',
|
||||
'owner_host' => $v['PARTNERSHIP_OWNER_HOST'] ?? '',
|
||||
'sync_min' => (int)($v['PARTNERSHIP_SYNC_INTERVAL'] ?? 15),
|
||||
'grace_hours' => (int)($v['PARTNERSHIP_GRACE_HOURS'] ?? 6),
|
||||
'offline_threshold' => (int)($v['PARTNERSHIP_OFFLINE_THRESHOLD'] ?? 30),
|
||||
'offline_days' => $offlineDays,
|
||||
'remove_tailscale' => ($v['PARTNERSHIP_REMOVE_TAILSCALE'] ?? 'true') === 'true',
|
||||
'folderview3' => ($v['PARTNERSHIP_FOLDERVIEW3'] ?? 'false') === 'true',
|
||||
'tailscale_configured' => !empty($v['TAILSCALE_API_KEY']) && !empty($v['TAILSCALE_TAILNET']),
|
||||
'transfer_confirm' => $v['PARTNERSHIP_TRANSFER_CONFIRM'] ?? 'i-understand-this-transfers-ownership',
|
||||
];
|
||||
}
|
||||
|
||||
// ── Mirror sync health — the partnership's actual job (rsync orchestrators) ──────
|
||||
|
||||
function vv_pt_sync_summary(string $logFile): string {
|
||||
if (!file_exists($logFile)) return '';
|
||||
$tail = array_filter(array_slice(file($logFile, FILE_IGNORE_NEW_LINES), -30), 'strlen');
|
||||
foreach (array_reverse(array_values($tail)) as $raw) {
|
||||
// Strip emoji / Unicode decoration for regex matching
|
||||
$line = trim(preg_replace('/[\x{1F000}-\x{1FFFF}\x{2600}-\x{27BF}\x{FE0F}]/u', '', $raw));
|
||||
$line = preg_replace('/\s+/', ' ', $line);
|
||||
// "Critical sync complete — HOST1 — 1m39s — 2 share(s)"
|
||||
if (preg_match('/Critical sync complete\s*—\s*\S+\s*—\s*([\w]+)\s*—\s*(.+)/i', $line, $m))
|
||||
return trim($m[2]) . ' in ' . $m[1];
|
||||
// "Status: all complete — 0 share(s) synced, 11 job(s) run"
|
||||
if (preg_match('/Status:\s*all complete\s*—\s*(.+)/i', $line, $m))
|
||||
return trim($m[1]);
|
||||
// "Status: X failure(s)" or "Failures: N"
|
||||
if (preg_match('/Failures:\s*(\d+)/i', $line, $m) && (int)$m[1] > 0)
|
||||
return (int)$m[1] . ' failure(s)';
|
||||
}
|
||||
return '';
|
||||
}
|
||||
|
||||
function vv_pt_sync(): array {
|
||||
$jobs = [
|
||||
'critical' => 'Orchestrators/critical_sync_maintenance',
|
||||
'daily' => 'Orchestrators/daily_sync_maintenance',
|
||||
'weekly' => 'Orchestrators/weekly_sync_maintenance',
|
||||
];
|
||||
$out = ['jobs' => []];
|
||||
foreach ($jobs as $key => $base) {
|
||||
$statFile = LOG_DIR . '/' . $base . '.json';
|
||||
$logFile = LOG_DIR . '/' . $base . '.log';
|
||||
$s = file_exists($statFile) ? json_decode(@file_get_contents($statFile), true) : null;
|
||||
$out['jobs'][$key] = is_array($s) ? [
|
||||
'status' => $s['status'] ?? 'unknown',
|
||||
'start' => isset($s['start']) ? (int)$s['start'] : null,
|
||||
'end' => isset($s['end']) ? (int)$s['end'] : null,
|
||||
'summary' => vv_pt_sync_summary($logFile),
|
||||
] : null;
|
||||
}
|
||||
$v = vv_conf_vars();
|
||||
$out['interval_min'] = (int)($v['PARTNERSHIP_SYNC_INTERVAL'] ?? 30);
|
||||
// Rsync gate flags (master.conf) — global Tier 1 + per-tier Tier 2.
|
||||
$out['gates'] = [
|
||||
'global' => ['var' => 'RSYNC_ENABLED', 'on' => ($v['RSYNC_ENABLED'] ?? 'true') === 'true'],
|
||||
'critical' => ['var' => 'CRITICAL_RSYNC_ENABLED', 'on' => ($v['CRITICAL_RSYNC_ENABLED'] ?? 'true') === 'true'],
|
||||
'daily' => ['var' => 'DAILY_RSYNC_ENABLED', 'on' => ($v['DAILY_RSYNC_ENABLED'] ?? 'true') === 'true'],
|
||||
'weekly' => ['var' => 'WEEKLY_RSYNC_ENABLED', 'on' => ($v['WEEKLY_RSYNC_ENABLED'] ?? 'true') === 'true'],
|
||||
];
|
||||
// Back-compat keys still used by the warning line.
|
||||
$out['rsync_enabled'] = $out['gates']['global']['on'];
|
||||
$out['critical_enabled'] = $out['gates']['critical']['on'];
|
||||
return $out;
|
||||
}
|
||||
|
||||
// ── State file parser ─────────────────────────────────────────────────────────
|
||||
|
||||
function vv_pt_read_db(string $path): array {
|
||||
if (!file_exists($path)) return [];
|
||||
$out = [];
|
||||
foreach (file($path, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES) as $line) {
|
||||
[$k, $v] = array_pad(explode('=', trim($line), 2), 2, '');
|
||||
if ($k) $out[trim($k)] = trim($v, '"\'');
|
||||
}
|
||||
return $out;
|
||||
}
|
||||
|
||||
// ── Tailscale peers ───────────────────────────────────────────────────────────
|
||||
|
||||
function vv_pt_ts_peers(): array {
|
||||
$raw = shell_exec('tailscale status --json 2>/dev/null') ?: '{}';
|
||||
$data = json_decode($raw, true) ?: [];
|
||||
$peers = [];
|
||||
|
||||
// Self
|
||||
$self = $data['Self'] ?? [];
|
||||
$selfLabel = strtolower(explode('.', $self['DNSName'] ?? '')[0]);
|
||||
if ($selfLabel) {
|
||||
$peers[$selfLabel] = [
|
||||
'online' => true,
|
||||
'active' => true,
|
||||
'ip' => $self['TailscaleIPs'][0] ?? null,
|
||||
];
|
||||
}
|
||||
|
||||
// Peers
|
||||
foreach ($data['Peer'] ?? [] as $peer) {
|
||||
$label = strtolower(explode('.', $peer['DNSName'] ?? '')[0]);
|
||||
if (!$label) continue;
|
||||
$peers[$label] = [
|
||||
'online' => (bool)($peer['Online'] ?? false),
|
||||
'active' => (bool)($peer['Active'] ?? false),
|
||||
'ip' => $peer['TailscaleIPs'][0] ?? null,
|
||||
];
|
||||
}
|
||||
return $peers;
|
||||
}
|
||||
|
||||
// ── SSH helper — run a single command on a remote host ────────────────────────
|
||||
|
||||
function vv_pt_ssh(string $ip, string $sshKey, string $cmd, int $timeout = 4): string {
|
||||
if (!$ip || !$sshKey || !file_exists($sshKey)) return '';
|
||||
$full = sprintf(
|
||||
'ssh -i %s -o ConnectTimeout=%d -o StrictHostKeyChecking=no -o BatchMode=yes root@%s %s 2>/dev/null',
|
||||
escapeshellarg($sshKey), $timeout, escapeshellarg($ip), escapeshellarg($cmd)
|
||||
);
|
||||
return shell_exec($full) ?: '';
|
||||
}
|
||||
|
||||
// ── System info ───────────────────────────────────────────────────────────────
|
||||
|
||||
// vv_system_info() (common.php) provides version, load_avg, array_state.
|
||||
// /proc/uptime is the reliable uptime source (API uptime is an ISO date string, not seconds).
|
||||
// vv_docker_containers() (common.php) provides the running container list.
|
||||
function vv_pt_local_system(): array {
|
||||
$info = vv_system_info();
|
||||
$uptimeSec = file_exists('/proc/uptime')
|
||||
? (int)explode(' ', file_get_contents('/proc/uptime'))[0] : 0;
|
||||
return [
|
||||
'unraid_version' => $info['version'] ?? '',
|
||||
'uptime_sec' => $uptimeSec,
|
||||
'load_avg' => isset($info['load_avg']) ? $info['load_avg'][0] : null,
|
||||
'containers' => count(vv_docker_containers()),
|
||||
];
|
||||
}
|
||||
|
||||
// SSH fallback for remote nodes — version/uptime/load/containers in one call.
|
||||
// API stats from vv_remote_hosts_stats() take priority when available; SSH fills gaps.
|
||||
function vv_pt_remote_system(string $ip, string $sshKey): array {
|
||||
$out = vv_pt_ssh($ip, $sshKey,
|
||||
'printf "%s\nUPTIME:%s\nLOAD:%s\nCONTAINERS:%s\n" ' .
|
||||
'"$(cat /etc/unraid-version 2>/dev/null)" ' .
|
||||
'"$(cat /proc/uptime 2>/dev/null)" ' .
|
||||
'"$(awk \'{print $1}\' /proc/loadavg 2>/dev/null)" ' .
|
||||
'"$(docker ps -q 2>/dev/null | wc -l)"');
|
||||
$ver = '';
|
||||
preg_match('/VERSION="([^"]+)"/', $out, $m); if ($m) $ver = $m[1];
|
||||
$uptime = 0;
|
||||
if (preg_match('/UPTIME:([\d.]+)/', $out, $m)) $uptime = (int)$m[1];
|
||||
$load = null;
|
||||
if (preg_match('/LOAD:([\d.]+)/', $out, $m)) $load = round((float)$m[1], 2);
|
||||
$containers = null;
|
||||
if (preg_match('/CONTAINERS:(\d+)/', $out, $m)) $containers = (int)$m[1];
|
||||
return [
|
||||
'unraid_version' => $ver,
|
||||
'uptime_sec' => $uptime,
|
||||
'load_avg' => $load,
|
||||
'containers' => $containers,
|
||||
];
|
||||
}
|
||||
|
||||
// ── Per-node data ─────────────────────────────────────────────────────────────
|
||||
|
||||
function vv_pt_nodes(): array {
|
||||
$currentHost = vv_detect_host();
|
||||
$hosts = vv_arr_known_hosts(); // ['host1' => 'hostname', ...]
|
||||
$vars = vv_conf_vars();
|
||||
$tsPeers = vv_pt_ts_peers();
|
||||
$ownerSlot = strtolower($vars['PARTNERSHIP_OWNER_HOST'] ?? '');
|
||||
$setupDb = vv_setup_state_read();
|
||||
|
||||
// Remote host stats (API + 30s /tmp cache) — includes version, uptime, cpu/ram/array/temp/vms
|
||||
$remoteStats = vv_remote_hosts_stats();
|
||||
|
||||
// SSH key for this host
|
||||
$myId = strtoupper($currentHost);
|
||||
$myRaw = vv_read_conf_raw($currentHost . '.conf');
|
||||
$mySshKey = vv_arr_scalar($myRaw, $myId . '_SSH_KEY');
|
||||
|
||||
$nodes = [];
|
||||
foreach ($hosts as $slot => $hostname) {
|
||||
$isMe = ($slot === $currentHost || $currentHost === 'unknown');
|
||||
$isOwner = (strtolower($ownerSlot) === $slot);
|
||||
|
||||
// Tailscale
|
||||
$tsLabel = strtolower($hostname);
|
||||
$ts = $tsPeers[$tsLabel] ?? ['online' => null, 'active' => false, 'ip' => null];
|
||||
|
||||
// Fallback state
|
||||
$fbState = 'UNKNOWN';
|
||||
$fbPath = '/boot/config/fallback_state.db';
|
||||
if ($isMe) {
|
||||
$fb = vv_pt_read_db($fbPath);
|
||||
$fbState = $fb['state'] ?? 'UNKNOWN';
|
||||
} elseif ($ts['online'] && $ts['ip'] && $mySshKey) {
|
||||
$out = vv_pt_ssh($ts['ip'], $mySshKey, 'cat /boot/config/fallback_state.db 2>/dev/null');
|
||||
if ($out) {
|
||||
$fb = [];
|
||||
foreach (explode("\n", $out) as $line) {
|
||||
[$k, $v] = array_pad(explode('=', trim($line), 2), 2, '');
|
||||
if ($k) $fb[trim($k)] = trim($v, '"\'');
|
||||
}
|
||||
$fbState = $fb['state'] ?? 'UNKNOWN';
|
||||
}
|
||||
}
|
||||
|
||||
// Partnership DB — local only (each server writes its own)
|
||||
$dbPath = "/boot/config/partnership_{$hostname}.db";
|
||||
$ptDb = vv_pt_read_db($dbPath);
|
||||
|
||||
// System info
|
||||
$system = $isMe
|
||||
? vv_pt_local_system()
|
||||
: ($ts['online'] && $ts['ip'] && $mySshKey ? vv_pt_remote_system($ts['ip'], $mySshKey) : []);
|
||||
|
||||
// Onboard phase from setup.db — null=self, 0=not started, 1=SSH+conf done, 2=fully onboarded
|
||||
$nodeIdUpper = strtoupper($slot);
|
||||
$onboardPhase = $isMe ? null
|
||||
: (($setupDb[$nodeIdUpper . '_PHASE2_DONE'] ?? '') === 'true' ? 2
|
||||
: (($setupDb[$nodeIdUpper . '_PHASE1_DONE'] ?? '') === 'true' ? 1 : 0));
|
||||
// key_ready: local key generated but not yet installed on HOST2 (SSH pending manual step)
|
||||
$keyReady = !$isMe && ($setupDb[$nodeIdUpper . '_KEY_READY'] ?? '') === 'true';
|
||||
// For self: local setup complete flag (set by partnership_manager --onboard --local-only)
|
||||
$localDone = $isMe && ($setupDb[$nodeIdUpper . '_LOCAL_DONE'] ?? '') === 'true';
|
||||
|
||||
// Unraid API key status — checks Unraid's key store directly so deletions are reflected.
|
||||
$apiKeySet = false;
|
||||
$apiKeyPreview = '';
|
||||
if ($isMe) {
|
||||
$hn = trim((string)shell_exec("hostname -s 2>/dev/null | sed 's/^[Uu][Nn][Rr][Aa][Ii][Dd]-//'")) ?: 'Varaverk';
|
||||
$keyName = 'Varaverk ' . $hn;
|
||||
$apiOut = shell_exec('/usr/local/sbin/unraid-api apikey --name ' . escapeshellarg($keyName) . ' --json </dev/null 2>/dev/null');
|
||||
$apiData = json_decode(trim($apiOut ?? ''), true);
|
||||
if (is_array($apiData) && !empty($apiData['key'])) {
|
||||
$apiKeySet = true;
|
||||
$apiKeyPreview = substr($apiData['key'], 0, 8) . '...' . substr($apiData['key'], -4);
|
||||
}
|
||||
}
|
||||
|
||||
// Live metrics: local uses vv_api_data() (cached); remote uses vv_remote_hosts_stats() (30s cache)
|
||||
if ($isMe) {
|
||||
$metrics = array_merge(
|
||||
vv_api_node_metrics(vv_api_data()),
|
||||
array_filter([
|
||||
'load_avg' => $system['load_avg'] ?? null,
|
||||
'containers' => $system['containers'] ?? null,
|
||||
], fn($v) => $v !== null)
|
||||
);
|
||||
} else {
|
||||
$rStat = $remoteStats[$nodeIdUpper] ?? [];
|
||||
// Merge API metrics from remote stats with SSH extras (load, containers)
|
||||
$metrics = array_filter([
|
||||
'cpu_pct' => $rStat['cpu_pct'] ?? null,
|
||||
'ram_used_gb' => $rStat['ram_used_gb'] ?? null,
|
||||
'ram_total_gb' => $rStat['ram_total_gb'] ?? null,
|
||||
'array_used_tb' => $rStat['array_used_tb'] ?? null,
|
||||
'array_total_tb' => $rStat['array_total_tb'] ?? null,
|
||||
'max_disk_temp' => $rStat['max_disk_temp'] ?? null,
|
||||
'vm_count' => $rStat['vm_count'] ?? null,
|
||||
'load_avg' => $system['load_avg'] ?? null,
|
||||
'containers' => $system['containers'] ?? null,
|
||||
], fn($v) => $v !== null);
|
||||
// Fill version/uptime from API stats if SSH didn't provide them
|
||||
if (empty($system['unraid_version']) && !empty($rStat['version'])) {
|
||||
$system['unraid_version'] = $rStat['version'];
|
||||
}
|
||||
if (empty($system['uptime_sec']) && !empty($rStat['uptime_sec'])) {
|
||||
$system['uptime_sec'] = $rStat['uptime_sec'];
|
||||
}
|
||||
}
|
||||
|
||||
$nodes[] = [
|
||||
'slot' => $slot,
|
||||
'id' => $nodeIdUpper,
|
||||
'hostname' => $hostname,
|
||||
'is_me' => $isMe,
|
||||
'is_owner' => $isOwner,
|
||||
'ts_online' => $ts['online'],
|
||||
'ts_active' => $ts['active'],
|
||||
'ts_ip' => $ts['ip'],
|
||||
'fallback' => $fbState,
|
||||
'partnership' => $ptDb,
|
||||
'system' => $system,
|
||||
'onboard_phase' => $onboardPhase,
|
||||
'key_ready' => $keyReady,
|
||||
'local_done' => $localDone,
|
||||
'api_key_set' => $apiKeySet,
|
||||
'api_key_preview' => $apiKeyPreview,
|
||||
'metrics' => $metrics,
|
||||
];
|
||||
}
|
||||
return $nodes;
|
||||
}
|
||||
|
||||
// ── Connectivity test — SSH echo with round-trip timing ─────────────────────────
|
||||
function vv_pt_ping(string $slot): array {
|
||||
$slot = strtolower($slot);
|
||||
$vars = vv_conf_vars();
|
||||
$hostname = $vars[strtoupper($slot)] ?? '';
|
||||
if (!$hostname) return ['ok' => false, 'error' => 'Unknown host slot'];
|
||||
|
||||
$currentHost = vv_detect_host();
|
||||
$myRaw = vv_read_conf_raw($currentHost . '.conf');
|
||||
$sshKey = vv_arr_scalar($myRaw, strtoupper($currentHost) . '_SSH_KEY');
|
||||
if (!$sshKey || !file_exists($sshKey)) {
|
||||
return ['ok' => false, 'error' => 'No SSH key configured on this host'];
|
||||
}
|
||||
|
||||
$ip = vv_resolve_tailscale_ip($hostname);
|
||||
if (!$ip) return ['ok' => false, 'error' => "Cannot resolve Tailscale IP for $hostname"];
|
||||
|
||||
$t0 = microtime(true);
|
||||
$out = vv_pt_ssh($ip, $sshKey, 'echo ok', 8);
|
||||
$ms = (int)round((microtime(true) - $t0) * 1000);
|
||||
|
||||
if (trim($out) === 'ok') {
|
||||
return ['ok' => true, 'latency_ms' => $ms, 'host' => $hostname, 'ip' => $ip];
|
||||
}
|
||||
return ['ok' => false, 'error' => "SSH to $hostname ($ip) failed or timed out", 'host' => $hostname];
|
||||
}
|
||||
|
||||
// ── Entry point ───────────────────────────────────────────────────────────────
|
||||
|
||||
function vv_partnership_all(): array {
|
||||
return [
|
||||
'config' => vv_pt_config(),
|
||||
'nodes' => vv_pt_nodes(),
|
||||
'sync' => vv_pt_sync(),
|
||||
'ts' => time(),
|
||||
];
|
||||
}
|
||||
+153
@@ -0,0 +1,153 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ============================= Conf Cache Sync ================================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Maintains a RAM-resident conf cache at /tmp/.vv/config/cached/.confs/.
|
||||
# Credentials and partner keys live in RAM only — never on disk across hosts.
|
||||
#
|
||||
# On array start (default / --array-start):
|
||||
# 1. Copy own conf to local cache
|
||||
# 2. Pull each available partner's conf from their disk → local cache
|
||||
# 3. Push own conf to each available partner's /tmp/.vv/ cache
|
||||
#
|
||||
# On conf save (--push-only):
|
||||
# Fast path — push updated own conf to all partners' /tmp/.vv/ cache only.
|
||||
# No pulls, no local cache rebuild.
|
||||
#
|
||||
# Cache is /tmp (tmpfs) — cleared every reboot, repopulated by this script
|
||||
# on next array start. Scripts source from cache for partner vars; own vars
|
||||
# always come from disk (load_config.sh skips cached copy of own conf).
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# conf_sync.sh Full sync: pull from all partners + push to all partners
|
||||
# conf_sync.sh --push-only Push own conf to all partners (fast, for conf-save hook)
|
||||
# conf_sync.sh --pull-only Pull partner confs into local cache only (for intermediate orch)
|
||||
# conf_sync.sh --dry-run Show what would happen, no changes
|
||||
# conf_sync.sh --log Verbose output
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
PUSH_ONLY=false
|
||||
PULL_ONLY=false
|
||||
FILTERED_ARGS=()
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--push-only) PUSH_ONLY=true ;;
|
||||
--pull-only) PULL_ONLY=true ;;
|
||||
*) FILTERED_ARGS+=("$arg") ;;
|
||||
esac
|
||||
done
|
||||
parse_args "${FILTERED_ARGS[@]}"
|
||||
detect_hosts
|
||||
|
||||
CACHE_DIR="/tmp/.vv/config/cached/.confs"
|
||||
MY_CONF="$SCRIPTS_ROOT/Configurations/${MY_ID,,}.conf"
|
||||
SSH_TIMEOUT=10
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
|
||||
# ── Ensure cache dir exists ───────────────────────────────────────────────────
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
mkdir -p "$CACHE_DIR"
|
||||
fi
|
||||
|
||||
# ── Copy own conf into local cache ───────────────────────────────────────────
|
||||
if [[ "$PUSH_ONLY" == false ]] && [[ "$PULL_ONLY" == false ]]; then
|
||||
if [[ -f "$MY_CONF" ]]; then
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would copy $(basename "$MY_CONF") → $CACHE_DIR/"
|
||||
else
|
||||
cp "$MY_CONF" "$CACHE_DIR/${MY_ID,,}.conf" && \
|
||||
log "Own conf cached ✅" || warn "Failed to cache own conf"
|
||||
fi
|
||||
else
|
||||
warn "Own conf not found: $MY_CONF"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Per-partner sync ──────────────────────────────────────────────────────────
|
||||
PUSHED=0
|
||||
PULLED=0
|
||||
FAILED=0
|
||||
|
||||
for host_var in $(compgen -v | grep -E '^HOST[0-9]+$' | sort); do
|
||||
partner_host="${!host_var}"
|
||||
[[ -z "$partner_host" ]] && continue
|
||||
[[ "${host_var,,}" == "${MY_ID,,}" ]] && continue
|
||||
|
||||
partner_slot="${host_var,,}" # e.g. host2
|
||||
partner_ip=$(resolve_tailscale_ip "$partner_host" 2>/dev/null || true)
|
||||
|
||||
if [[ -z "$partner_ip" ]]; then
|
||||
warn "$partner_host — cannot resolve Tailscale IP, skipping"
|
||||
(( FAILED++ ))
|
||||
continue
|
||||
fi
|
||||
|
||||
# ── Pull: grab partner's conf from their disk → our local cache ──────────
|
||||
if [[ "$PUSH_ONLY" == false ]]; then
|
||||
remote_conf="/boot/config/plugins/varaverk/Configurations/${partner_slot}.conf"
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would pull $partner_host:$remote_conf → $CACHE_DIR/${partner_slot}.conf"
|
||||
elif timeout "$SSH_TIMEOUT" scp -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \
|
||||
"root@${partner_ip}:${remote_conf}" \
|
||||
"$CACHE_DIR/${partner_slot}.conf" 2>/dev/null; then
|
||||
log "Pulled ${partner_slot}.conf from $partner_host ✅"
|
||||
(( PULLED++ ))
|
||||
else
|
||||
warn "Could not pull ${partner_slot}.conf from $partner_host"
|
||||
(( FAILED++ ))
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Push: send own conf to partner's /tmp/.vv/ cache ────────────────────
|
||||
if [[ "$PULL_ONLY" == true ]]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would push ${MY_ID,,}.conf → $partner_host:/tmp/.vv/config/cached/.confs/"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Ensure partner's cache dir exists, then SCP own conf into it
|
||||
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \
|
||||
"root@${partner_ip}" "mkdir -p '$CACHE_DIR'" 2>/dev/null
|
||||
|
||||
if timeout "$SSH_TIMEOUT" scp -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \
|
||||
"$MY_CONF" \
|
||||
"root@${partner_ip}:${CACHE_DIR}/${MY_ID,,}.conf" 2>/dev/null; then
|
||||
log "Pushed ${MY_ID,,}.conf to $partner_host ✅"
|
||||
(( PUSHED++ ))
|
||||
else
|
||||
warn "Could not push to $partner_host"
|
||||
(( FAILED++ ))
|
||||
fi
|
||||
done
|
||||
|
||||
# ── Summary ───────────────────────────────────────────────────────────────────
|
||||
if [[ "$PUSH_ONLY" == true ]]; then
|
||||
info "Conf push complete — pushed to $PUSHED host(s)${FAILED:+, $FAILED failed}"
|
||||
elif [[ "$PULL_ONLY" == true ]]; then
|
||||
info "Conf pull complete — pulled $PULLED partner conf(s)${FAILED:+, $FAILED failed}"
|
||||
else
|
||||
info "Conf sync complete — pulled $PULLED, pushed $PUSHED${FAILED:+, $FAILED failed}"
|
||||
fi
|
||||
|
||||
if [[ "$FAILED" -gt 0 ]]; then
|
||||
notify "Conf sync on $LOCAL_SERVER_NAME ($MY_ID) — $FAILED partner(s) failed. Partner config cache may be stale." \
|
||||
"Conf Sync" "warning"
|
||||
exit 1
|
||||
fi
|
||||
+158
@@ -0,0 +1,158 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ============================= Conf Cache Sync ================================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Maintains a RAM-resident conf cache at /tmp/.vv/config/cached/.confs/.
|
||||
# Credentials and partner keys live in RAM only — never on disk across hosts.
|
||||
#
|
||||
# On array start (default / --array-start):
|
||||
# 1. Copy own conf to local cache
|
||||
# 2. Pull each available partner's conf from their disk → local cache
|
||||
# 3. Push own conf to each available partner's /tmp/.vv/ cache
|
||||
#
|
||||
# On conf save (--push-only):
|
||||
# Fast path — push updated own conf to all partners' /tmp/.vv/ cache only.
|
||||
# No pulls, no local cache rebuild.
|
||||
#
|
||||
# Cache is /tmp (tmpfs) — cleared every reboot, repopulated by this script
|
||||
# on next array start. Scripts source from cache for partner vars; own vars
|
||||
# always come from disk (load_config.sh skips cached copy of own conf).
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# conf_sync.sh Full sync: pull from all partners + push to all partners
|
||||
# conf_sync.sh --push-only Push own conf to all partners (fast, for conf-save hook)
|
||||
# conf_sync.sh --pull-only Pull partner confs into local cache only (for intermediate orch)
|
||||
# conf_sync.sh --dry-run Show what would happen, no changes
|
||||
# conf_sync.sh --log Verbose output
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
PUSH_ONLY=false
|
||||
PULL_ONLY=false
|
||||
FILTERED_ARGS=()
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--push-only) PUSH_ONLY=true ;;
|
||||
--pull-only) PULL_ONLY=true ;;
|
||||
*) FILTERED_ARGS+=("$arg") ;;
|
||||
esac
|
||||
done
|
||||
parse_args "${FILTERED_ARGS[@]}"
|
||||
detect_hosts
|
||||
|
||||
if [[ "${CONF_SYNC_ENABLED:-true}" == false ]]; then
|
||||
log "CONF_SYNC_ENABLED=false — skipping"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
CACHE_DIR="/tmp/.vv/config/cached/.confs"
|
||||
MY_CONF="$SCRIPTS_ROOT/Configurations/${MY_ID,,}.conf"
|
||||
SSH_TIMEOUT=10
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
|
||||
# ── Ensure cache dir exists ───────────────────────────────────────────────────
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
mkdir -p "$CACHE_DIR"
|
||||
fi
|
||||
|
||||
# ── Copy own conf into local cache ───────────────────────────────────────────
|
||||
if [[ "$PUSH_ONLY" == false ]] && [[ "$PULL_ONLY" == false ]]; then
|
||||
if [[ -f "$MY_CONF" ]]; then
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would copy $(basename "$MY_CONF") → $CACHE_DIR/"
|
||||
else
|
||||
cp "$MY_CONF" "$CACHE_DIR/${MY_ID,,}.conf" && \
|
||||
log "Own conf cached ✅" || warn "Failed to cache own conf"
|
||||
fi
|
||||
else
|
||||
warn "Own conf not found: $MY_CONF"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Per-partner sync ──────────────────────────────────────────────────────────
|
||||
PUSHED=0
|
||||
PULLED=0
|
||||
FAILED=0
|
||||
|
||||
for host_var in $(compgen -v | grep -E '^HOST[0-9]+$' | sort); do
|
||||
partner_host="${!host_var}"
|
||||
[[ -z "$partner_host" ]] && continue
|
||||
[[ "${host_var,,}" == "${MY_ID,,}" ]] && continue
|
||||
|
||||
partner_slot="${host_var,,}" # e.g. host2
|
||||
partner_ip=$(resolve_tailscale_ip "$partner_host" 2>/dev/null || true)
|
||||
|
||||
if [[ -z "$partner_ip" ]]; then
|
||||
warn "$partner_host — cannot resolve Tailscale IP, skipping"
|
||||
(( FAILED++ ))
|
||||
continue
|
||||
fi
|
||||
|
||||
# ── Pull: grab partner's conf from their disk → our local cache ──────────
|
||||
if [[ "$PUSH_ONLY" == false ]]; then
|
||||
remote_conf="/boot/config/plugins/varaverk/Configurations/${partner_slot}.conf"
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would pull $partner_host:$remote_conf → $CACHE_DIR/${partner_slot}.conf"
|
||||
elif timeout "$SSH_TIMEOUT" scp -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \
|
||||
"root@${partner_ip}:${remote_conf}" \
|
||||
"$CACHE_DIR/${partner_slot}.conf" 2>/dev/null; then
|
||||
log "Pulled ${partner_slot}.conf from $partner_host ✅"
|
||||
(( PULLED++ ))
|
||||
else
|
||||
warn "Could not pull ${partner_slot}.conf from $partner_host"
|
||||
(( FAILED++ ))
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Push: send own conf to partner's /tmp/.vv/ cache ────────────────────
|
||||
if [[ "$PULL_ONLY" == true ]]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would push ${MY_ID,,}.conf → $partner_host:/tmp/.vv/config/cached/.confs/"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Ensure partner's cache dir exists, then SCP own conf into it
|
||||
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \
|
||||
"root@${partner_ip}" "mkdir -p '$CACHE_DIR'" 2>/dev/null
|
||||
|
||||
if timeout "$SSH_TIMEOUT" scp -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \
|
||||
"$MY_CONF" \
|
||||
"root@${partner_ip}:${CACHE_DIR}/${MY_ID,,}.conf" 2>/dev/null; then
|
||||
log "Pushed ${MY_ID,,}.conf to $partner_host ✅"
|
||||
(( PUSHED++ ))
|
||||
else
|
||||
warn "Could not push to $partner_host"
|
||||
(( FAILED++ ))
|
||||
fi
|
||||
done
|
||||
|
||||
# ── Summary ───────────────────────────────────────────────────────────────────
|
||||
if [[ "$PUSH_ONLY" == true ]]; then
|
||||
info "Conf push complete — pushed to $PUSHED host(s)${FAILED:+, $FAILED failed}"
|
||||
elif [[ "$PULL_ONLY" == true ]]; then
|
||||
info "Conf pull complete — pulled $PULLED partner conf(s)${FAILED:+, $FAILED failed}"
|
||||
else
|
||||
info "Conf sync complete — pulled $PULLED, pushed $PUSHED${FAILED:+, $FAILED failed}"
|
||||
fi
|
||||
|
||||
if [[ "$FAILED" -gt 0 ]]; then
|
||||
notify "Conf sync on $LOCAL_SERVER_NAME ($MY_ID) — $FAILED partner(s) failed. Partner config cache may be stale." \
|
||||
"Conf Sync" "warning"
|
||||
exit 1
|
||||
fi
|
||||
@@ -0,0 +1,147 @@
|
||||
# Varaverk — Claude Code Context
|
||||
|
||||
## Working Rules (read first)
|
||||
|
||||
- **Workspace is always** `/boot/config/plugins/varaverk` — every edit goes here.
|
||||
- **Never touch** `/mnt/user/Important Shit/Git/Development/Varaverk` — stale dev folder, ignore it.
|
||||
- **No Co-Authored-By** in commit messages unless explicitly asked.
|
||||
- **No comments** unless the WHY is genuinely non-obvious.
|
||||
- The `.plg` symlinks the installed plugin location directly to this workspace — one copy, no drift.
|
||||
|
||||
---
|
||||
|
||||
## Project: What Varaverk Is
|
||||
|
||||
Self-healing, self-maintaining, mutually-redundant two-server Unraid home media ecosystem.
|
||||
One codebase runs on both servers. No primary/standby — both run independently and cover each other.
|
||||
|
||||
**HOST1 — unRAID-Gmer4Lfe** (`gmer4lfe@gmail.com`)
|
||||
- Hardware: Threadripper 1950X, 128 GB RAM, ZFS cache pools
|
||||
- Domain: Gmer4Lfe.com
|
||||
- Runs: full arr stack (Sonarr/Radarr/Lidarr), auth stack (source of truth), Emby primary
|
||||
|
||||
**HOST2 — unRAID-Jayred365**
|
||||
- Hardware: Intel i5 10th gen, 64 GB RAM
|
||||
- Domain: Gmer4Lfe.us
|
||||
- Status: being rebuilt — most host2.conf sections scaffolded, not yet fully online
|
||||
|
||||
Networking between hosts: Tailscale mesh. No hardcoded IPs — hostnames resolve via Tailscale.
|
||||
|
||||
---
|
||||
|
||||
## Configuration System (three-file model)
|
||||
|
||||
Every script sources all three at startup:
|
||||
|
||||
```
|
||||
master.conf ← shared: thresholds, toggles, profiles, orchestrator job lists
|
||||
host1.conf ← HOST1 credentials, shares, container names, keys
|
||||
host2.conf ← HOST2 credentials, shares, container names, keys
|
||||
```
|
||||
|
||||
Sparse checkout (git) means each server only pulls its own `host*.conf`.
|
||||
HOST1 never sees HOST2 credentials and vice versa.
|
||||
|
||||
**Rule:** thresholds/toggles → `master.conf`; credentials/paths/container names → `host*.conf`.
|
||||
|
||||
`detect_hosts()` in `common.sh` matches `$(hostname)` against `HOST1`/`HOST2` in `master.conf`
|
||||
and sets `MY_ID` / `REMOTE_ID` for the rest of the script.
|
||||
|
||||
---
|
||||
|
||||
## Platform Adapter Layer
|
||||
|
||||
`Plugin/unraid/adapter.sh` isolates all OS-specific calls.
|
||||
Scripts never branch on OS directly — always call adapter functions.
|
||||
This is intentional architecture — don't bypass it.
|
||||
|
||||
---
|
||||
|
||||
## Key Paths
|
||||
|
||||
| Path | Purpose |
|
||||
|------|---------|
|
||||
| `master.conf` | Shared config — all thresholds, toggles, profiles |
|
||||
| `host1.conf` / `host2.conf` | Per-host credentials, shares, container lists |
|
||||
| `common.sh` | Shared functions — `detect_hosts()`, `log()`, `notify()`, etc. |
|
||||
| `load_config.sh` | Sources all three conf files + common.sh |
|
||||
| `State_Files/` | Runtime state (watchdogs, fallback, transcode) — survives reboots |
|
||||
| `data/` | Historical logs and stats |
|
||||
| `Plugin/unraid/` | Unraid WebGUI plugin (PHP pages, API endpoints, adapter) |
|
||||
| `Orchestrators/` | Top-level schedulers (array_started, daily, weekly, watchdog) |
|
||||
| `Watchdogs/` | docker_watchdog, system_watchdog, resource_watchdog, stability |
|
||||
| `Fallback/` | Mutual container failover logic |
|
||||
| `Rsync/` | rsync.sh + profile system |
|
||||
| `Media/` | Arr cleanup, discovery, permissions, play state sync |
|
||||
| `Tools/` | Manual one-off tools including `claude_startup.sh` |
|
||||
|
||||
---
|
||||
|
||||
## Orchestrator Schedule
|
||||
|
||||
| When | What |
|
||||
|------|------|
|
||||
| Array start | `Orchestrators/array_started.sh` → runs `ARRAY_START_SCRIPTS` |
|
||||
| Every minute | `watchdog_orchestrator.sh` → resource → docker → system → stability watchdogs |
|
||||
| Every 30 min | `critical_sync_maintenance.sh` → downloaders_reset, play_state_sync, critical rsync |
|
||||
| Every 4 hours | `intermediate_sync_maintenance.sh` → arr_sync, arrs_failed_stalled_recovery |
|
||||
| Daily 1am | `daily_sync_maintenance.sh` → git pull, permissions, cleaners, arr cleanup, docker updates |
|
||||
| Sunday 2:30am | `weekly_sync_maintenance.sh` → full Emby + Critical-Data sync, weekly restarts |
|
||||
| Sunday 3am+ | `monthly_maintenance.sh` (self-gated on 30-day uptime) → ZFS scrub, SMART tests |
|
||||
| Sunday 7am | `sunday_morning_coffee_report.sh` → ZFS, SMART, certs, backup verify, bandwidth, Emby report |
|
||||
|
||||
---
|
||||
|
||||
## Rsync Toggle State (current)
|
||||
|
||||
```bash
|
||||
RSYNC_ENABLED=true
|
||||
CRITICAL_RSYNC_ENABLED=true
|
||||
INTERMEDIATE_RSYNC_ENABLED=true
|
||||
DAILY_RSYNC_ENABLED=false # HOST2 rebuild in progress — re-enable when ready
|
||||
WEEKLY_RSYNC_ENABLED=true
|
||||
FALLBACK_RSYNC_ENABLED=true
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Fallback System
|
||||
|
||||
`fallback.sh` runs continuously from array start.
|
||||
States: `NORMAL | FALLBACK | NO_INTERNET | DARK`
|
||||
|
||||
DDNS rules are absolute:
|
||||
- Internet loss → stop own DDNS immediately
|
||||
- Failover → start remote's DDNS as Tier 1 first
|
||||
- Handback → stop remote DDNS → rsync → start containers → start local DDNS last
|
||||
|
||||
Tier delays before activating higher tiers are in `host*.conf` (`HOST1_TIER*_DELAY`, `HOST2_TIER*_DELAY`).
|
||||
|
||||
---
|
||||
|
||||
## Known Gaps / Active Work
|
||||
|
||||
- **host1.conf is missing the NPM auth credentials section** — no `HOST1_NPM_URL`, `HOST1_NPM_USER`, `HOST1_NPM_PASS`. `auth.php` reads `{$host}_NPM_USER/PASS/URL` — HOST1 falls back to empty strings. Needs the section added (same pattern as host2.conf lines 643–661).
|
||||
- HOST2 has the NPM section but `HOST2_NPM_USER` and `HOST2_NPM_PASS` are empty — fill in when HOST2 is back online.
|
||||
- `PARTNERSHIP_ENABLED=false` — not yet active.
|
||||
- `FALLBACK_ENABLED=true` — fallback is running.
|
||||
- `DAILY_RSYNC_ENABLED=false` — paused during HOST2 rebuild.
|
||||
|
||||
---
|
||||
|
||||
## Claude Code Persistence on Unraid
|
||||
|
||||
`/root` is a RAM filesystem — wiped on every reboot.
|
||||
`Tools/claude_startup.sh` runs at array start (via `ARRAY_START_SCRIPTS`) and:
|
||||
- Symlinks `/root/.claude` → `/mnt/user/appdata/claude-code/.claude`
|
||||
- Symlinks `/root/.local/share/claude` → `/mnt/user/appdata/claude-code/local/share/claude`
|
||||
- Symlinks `/root/CLAUDE.md` → `/boot/config/plugins/varaverk/CLAUDE.md` (this file)
|
||||
|
||||
This file lives on `/boot` (USB flash) and is always available regardless of array state.
|
||||
|
||||
---
|
||||
|
||||
## Commit Style
|
||||
|
||||
Plain, concise messages. No Co-Authored-By trailers. No bullet-point summaries in the body.
|
||||
One sentence on the why, not the what.
|
||||
@@ -0,0 +1,146 @@
|
||||
# Varaverk — Claude Code Context
|
||||
|
||||
## Working Rules (read first)
|
||||
|
||||
- **Workspace is always** `/boot/config/plugins/varaverk` — every edit goes here.
|
||||
- **Never touch** `/mnt/user/Important Shit/Git/Development/Varaverk` — stale dev folder, ignore it.
|
||||
- **No Co-Authored-By** in commit messages unless explicitly asked.
|
||||
- **No comments** unless the WHY is genuinely non-obvious.
|
||||
- The `.plg` symlinks the installed plugin location directly to this workspace — one copy, no drift.
|
||||
|
||||
---
|
||||
|
||||
## Project: What Varaverk Is
|
||||
|
||||
Self-healing, self-maintaining, mutually-redundant two-server Unraid home media ecosystem.
|
||||
One codebase runs on both servers. No primary/standby — both run independently and cover each other.
|
||||
|
||||
**HOST1 — unRAID-Gmer4Lfe** (`gmer4lfe@gmail.com`)
|
||||
- Hardware: Threadripper 1950X, 128 GB RAM, ZFS cache pools
|
||||
- Domain: Gmer4Lfe.com
|
||||
- Runs: full arr stack (Sonarr/Radarr/Lidarr), auth stack (source of truth), Emby primary
|
||||
|
||||
**HOST2 — unRAID-Jayred365**
|
||||
- Hardware: Intel i5 10th gen, 64 GB RAM
|
||||
- Domain: Gmer4Lfe.us
|
||||
- Status: being rebuilt — most host2.conf sections scaffolded, not yet fully online
|
||||
|
||||
Networking between hosts: Tailscale mesh. No hardcoded IPs — hostnames resolve via Tailscale.
|
||||
|
||||
---
|
||||
|
||||
## Configuration System (three-file model)
|
||||
|
||||
Every script sources all three at startup:
|
||||
|
||||
```
|
||||
master.conf ← shared: thresholds, toggles, profiles, orchestrator job lists
|
||||
host1.conf ← HOST1 credentials, shares, container names, keys
|
||||
host2.conf ← HOST2 credentials, shares, container names, keys
|
||||
```
|
||||
|
||||
Sparse checkout (git) means each server only pulls its own `host*.conf`.
|
||||
HOST1 never sees HOST2 credentials and vice versa.
|
||||
|
||||
**Rule:** thresholds/toggles → `master.conf`; credentials/paths/container names → `host*.conf`.
|
||||
|
||||
`detect_hosts()` in `common.sh` matches `$(hostname)` against `HOST1`/`HOST2` in `master.conf`
|
||||
and sets `MY_ID` / `REMOTE_ID` for the rest of the script.
|
||||
|
||||
---
|
||||
|
||||
## Platform Adapter Layer
|
||||
|
||||
`Plugin/unraid/adapter.sh` isolates all OS-specific calls.
|
||||
Scripts never branch on OS directly — always call adapter functions.
|
||||
This is intentional architecture — don't bypass it.
|
||||
|
||||
---
|
||||
|
||||
## Key Paths
|
||||
|
||||
| Path | Purpose |
|
||||
|------|---------|
|
||||
| `master.conf` | Shared config — all thresholds, toggles, profiles |
|
||||
| `host1.conf` / `host2.conf` | Per-host credentials, shares, container lists |
|
||||
| `common.sh` | Shared functions — `detect_hosts()`, `log()`, `notify()`, etc. |
|
||||
| `load_config.sh` | Sources all three conf files + common.sh |
|
||||
| `State_Files/` | Runtime state (watchdogs, fallback, transcode) — survives reboots |
|
||||
| `data/` | Historical logs and stats |
|
||||
| `Plugin/unraid/` | Unraid WebGUI plugin (PHP pages, API endpoints, adapter) |
|
||||
| `Orchestrators/` | Top-level schedulers (array_started, daily, weekly, watchdog) |
|
||||
| `Watchdogs/` | docker_watchdog, system_watchdog, resource_watchdog, stability |
|
||||
| `Fallback/` | Mutual container failover logic |
|
||||
| `Rsync/` | rsync.sh + profile system |
|
||||
| `Media/` | Arr cleanup, discovery, permissions, play state sync |
|
||||
| `Tools/` | Manual one-off tools including `claude_startup.sh` |
|
||||
|
||||
---
|
||||
|
||||
## Orchestrator Schedule
|
||||
|
||||
| When | What |
|
||||
|------|------|
|
||||
| Array start | `Orchestrators/array_started.sh` → runs `ARRAY_START_SCRIPTS` |
|
||||
| Every minute | `watchdog_orchestrator.sh` → resource → docker → system → stability watchdogs |
|
||||
| Every 30 min | `critical_sync_maintenance.sh` → downloaders_reset, play_state_sync, critical rsync |
|
||||
| Every 4 hours | `intermediate_sync_maintenance.sh` → arr_sync, arrs_failed_stalled_recovery |
|
||||
| Daily 1am | `daily_sync_maintenance.sh` → git pull, permissions, cleaners, arr cleanup, docker updates |
|
||||
| Sunday 2:30am | `weekly_sync_maintenance.sh` → full Emby + Critical-Data sync, weekly restarts |
|
||||
| Sunday 3am+ | `monthly_maintenance.sh` (self-gated on 30-day uptime) → ZFS scrub, SMART tests |
|
||||
| Sunday 7am | `sunday_morning_coffee_report.sh` → ZFS, SMART, certs, backup verify, bandwidth, Emby report |
|
||||
|
||||
---
|
||||
|
||||
## Rsync Toggle State (current)
|
||||
|
||||
```bash
|
||||
RSYNC_ENABLED=true
|
||||
CRITICAL_RSYNC_ENABLED=true
|
||||
INTERMEDIATE_RSYNC_ENABLED=true
|
||||
DAILY_RSYNC_ENABLED=false # HOST2 rebuild in progress — re-enable when ready
|
||||
WEEKLY_RSYNC_ENABLED=true
|
||||
FALLBACK_RSYNC_ENABLED=true
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Fallback System
|
||||
|
||||
`fallback.sh` runs continuously from array start.
|
||||
States: `NORMAL | FALLBACK | NO_INTERNET | DARK`
|
||||
|
||||
DDNS rules are absolute:
|
||||
- Internet loss → stop own DDNS immediately
|
||||
- Failover → start remote's DDNS as Tier 1 first
|
||||
- Handback → stop remote DDNS → rsync → start containers → start local DDNS last
|
||||
|
||||
Tier delays before activating higher tiers are in `host*.conf` (`HOST1_TIER*_DELAY`, `HOST2_TIER*_DELAY`).
|
||||
|
||||
---
|
||||
|
||||
## Known Gaps / Active Work
|
||||
|
||||
- HOST2 NPM/lldap credentials (`HOST2_NPM_USER`, `HOST2_NPM_PASS`, `HOST2_LLDAP_PASS`) are empty in `host2.conf` — fill in when HOST2 is back online.
|
||||
- `PARTNERSHIP_ENABLED=false` — not yet active.
|
||||
- `FALLBACK_ENABLED=true` — fallback is running.
|
||||
- `DAILY_RSYNC_ENABLED=false` — paused during HOST2 rebuild.
|
||||
|
||||
---
|
||||
|
||||
## Claude Code Persistence on Unraid
|
||||
|
||||
`/root` is a RAM filesystem — wiped on every reboot.
|
||||
`Tools/claude_startup.sh` runs at array start (via `ARRAY_START_SCRIPTS`) and:
|
||||
- Symlinks `/root/.claude` → `/mnt/user/appdata/claude-code/.claude`
|
||||
- Symlinks `/root/.local/share/claude` → `/mnt/user/appdata/claude-code/local/share/claude`
|
||||
- Symlinks `/root/CLAUDE.md` → `/boot/config/plugins/varaverk/CLAUDE.md` (this file)
|
||||
|
||||
This file lives on `/boot` (USB flash) and is always available regardless of array state.
|
||||
|
||||
---
|
||||
|
||||
## Commit Style
|
||||
|
||||
Plain, concise messages. No Co-Authored-By trailers. No bullet-point summaries in the body.
|
||||
One sentence on the why, not the what.
|
||||
@@ -0,0 +1,151 @@
|
||||
# Varaverk — Claude Code Context
|
||||
|
||||
## Working Rules (read first)
|
||||
|
||||
- **Workspace is always** `/boot/config/plugins/varaverk` — every edit goes here.
|
||||
- **Never touch** `/mnt/user/Important Shit/Git/Development/Varaverk` — stale dev folder, ignore it.
|
||||
- **No Co-Authored-By** in commit messages unless explicitly asked.
|
||||
- **No comments** unless the WHY is genuinely non-obvious.
|
||||
- The `.plg` symlinks the installed plugin location directly to this workspace — one copy, no drift.
|
||||
|
||||
---
|
||||
|
||||
## Project: What Varaverk Is
|
||||
|
||||
Self-healing, self-maintaining, mutually-redundant two-server Unraid home media ecosystem.
|
||||
One codebase runs on both servers. No primary/standby — both run independently and cover each other.
|
||||
|
||||
**HOST1 — unRAID-Gmer4Lfe** (`gmer4lfe@gmail.com`)
|
||||
- Hardware: Threadripper 1950X, 128 GB RAM, ZFS cache pools
|
||||
- Domain: Gmer4Lfe.com
|
||||
- Runs: full arr stack (Sonarr/Radarr/Lidarr), auth stack (source of truth), Emby primary
|
||||
|
||||
**HOST2 — unRAID-Jayred365**
|
||||
- Hardware: Intel i5 10th gen, 64 GB RAM
|
||||
- Domain: Gmer4Lfe.us
|
||||
- Status: being rebuilt — most host2.conf sections scaffolded, not yet fully online
|
||||
|
||||
Networking between hosts: Tailscale mesh. No hardcoded IPs — hostnames resolve via Tailscale.
|
||||
|
||||
---
|
||||
|
||||
## Configuration System (three-file model)
|
||||
|
||||
Every script sources all three at startup:
|
||||
|
||||
```
|
||||
master.conf ← shared: thresholds, toggles, profiles, orchestrator job lists
|
||||
host1.conf ← HOST1 credentials, shares, container names, keys
|
||||
host2.conf ← HOST2 credentials, shares, container names, keys
|
||||
```
|
||||
|
||||
Sparse checkout (git) means each server only pulls its own `host*.conf`.
|
||||
HOST1 never sees HOST2 credentials and vice versa.
|
||||
|
||||
**Rule:** thresholds/toggles → `master.conf`; credentials/paths/container names → `host*.conf`.
|
||||
|
||||
`detect_hosts()` in `common.sh` matches `$(hostname)` against `HOST1`/`HOST2` in `master.conf`
|
||||
and sets `MY_ID` / `REMOTE_ID` for the rest of the script.
|
||||
|
||||
---
|
||||
|
||||
## Platform Adapter Layer
|
||||
|
||||
`Plugin/unraid/adapter.sh` isolates all OS-specific calls.
|
||||
Scripts never branch on OS directly — always call adapter functions.
|
||||
This is intentional architecture — don't bypass it.
|
||||
|
||||
---
|
||||
|
||||
## Key Paths
|
||||
|
||||
| Path | Purpose |
|
||||
|------|---------|
|
||||
| `master.conf` | Shared config — all thresholds, toggles, profiles |
|
||||
| `host1.conf` / `host2.conf` | Per-host credentials, shares, container lists |
|
||||
| `common.sh` | Shared functions — `detect_hosts()`, `log()`, `notify()`, etc. |
|
||||
| `load_config.sh` | Sources all three conf files + common.sh |
|
||||
| `State_Files/` | Runtime state (watchdogs, fallback, transcode) — survives reboots |
|
||||
| `data/` | Historical logs and stats |
|
||||
| `Plugin/unraid/` | Unraid WebGUI plugin (PHP pages, API endpoints, adapter) |
|
||||
| `Orchestrators/` | Top-level schedulers (array_started, daily, weekly, watchdog) |
|
||||
| `Watchdogs/` | docker_watchdog, system_watchdog, resource_watchdog, stability |
|
||||
| `Fallback/` | Mutual container failover logic |
|
||||
| `Rsync/` | rsync.sh + profile system |
|
||||
| `Media/` | Arr cleanup, discovery, permissions, play state sync |
|
||||
| `Tools/` | Manual one-off tools including `claude_startup.sh` |
|
||||
|
||||
---
|
||||
|
||||
## Orchestrator Schedule
|
||||
|
||||
| When | What |
|
||||
|------|------|
|
||||
| Array start | `Orchestrators/array_started.sh` → runs `ARRAY_START_SCRIPTS` |
|
||||
| Every minute | `watchdog_orchestrator.sh` → resource → docker → system → stability watchdogs |
|
||||
| Every 30 min | `critical_sync_maintenance.sh` → downloaders_reset, play_state_sync, critical rsync |
|
||||
| Every 4 hours | `intermediate_sync_maintenance.sh` → arr_sync, arrs_failed_stalled_recovery |
|
||||
| Daily 1am | `daily_sync_maintenance.sh` → git pull, permissions, cleaners, arr cleanup, docker updates |
|
||||
| Sunday 2:30am | `weekly_sync_maintenance.sh` → full Emby + Critical-Data sync, weekly restarts |
|
||||
| Sunday 3am+ | `monthly_maintenance.sh` (self-gated on 30-day uptime) → ZFS scrub, SMART tests |
|
||||
| Sunday 7am | `sunday_morning_coffee_report.sh` → ZFS, SMART, certs, backup verify, bandwidth, Emby report |
|
||||
|
||||
---
|
||||
|
||||
## Rsync Toggle State (current)
|
||||
|
||||
```bash
|
||||
RSYNC_ENABLED=true
|
||||
CRITICAL_RSYNC_ENABLED=true
|
||||
INTERMEDIATE_RSYNC_ENABLED=true
|
||||
DAILY_RSYNC_ENABLED=false # HOST2 rebuild in progress — re-enable when ready
|
||||
WEEKLY_RSYNC_ENABLED=true
|
||||
FALLBACK_RSYNC_ENABLED=true
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Fallback System
|
||||
|
||||
`fallback.sh` runs continuously from array start.
|
||||
States: `NORMAL | FALLBACK | NO_INTERNET | DARK`
|
||||
|
||||
DDNS rules are absolute:
|
||||
- Internet loss → stop own DDNS immediately
|
||||
- Failover → start remote's DDNS as Tier 1 first
|
||||
- Handback → stop remote DDNS → rsync → start containers → start local DDNS last
|
||||
|
||||
Tier delays before activating higher tiers are in `host*.conf` (`HOST1_TIER*_DELAY`, `HOST2_TIER*_DELAY`).
|
||||
|
||||
---
|
||||
|
||||
## Port Notes
|
||||
|
||||
- **NPM admin API (`HOST1_NPM_URL`)** — port **7818**. Port 81 is the partnership WebUI port (`HOST1_PARTNERSHIP_AUTH_WEBUIS`), not the API. Easy to confuse.
|
||||
- **HOST1_NETWORK_WATCHDOG_NPM_URL** — external HTTPS domain, completely separate from the admin API.
|
||||
|
||||
## Known Gaps / Active Work
|
||||
|
||||
- HOST2 NPM/lldap credentials (`HOST2_NPM_USER`, `HOST2_NPM_PASS`, `HOST2_LLDAP_PASS`) are empty in `host2.conf` — fill in when HOST2 is back online.
|
||||
- `PARTNERSHIP_ENABLED=false` — not yet active.
|
||||
- `FALLBACK_ENABLED=true` — fallback is running.
|
||||
- `DAILY_RSYNC_ENABLED=false` — paused during HOST2 rebuild.
|
||||
|
||||
---
|
||||
|
||||
## Claude Code Persistence on Unraid
|
||||
|
||||
`/root` is a RAM filesystem — wiped on every reboot.
|
||||
`Tools/claude_startup.sh` runs at array start (via `ARRAY_START_SCRIPTS`) and:
|
||||
- Symlinks `/root/.claude` → `/mnt/user/appdata/claude-code/.claude`
|
||||
- Symlinks `/root/.local/share/claude` → `/mnt/user/appdata/claude-code/local/share/claude`
|
||||
- Symlinks `/root/CLAUDE.md` → `/boot/config/plugins/varaverk/CLAUDE.md` (this file)
|
||||
|
||||
This file lives on `/boot` (USB flash) and is always available regardless of array state.
|
||||
|
||||
---
|
||||
|
||||
## Commit Style
|
||||
|
||||
Plain, concise messages. No Co-Authored-By trailers. No bullet-point summaries in the body.
|
||||
One sentence on the why, not the what.
|
||||
@@ -0,0 +1,285 @@
|
||||
<?php
|
||||
require_once __DIR__ . '/config.php';
|
||||
|
||||
// ── Config ────────────────────────────────────────────────────────────────────
|
||||
|
||||
function vv_auth_conf(): array {
|
||||
$v = vv_conf_vars();
|
||||
$host = strtoupper(vv_detect_host());
|
||||
return [
|
||||
'npm_url' => rtrim($v["{$host}_NPM_URL"] ?? 'http://localhost:81', '/'),
|
||||
'npm_user' => $v["{$host}_NPM_USER"] ?? '',
|
||||
'npm_pass' => $v["{$host}_NPM_PASS"] ?? '',
|
||||
'lldap_url' => rtrim($v["{$host}_LLDAP_URL"] ?? 'http://localhost:17170', '/'),
|
||||
'lldap_user' => $v["{$host}_LLDAP_USER"] ?? '',
|
||||
'lldap_pass' => $v["{$host}_LLDAP_PASS"] ?? '',
|
||||
'authelia_config' => $v["{$host}_AUTHELIA_CONFIG"] ?? '/mnt/user/appdata-Fallback/Critical-Data/Authelia/configuration.yml',
|
||||
'authelia_container' => $v["{$host}_AUTHELIA_CONTAINER"] ?? 'Authelia',
|
||||
'is_owner' => vv_is_owner(),
|
||||
];
|
||||
}
|
||||
|
||||
// ── NPM ───────────────────────────────────────────────────────────────────────
|
||||
|
||||
function vv_npm_token(): string {
|
||||
if (!session_id()) session_start();
|
||||
$conf = vv_auth_conf();
|
||||
$cached = $_SESSION['vv_npm_token'] ?? '';
|
||||
$expiry = $_SESSION['vv_npm_token_exp'] ?? 0;
|
||||
if ($cached && time() < $expiry) return $cached;
|
||||
|
||||
$resp = vv_npm_raw('POST', '/api/tokens', [
|
||||
'identity' => $conf['npm_user'],
|
||||
'secret' => $conf['npm_pass'],
|
||||
'expiry' => '1d',
|
||||
], '', $conf);
|
||||
$token = $resp['token'] ?? '';
|
||||
if ($token) {
|
||||
$_SESSION['vv_npm_token'] = $token;
|
||||
$_SESSION['vv_npm_token_exp'] = time() + 82800;
|
||||
}
|
||||
return $token;
|
||||
}
|
||||
|
||||
function vv_npm_raw(string $method, string $path, array $data, string $token, array $conf = []): array {
|
||||
if (!$conf) $conf = vv_auth_conf();
|
||||
$url = $conf['npm_url'] . $path;
|
||||
$headers = ['Content-Type: application/json', 'Accept: application/json'];
|
||||
if ($token) $headers[] = 'Authorization: Bearer ' . $token;
|
||||
|
||||
$ch = curl_init($url);
|
||||
curl_setopt_array($ch, [
|
||||
CURLOPT_RETURNTRANSFER => true,
|
||||
CURLOPT_TIMEOUT => 10,
|
||||
CURLOPT_HTTPHEADER => $headers,
|
||||
CURLOPT_CUSTOMREQUEST => $method,
|
||||
]);
|
||||
if ($data && in_array($method, ['POST', 'PUT'], true))
|
||||
curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($data));
|
||||
$body = curl_exec($ch);
|
||||
curl_close($ch);
|
||||
return json_decode($body ?: '{}', true) ?: [];
|
||||
}
|
||||
|
||||
function vv_npm_req(string $method, string $path, array $data = []): array {
|
||||
$token = vv_npm_token();
|
||||
if (!$token) return ['_err' => 'NPM auth failed — check credentials in host conf'];
|
||||
return vv_npm_raw($method, $path, $data, $token);
|
||||
}
|
||||
|
||||
function vv_npm_list_proxies(): array {
|
||||
$list = vv_npm_req('GET', '/api/nginx/proxy-hosts?expand=certificate');
|
||||
if (!is_array($list) || isset($list['_err']))
|
||||
return ['ok' => false, 'error' => $list['_err'] ?? 'Invalid response from NPM'];
|
||||
return ['ok' => true, 'proxies' => $list];
|
||||
}
|
||||
|
||||
function vv_npm_list_certs(): array {
|
||||
$list = vv_npm_req('GET', '/api/nginx/certificates');
|
||||
return is_array($list) ? $list : [];
|
||||
}
|
||||
|
||||
function vv_npm_create_proxy(array $data): array {
|
||||
$r = vv_npm_req('POST', '/api/nginx/proxy-hosts', $data);
|
||||
return isset($r['id']) ? ['ok' => true, 'proxy' => $r] : ['ok' => false, 'error' => $r['error'] ?? ($r['_err'] ?? 'Create failed')];
|
||||
}
|
||||
|
||||
function vv_npm_update_proxy(int $id, array $data): array {
|
||||
$r = vv_npm_req('PUT', "/api/nginx/proxy-hosts/$id", $data);
|
||||
return isset($r['id']) ? ['ok' => true, 'proxy' => $r] : ['ok' => false, 'error' => $r['error'] ?? ($r['_err'] ?? 'Update failed')];
|
||||
}
|
||||
|
||||
function vv_npm_delete_proxy(int $id): array {
|
||||
vv_npm_req('DELETE', "/api/nginx/proxy-hosts/$id");
|
||||
return ['ok' => true];
|
||||
}
|
||||
|
||||
function vv_npm_toggle_proxy(int $id, bool $enabled): array {
|
||||
vv_npm_req('POST', "/api/nginx/proxy-hosts/$id/" . ($enabled ? 'enable' : 'disable'));
|
||||
return ['ok' => true];
|
||||
}
|
||||
|
||||
// ── lldap ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
function vv_lldap_token(): string {
|
||||
if (!session_id()) session_start();
|
||||
$conf = vv_auth_conf();
|
||||
$cached = $_SESSION['vv_lldap_token'] ?? '';
|
||||
$expiry = $_SESSION['vv_lldap_token_exp'] ?? 0;
|
||||
if ($cached && time() < $expiry) return $cached;
|
||||
|
||||
$ch = curl_init($conf['lldap_url'] . '/auth/simple/login');
|
||||
curl_setopt_array($ch, [
|
||||
CURLOPT_RETURNTRANSFER => true,
|
||||
CURLOPT_TIMEOUT => 10,
|
||||
CURLOPT_POST => true,
|
||||
CURLOPT_POSTFIELDS => json_encode(['username' => $conf['lldap_user'], 'password' => $conf['lldap_pass']]),
|
||||
CURLOPT_HTTPHEADER => ['Content-Type: application/json'],
|
||||
]);
|
||||
$body = curl_exec($ch);
|
||||
curl_close($ch);
|
||||
$resp = json_decode($body ?: '{}', true) ?: [];
|
||||
$token = $resp['token'] ?? '';
|
||||
if ($token) {
|
||||
$_SESSION['vv_lldap_token'] = $token;
|
||||
$_SESSION['vv_lldap_token_exp'] = time() + 3500;
|
||||
}
|
||||
return $token;
|
||||
}
|
||||
|
||||
function vv_lldap_gql(string $query, array $variables = []): array {
|
||||
$conf = vv_auth_conf();
|
||||
$token = vv_lldap_token();
|
||||
if (!$token) return ['errors' => [['message' => 'lldap auth failed — check credentials']]];
|
||||
|
||||
$ch = curl_init($conf['lldap_url'] . '/api/graphql');
|
||||
curl_setopt_array($ch, [
|
||||
CURLOPT_RETURNTRANSFER => true,
|
||||
CURLOPT_TIMEOUT => 10,
|
||||
CURLOPT_POST => true,
|
||||
CURLOPT_POSTFIELDS => json_encode(['query' => $query, 'variables' => $variables]),
|
||||
CURLOPT_HTTPHEADER => ['Content-Type: application/json', 'Authorization: Bearer ' . $token],
|
||||
]);
|
||||
$body = curl_exec($ch);
|
||||
curl_close($ch);
|
||||
return json_decode($body ?: '{}', true) ?: [];
|
||||
}
|
||||
|
||||
function vv_lldap_list_users(): array {
|
||||
$r = vv_lldap_gql('query { listUsers { id displayName email creationDate groups { id displayName } } }');
|
||||
if (isset($r['errors'])) return ['ok' => false, 'error' => $r['errors'][0]['message'] ?? 'Query failed'];
|
||||
return ['ok' => true, 'users' => $r['data']['listUsers'] ?? []];
|
||||
}
|
||||
|
||||
function vv_lldap_list_groups(): array {
|
||||
$r = vv_lldap_gql('query { listGroups { id displayName users { id displayName } } }');
|
||||
if (isset($r['errors'])) return ['ok' => false, 'error' => $r['errors'][0]['message'] ?? 'Query failed'];
|
||||
return ['ok' => true, 'groups' => $r['data']['listGroups'] ?? []];
|
||||
}
|
||||
|
||||
function vv_lldap_create_user(string $id, string $email, string $displayName, string $password): array {
|
||||
$r = vv_lldap_gql(
|
||||
'mutation CreateUser($user: CreateUserInput!) { createUser(user: $user) { id displayName email } }',
|
||||
['user' => ['id' => $id, 'email' => $email, 'displayName' => $displayName]]
|
||||
);
|
||||
if (isset($r['errors'])) return ['ok' => false, 'error' => $r['errors'][0]['message'] ?? 'Create failed'];
|
||||
if ($password) vv_lldap_set_password($id, $password);
|
||||
return ['ok' => true, 'user' => $r['data']['createUser'] ?? []];
|
||||
}
|
||||
|
||||
function vv_lldap_update_user(string $id, string $email, string $displayName): array {
|
||||
$r = vv_lldap_gql(
|
||||
'mutation UpdateUser($user: UpdateUserInput!) { updateUser(user: $user) { ok } }',
|
||||
['user' => ['id' => $id, 'email' => $email, 'displayName' => $displayName]]
|
||||
);
|
||||
if (isset($r['errors'])) return ['ok' => false, 'error' => $r['errors'][0]['message'] ?? 'Update failed'];
|
||||
return ['ok' => true];
|
||||
}
|
||||
|
||||
function vv_lldap_delete_user(string $id): array {
|
||||
$r = vv_lldap_gql(
|
||||
'mutation DeleteUser($userId: String!) { deleteUser(userId: $userId) { ok } }',
|
||||
['userId' => $id]
|
||||
);
|
||||
if (isset($r['errors'])) return ['ok' => false, 'error' => $r['errors'][0]['message'] ?? 'Delete failed'];
|
||||
return ['ok' => true];
|
||||
}
|
||||
|
||||
function vv_lldap_set_password(string $userId, string $password): array {
|
||||
$r = vv_lldap_gql(
|
||||
'mutation ChangePassword($userId: String!, $password: String!) { changeUserPassword(userId: $userId, password: $password) }',
|
||||
['userId' => $userId, 'password' => $password]
|
||||
);
|
||||
if (isset($r['errors'])) return ['ok' => false, 'error' => $r['errors'][0]['message'] ?? 'Password change failed'];
|
||||
return ['ok' => true];
|
||||
}
|
||||
|
||||
function vv_lldap_create_group(string $name): array {
|
||||
$r = vv_lldap_gql(
|
||||
'mutation CreateGroup($name: String!) { createGroup(name: $name) { id displayName } }',
|
||||
['name' => $name]
|
||||
);
|
||||
if (isset($r['errors'])) return ['ok' => false, 'error' => $r['errors'][0]['message'] ?? 'Create failed'];
|
||||
return ['ok' => true, 'group' => $r['data']['createGroup'] ?? []];
|
||||
}
|
||||
|
||||
function vv_lldap_delete_group(int $id): array {
|
||||
$r = vv_lldap_gql(
|
||||
'mutation DeleteGroup($groupId: Int!) { deleteGroup(groupId: $groupId) { ok } }',
|
||||
['groupId' => $id]
|
||||
);
|
||||
if (isset($r['errors'])) return ['ok' => false, 'error' => $r['errors'][0]['message'] ?? 'Delete failed'];
|
||||
return ['ok' => true];
|
||||
}
|
||||
|
||||
function vv_lldap_add_to_group(string $userId, int $groupId): array {
|
||||
$r = vv_lldap_gql(
|
||||
'mutation AddUserToGroup($userId: String!, $groupId: Int!) { addUserToGroup(userId: $userId, groupId: $groupId) { ok } }',
|
||||
['userId' => $userId, 'groupId' => $groupId]
|
||||
);
|
||||
if (isset($r['errors'])) return ['ok' => false, 'error' => $r['errors'][0]['message'] ?? 'Failed'];
|
||||
return ['ok' => true];
|
||||
}
|
||||
|
||||
function vv_lldap_remove_from_group(string $userId, int $groupId): array {
|
||||
$r = vv_lldap_gql(
|
||||
'mutation RemoveUserFromGroup($userId: String!, $groupId: Int!) { removeUserFromGroup(userId: $userId, groupId: $groupId) { ok } }',
|
||||
['userId' => $userId, 'groupId' => $groupId]
|
||||
);
|
||||
if (isset($r['errors'])) return ['ok' => false, 'error' => $r['errors'][0]['message'] ?? 'Failed'];
|
||||
return ['ok' => true];
|
||||
}
|
||||
|
||||
// ── Authelia ──────────────────────────────────────────────────────────────────
|
||||
|
||||
function vv_authelia_read_rules(): array {
|
||||
$conf = vv_auth_conf();
|
||||
$file = $conf['authelia_config'];
|
||||
if (!file_exists($file)) return ['ok' => false, 'error' => 'Config not found: ' . $file];
|
||||
|
||||
$py = "import yaml,json,sys\n"
|
||||
. "d=yaml.safe_load(open(sys.argv[1]))\n"
|
||||
. "ac=d.get('access_control',{})\n"
|
||||
. "print(json.dumps({'default_policy':ac.get('default_policy','deny'),'rules':ac.get('rules',[])}))\n";
|
||||
$tmp = '/tmp/vv_auth_rd_' . getmypid() . '.py';
|
||||
file_put_contents($tmp, $py);
|
||||
$out = shell_exec('python3 ' . escapeshellarg($tmp) . ' ' . escapeshellarg($file) . ' 2>/dev/null');
|
||||
@unlink($tmp);
|
||||
|
||||
if (!$out) return ['ok' => false, 'error' => 'Parse failed — python3 with PyYAML required'];
|
||||
$data = json_decode(trim($out), true);
|
||||
if (!$data) return ['ok' => false, 'error' => 'Invalid YAML response'];
|
||||
return ['ok' => true, 'default_policy' => $data['default_policy'], 'rules' => $data['rules']];
|
||||
}
|
||||
|
||||
function vv_authelia_write_rules(array $rules, string $defaultPolicy): array {
|
||||
$conf = vv_auth_conf();
|
||||
$file = $conf['authelia_config'];
|
||||
if (!file_exists($file)) return ['ok' => false, 'error' => 'Config not found: ' . $file];
|
||||
|
||||
$acJson = json_encode(['default_policy' => $defaultPolicy, 'rules' => $rules]);
|
||||
$py = <<<'PYEOF'
|
||||
import yaml, json, sys, re
|
||||
config_file = sys.argv[1]
|
||||
new_ac = json.loads(sys.argv[2])
|
||||
with open(config_file, 'r') as f:
|
||||
content = f.read()
|
||||
new_block = yaml.dump({'access_control': new_ac}, default_flow_style=False, allow_unicode=True, sort_keys=False)
|
||||
pattern = r'(?ms)^access_control:.*?(?=^[a-zA-Z#]|\Z)'
|
||||
if re.search(pattern, content):
|
||||
content = re.sub(pattern, new_block + '\n', content)
|
||||
else:
|
||||
content = content.rstrip('\n') + '\n\n' + new_block + '\n'
|
||||
with open(config_file, 'w') as f:
|
||||
f.write(content)
|
||||
print('ok')
|
||||
PYEOF;
|
||||
$tmp = '/tmp/vv_auth_wr_' . getmypid() . '.py';
|
||||
file_put_contents($tmp, $py);
|
||||
$out = shell_exec('python3 ' . escapeshellarg($tmp) . ' ' . escapeshellarg($file) . ' ' . escapeshellarg($acJson) . ' 2>&1');
|
||||
@unlink($tmp);
|
||||
|
||||
if (trim($out) !== 'ok') return ['ok' => false, 'error' => 'Write failed: ' . trim($out)];
|
||||
shell_exec('docker restart ' . escapeshellarg($conf['authelia_container']) . ' >/dev/null 2>&1 &');
|
||||
return ['ok' => true];
|
||||
}
|
||||
@@ -0,0 +1,284 @@
|
||||
<?php
|
||||
require_once __DIR__ . '/config.php';
|
||||
|
||||
// ── Config ────────────────────────────────────────────────────────────────────
|
||||
|
||||
function vv_auth_conf(): array {
|
||||
$v = vv_conf_vars();
|
||||
$host = strtoupper(vv_detect_host());
|
||||
return [
|
||||
'npm_url' => rtrim($v["{$host}_NPM_URL"] ?? 'http://localhost:81', '/'),
|
||||
'npm_user' => $v["{$host}_NPM_USER"] ?? '',
|
||||
'npm_pass' => $v["{$host}_NPM_PASS"] ?? '',
|
||||
'lldap_url' => rtrim($v["{$host}_LLDAP_URL"] ?? 'http://localhost:17170', '/'),
|
||||
'lldap_user' => $v["{$host}_LLDAP_USER"] ?? '',
|
||||
'lldap_pass' => $v["{$host}_LLDAP_PASS"] ?? '',
|
||||
'authelia_config' => $v["{$host}_AUTHELIA_CONFIG"] ?? '/mnt/user/appdata-Fallback/Critical-Data/Authelia/configuration.yml',
|
||||
'authelia_container' => $v["{$host}_AUTHELIA_CONTAINER"] ?? 'Authelia',
|
||||
'is_owner' => vv_is_owner(),
|
||||
];
|
||||
}
|
||||
|
||||
// ── NPM ───────────────────────────────────────────────────────────────────────
|
||||
|
||||
function vv_npm_token(): string {
|
||||
if (!session_id()) session_start();
|
||||
$conf = vv_auth_conf();
|
||||
$cached = $_SESSION['vv_npm_token'] ?? '';
|
||||
$expiry = $_SESSION['vv_npm_token_exp'] ?? 0;
|
||||
if ($cached && time() < $expiry) return $cached;
|
||||
|
||||
$resp = vv_npm_raw('POST', '/api/tokens', [
|
||||
'identity' => $conf['npm_user'],
|
||||
'secret' => $conf['npm_pass'],
|
||||
], '', $conf);
|
||||
$token = $resp['token'] ?? '';
|
||||
if ($token) {
|
||||
$_SESSION['vv_npm_token'] = $token;
|
||||
$_SESSION['vv_npm_token_exp'] = time() + 82800;
|
||||
}
|
||||
return $token;
|
||||
}
|
||||
|
||||
function vv_npm_raw(string $method, string $path, array $data, string $token, array $conf = []): array {
|
||||
if (!$conf) $conf = vv_auth_conf();
|
||||
$url = $conf['npm_url'] . $path;
|
||||
$headers = ['Content-Type: application/json', 'Accept: application/json'];
|
||||
if ($token) $headers[] = 'Authorization: Bearer ' . $token;
|
||||
|
||||
$ch = curl_init($url);
|
||||
curl_setopt_array($ch, [
|
||||
CURLOPT_RETURNTRANSFER => true,
|
||||
CURLOPT_TIMEOUT => 10,
|
||||
CURLOPT_HTTPHEADER => $headers,
|
||||
CURLOPT_CUSTOMREQUEST => $method,
|
||||
]);
|
||||
if ($data && in_array($method, ['POST', 'PUT'], true))
|
||||
curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($data));
|
||||
$body = curl_exec($ch);
|
||||
curl_close($ch);
|
||||
return json_decode($body ?: '{}', true) ?: [];
|
||||
}
|
||||
|
||||
function vv_npm_req(string $method, string $path, array $data = []): array {
|
||||
$token = vv_npm_token();
|
||||
if (!$token) return ['_err' => 'NPM auth failed — check credentials in host conf'];
|
||||
return vv_npm_raw($method, $path, $data, $token);
|
||||
}
|
||||
|
||||
function vv_npm_list_proxies(): array {
|
||||
$list = vv_npm_req('GET', '/api/nginx/proxy-hosts?expand=certificate');
|
||||
if (!is_array($list) || isset($list['_err']))
|
||||
return ['ok' => false, 'error' => $list['_err'] ?? 'Invalid response from NPM'];
|
||||
return ['ok' => true, 'proxies' => $list];
|
||||
}
|
||||
|
||||
function vv_npm_list_certs(): array {
|
||||
$list = vv_npm_req('GET', '/api/nginx/certificates');
|
||||
return is_array($list) ? $list : [];
|
||||
}
|
||||
|
||||
function vv_npm_create_proxy(array $data): array {
|
||||
$r = vv_npm_req('POST', '/api/nginx/proxy-hosts', $data);
|
||||
return isset($r['id']) ? ['ok' => true, 'proxy' => $r] : ['ok' => false, 'error' => $r['error'] ?? ($r['_err'] ?? 'Create failed')];
|
||||
}
|
||||
|
||||
function vv_npm_update_proxy(int $id, array $data): array {
|
||||
$r = vv_npm_req('PUT', "/api/nginx/proxy-hosts/$id", $data);
|
||||
return isset($r['id']) ? ['ok' => true, 'proxy' => $r] : ['ok' => false, 'error' => $r['error'] ?? ($r['_err'] ?? 'Update failed')];
|
||||
}
|
||||
|
||||
function vv_npm_delete_proxy(int $id): array {
|
||||
vv_npm_req('DELETE', "/api/nginx/proxy-hosts/$id");
|
||||
return ['ok' => true];
|
||||
}
|
||||
|
||||
function vv_npm_toggle_proxy(int $id, bool $enabled): array {
|
||||
vv_npm_req('POST', "/api/nginx/proxy-hosts/$id/" . ($enabled ? 'enable' : 'disable'));
|
||||
return ['ok' => true];
|
||||
}
|
||||
|
||||
// ── lldap ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
function vv_lldap_token(): string {
|
||||
if (!session_id()) session_start();
|
||||
$conf = vv_auth_conf();
|
||||
$cached = $_SESSION['vv_lldap_token'] ?? '';
|
||||
$expiry = $_SESSION['vv_lldap_token_exp'] ?? 0;
|
||||
if ($cached && time() < $expiry) return $cached;
|
||||
|
||||
$ch = curl_init($conf['lldap_url'] . '/auth/simple/login');
|
||||
curl_setopt_array($ch, [
|
||||
CURLOPT_RETURNTRANSFER => true,
|
||||
CURLOPT_TIMEOUT => 10,
|
||||
CURLOPT_POST => true,
|
||||
CURLOPT_POSTFIELDS => json_encode(['username' => $conf['lldap_user'], 'password' => $conf['lldap_pass']]),
|
||||
CURLOPT_HTTPHEADER => ['Content-Type: application/json'],
|
||||
]);
|
||||
$body = curl_exec($ch);
|
||||
curl_close($ch);
|
||||
$resp = json_decode($body ?: '{}', true) ?: [];
|
||||
$token = $resp['token'] ?? '';
|
||||
if ($token) {
|
||||
$_SESSION['vv_lldap_token'] = $token;
|
||||
$_SESSION['vv_lldap_token_exp'] = time() + 3500;
|
||||
}
|
||||
return $token;
|
||||
}
|
||||
|
||||
function vv_lldap_gql(string $query, array $variables = []): array {
|
||||
$conf = vv_auth_conf();
|
||||
$token = vv_lldap_token();
|
||||
if (!$token) return ['errors' => [['message' => 'lldap auth failed — check credentials']]];
|
||||
|
||||
$ch = curl_init($conf['lldap_url'] . '/api/graphql');
|
||||
curl_setopt_array($ch, [
|
||||
CURLOPT_RETURNTRANSFER => true,
|
||||
CURLOPT_TIMEOUT => 10,
|
||||
CURLOPT_POST => true,
|
||||
CURLOPT_POSTFIELDS => json_encode(['query' => $query, 'variables' => $variables]),
|
||||
CURLOPT_HTTPHEADER => ['Content-Type: application/json', 'Authorization: Bearer ' . $token],
|
||||
]);
|
||||
$body = curl_exec($ch);
|
||||
curl_close($ch);
|
||||
return json_decode($body ?: '{}', true) ?: [];
|
||||
}
|
||||
|
||||
function vv_lldap_list_users(): array {
|
||||
$r = vv_lldap_gql('query { listUsers { id displayName email creationDate groups { id displayName } } }');
|
||||
if (isset($r['errors'])) return ['ok' => false, 'error' => $r['errors'][0]['message'] ?? 'Query failed'];
|
||||
return ['ok' => true, 'users' => $r['data']['listUsers'] ?? []];
|
||||
}
|
||||
|
||||
function vv_lldap_list_groups(): array {
|
||||
$r = vv_lldap_gql('query { listGroups { id displayName users { id displayName } } }');
|
||||
if (isset($r['errors'])) return ['ok' => false, 'error' => $r['errors'][0]['message'] ?? 'Query failed'];
|
||||
return ['ok' => true, 'groups' => $r['data']['listGroups'] ?? []];
|
||||
}
|
||||
|
||||
function vv_lldap_create_user(string $id, string $email, string $displayName, string $password): array {
|
||||
$r = vv_lldap_gql(
|
||||
'mutation CreateUser($user: CreateUserInput!) { createUser(user: $user) { id displayName email } }',
|
||||
['user' => ['id' => $id, 'email' => $email, 'displayName' => $displayName]]
|
||||
);
|
||||
if (isset($r['errors'])) return ['ok' => false, 'error' => $r['errors'][0]['message'] ?? 'Create failed'];
|
||||
if ($password) vv_lldap_set_password($id, $password);
|
||||
return ['ok' => true, 'user' => $r['data']['createUser'] ?? []];
|
||||
}
|
||||
|
||||
function vv_lldap_update_user(string $id, string $email, string $displayName): array {
|
||||
$r = vv_lldap_gql(
|
||||
'mutation UpdateUser($user: UpdateUserInput!) { updateUser(user: $user) { ok } }',
|
||||
['user' => ['id' => $id, 'email' => $email, 'displayName' => $displayName]]
|
||||
);
|
||||
if (isset($r['errors'])) return ['ok' => false, 'error' => $r['errors'][0]['message'] ?? 'Update failed'];
|
||||
return ['ok' => true];
|
||||
}
|
||||
|
||||
function vv_lldap_delete_user(string $id): array {
|
||||
$r = vv_lldap_gql(
|
||||
'mutation DeleteUser($userId: String!) { deleteUser(userId: $userId) { ok } }',
|
||||
['userId' => $id]
|
||||
);
|
||||
if (isset($r['errors'])) return ['ok' => false, 'error' => $r['errors'][0]['message'] ?? 'Delete failed'];
|
||||
return ['ok' => true];
|
||||
}
|
||||
|
||||
function vv_lldap_set_password(string $userId, string $password): array {
|
||||
$r = vv_lldap_gql(
|
||||
'mutation ChangePassword($userId: String!, $password: String!) { changeUserPassword(userId: $userId, password: $password) }',
|
||||
['userId' => $userId, 'password' => $password]
|
||||
);
|
||||
if (isset($r['errors'])) return ['ok' => false, 'error' => $r['errors'][0]['message'] ?? 'Password change failed'];
|
||||
return ['ok' => true];
|
||||
}
|
||||
|
||||
function vv_lldap_create_group(string $name): array {
|
||||
$r = vv_lldap_gql(
|
||||
'mutation CreateGroup($name: String!) { createGroup(name: $name) { id displayName } }',
|
||||
['name' => $name]
|
||||
);
|
||||
if (isset($r['errors'])) return ['ok' => false, 'error' => $r['errors'][0]['message'] ?? 'Create failed'];
|
||||
return ['ok' => true, 'group' => $r['data']['createGroup'] ?? []];
|
||||
}
|
||||
|
||||
function vv_lldap_delete_group(int $id): array {
|
||||
$r = vv_lldap_gql(
|
||||
'mutation DeleteGroup($groupId: Int!) { deleteGroup(groupId: $groupId) { ok } }',
|
||||
['groupId' => $id]
|
||||
);
|
||||
if (isset($r['errors'])) return ['ok' => false, 'error' => $r['errors'][0]['message'] ?? 'Delete failed'];
|
||||
return ['ok' => true];
|
||||
}
|
||||
|
||||
function vv_lldap_add_to_group(string $userId, int $groupId): array {
|
||||
$r = vv_lldap_gql(
|
||||
'mutation AddUserToGroup($userId: String!, $groupId: Int!) { addUserToGroup(userId: $userId, groupId: $groupId) { ok } }',
|
||||
['userId' => $userId, 'groupId' => $groupId]
|
||||
);
|
||||
if (isset($r['errors'])) return ['ok' => false, 'error' => $r['errors'][0]['message'] ?? 'Failed'];
|
||||
return ['ok' => true];
|
||||
}
|
||||
|
||||
function vv_lldap_remove_from_group(string $userId, int $groupId): array {
|
||||
$r = vv_lldap_gql(
|
||||
'mutation RemoveUserFromGroup($userId: String!, $groupId: Int!) { removeUserFromGroup(userId: $userId, groupId: $groupId) { ok } }',
|
||||
['userId' => $userId, 'groupId' => $groupId]
|
||||
);
|
||||
if (isset($r['errors'])) return ['ok' => false, 'error' => $r['errors'][0]['message'] ?? 'Failed'];
|
||||
return ['ok' => true];
|
||||
}
|
||||
|
||||
// ── Authelia ──────────────────────────────────────────────────────────────────
|
||||
|
||||
function vv_authelia_read_rules(): array {
|
||||
$conf = vv_auth_conf();
|
||||
$file = $conf['authelia_config'];
|
||||
if (!file_exists($file)) return ['ok' => false, 'error' => 'Config not found: ' . $file];
|
||||
|
||||
$py = "import yaml,json,sys\n"
|
||||
. "d=yaml.safe_load(open(sys.argv[1]))\n"
|
||||
. "ac=d.get('access_control',{})\n"
|
||||
. "print(json.dumps({'default_policy':ac.get('default_policy','deny'),'rules':ac.get('rules',[])}))\n";
|
||||
$tmp = '/tmp/vv_auth_rd_' . getmypid() . '.py';
|
||||
file_put_contents($tmp, $py);
|
||||
$out = shell_exec('python3 ' . escapeshellarg($tmp) . ' ' . escapeshellarg($file) . ' 2>/dev/null');
|
||||
@unlink($tmp);
|
||||
|
||||
if (!$out) return ['ok' => false, 'error' => 'Parse failed — python3 with PyYAML required'];
|
||||
$data = json_decode(trim($out), true);
|
||||
if (!$data) return ['ok' => false, 'error' => 'Invalid YAML response'];
|
||||
return ['ok' => true, 'default_policy' => $data['default_policy'], 'rules' => $data['rules']];
|
||||
}
|
||||
|
||||
function vv_authelia_write_rules(array $rules, string $defaultPolicy): array {
|
||||
$conf = vv_auth_conf();
|
||||
$file = $conf['authelia_config'];
|
||||
if (!file_exists($file)) return ['ok' => false, 'error' => 'Config not found: ' . $file];
|
||||
|
||||
$acJson = json_encode(['default_policy' => $defaultPolicy, 'rules' => $rules]);
|
||||
$py = <<<'PYEOF'
|
||||
import yaml, json, sys, re
|
||||
config_file = sys.argv[1]
|
||||
new_ac = json.loads(sys.argv[2])
|
||||
with open(config_file, 'r') as f:
|
||||
content = f.read()
|
||||
new_block = yaml.dump({'access_control': new_ac}, default_flow_style=False, allow_unicode=True, sort_keys=False)
|
||||
pattern = r'(?ms)^access_control:.*?(?=^[a-zA-Z#]|\Z)'
|
||||
if re.search(pattern, content):
|
||||
content = re.sub(pattern, new_block + '\n', content)
|
||||
else:
|
||||
content = content.rstrip('\n') + '\n\n' + new_block + '\n'
|
||||
with open(config_file, 'w') as f:
|
||||
f.write(content)
|
||||
print('ok')
|
||||
PYEOF;
|
||||
$tmp = '/tmp/vv_auth_wr_' . getmypid() . '.py';
|
||||
file_put_contents($tmp, $py);
|
||||
$out = shell_exec('python3 ' . escapeshellarg($tmp) . ' ' . escapeshellarg($file) . ' ' . escapeshellarg($acJson) . ' 2>&1');
|
||||
@unlink($tmp);
|
||||
|
||||
if (trim($out) !== 'ok') return ['ok' => false, 'error' => 'Write failed: ' . trim($out)];
|
||||
shell_exec('docker restart ' . escapeshellarg($conf['authelia_container']) . ' >/dev/null 2>&1 &');
|
||||
return ['ok' => true];
|
||||
}
|
||||
@@ -0,0 +1,296 @@
|
||||
<?php
|
||||
require_once __DIR__ . '/config.php';
|
||||
|
||||
// ── Config ────────────────────────────────────────────────────────────────────
|
||||
|
||||
function vv_auth_conf(): array {
|
||||
$v = vv_conf_vars();
|
||||
$host = strtoupper(vv_detect_host());
|
||||
return [
|
||||
'npm_url' => rtrim($v["{$host}_NPM_URL"] ?? 'http://localhost:81', '/'),
|
||||
'npm_user' => $v["{$host}_NPM_USER"] ?? '',
|
||||
'npm_pass' => $v["{$host}_NPM_PASS"] ?? '',
|
||||
'lldap_url' => rtrim($v["{$host}_LLDAP_URL"] ?? 'http://localhost:17170', '/'),
|
||||
'lldap_user' => $v["{$host}_LLDAP_USER"] ?? '',
|
||||
'lldap_pass' => $v["{$host}_LLDAP_PASS"] ?? '',
|
||||
'authelia_config' => $v["{$host}_AUTHELIA_CONFIG"] ?? '/mnt/user/appdata-Fallback/Critical-Data/Authelia/configuration.yml',
|
||||
'authelia_container' => $v["{$host}_AUTHELIA_CONTAINER"] ?? 'Authelia',
|
||||
'is_owner' => vv_is_owner(),
|
||||
];
|
||||
}
|
||||
|
||||
// ── NPM ───────────────────────────────────────────────────────────────────────
|
||||
|
||||
function vv_npm_token(): string {
|
||||
if (!session_id()) session_start();
|
||||
$conf = vv_auth_conf();
|
||||
$cached = $_SESSION['vv_npm_token'] ?? '';
|
||||
$expiry = $_SESSION['vv_npm_token_exp'] ?? 0;
|
||||
if ($cached && time() < $expiry) return $cached;
|
||||
|
||||
$resp = vv_npm_raw('POST', '/api/tokens', [
|
||||
'identity' => $conf['npm_user'],
|
||||
'secret' => $conf['npm_pass'],
|
||||
], '', $conf);
|
||||
$token = $resp['token'] ?? '';
|
||||
if ($token) {
|
||||
$_SESSION['vv_npm_token'] = $token;
|
||||
$_SESSION['vv_npm_token_exp'] = time() + 82800;
|
||||
}
|
||||
return $token;
|
||||
}
|
||||
|
||||
function vv_npm_raw(string $method, string $path, array $data, string $token, array $conf = []): array {
|
||||
if (!$conf) $conf = vv_auth_conf();
|
||||
$url = $conf['npm_url'] . $path;
|
||||
$headers = ['Content-Type: application/json', 'Accept: application/json'];
|
||||
if ($token) $headers[] = 'Authorization: Bearer ' . $token;
|
||||
|
||||
$ch = curl_init($url);
|
||||
curl_setopt_array($ch, [
|
||||
CURLOPT_RETURNTRANSFER => true,
|
||||
CURLOPT_TIMEOUT => 10,
|
||||
CURLOPT_HTTPHEADER => $headers,
|
||||
CURLOPT_CUSTOMREQUEST => $method,
|
||||
]);
|
||||
if ($data && in_array($method, ['POST', 'PUT'], true))
|
||||
curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($data));
|
||||
$body = curl_exec($ch);
|
||||
curl_close($ch);
|
||||
return json_decode($body ?: '{}', true) ?: [];
|
||||
}
|
||||
|
||||
function vv_npm_req(string $method, string $path, array $data = []): array {
|
||||
$token = vv_npm_token();
|
||||
if (!$token) return ['_err' => 'NPM auth failed — check credentials in host conf'];
|
||||
return vv_npm_raw($method, $path, $data, $token);
|
||||
}
|
||||
|
||||
function vv_npm_list_proxies(): array {
|
||||
$list = vv_npm_req('GET', '/api/nginx/proxy-hosts?expand=certificate');
|
||||
if (!is_array($list) || isset($list['_err']))
|
||||
return ['ok' => false, 'error' => $list['_err'] ?? 'Invalid response from NPM'];
|
||||
return ['ok' => true, 'proxies' => $list];
|
||||
}
|
||||
|
||||
function vv_npm_list_certs(): array {
|
||||
$list = vv_npm_req('GET', '/api/nginx/certificates');
|
||||
return is_array($list) ? $list : [];
|
||||
}
|
||||
|
||||
function vv_npm_create_proxy(array $data): array {
|
||||
$r = vv_npm_req('POST', '/api/nginx/proxy-hosts', $data);
|
||||
return isset($r['id']) ? ['ok' => true, 'proxy' => $r] : ['ok' => false, 'error' => $r['error'] ?? ($r['_err'] ?? 'Create failed')];
|
||||
}
|
||||
|
||||
function vv_npm_update_proxy(int $id, array $data): array {
|
||||
$r = vv_npm_req('PUT', "/api/nginx/proxy-hosts/$id", $data);
|
||||
return isset($r['id']) ? ['ok' => true, 'proxy' => $r] : ['ok' => false, 'error' => $r['error'] ?? ($r['_err'] ?? 'Update failed')];
|
||||
}
|
||||
|
||||
function vv_npm_delete_proxy(int $id): array {
|
||||
vv_npm_req('DELETE', "/api/nginx/proxy-hosts/$id");
|
||||
return ['ok' => true];
|
||||
}
|
||||
|
||||
function vv_npm_toggle_proxy(int $id, bool $enabled): array {
|
||||
vv_npm_req('POST', "/api/nginx/proxy-hosts/$id/" . ($enabled ? 'enable' : 'disable'));
|
||||
return ['ok' => true];
|
||||
}
|
||||
|
||||
// ── lldap ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
function vv_lldap_token(): string {
|
||||
if (!session_id()) session_start();
|
||||
$conf = vv_auth_conf();
|
||||
$cached = $_SESSION['vv_lldap_token'] ?? '';
|
||||
$expiry = $_SESSION['vv_lldap_token_exp'] ?? 0;
|
||||
if ($cached && time() < $expiry) return $cached;
|
||||
|
||||
$ch = curl_init($conf['lldap_url'] . '/auth/simple/login');
|
||||
curl_setopt_array($ch, [
|
||||
CURLOPT_RETURNTRANSFER => true,
|
||||
CURLOPT_TIMEOUT => 10,
|
||||
CURLOPT_POST => true,
|
||||
CURLOPT_POSTFIELDS => json_encode(['username' => $conf['lldap_user'], 'password' => $conf['lldap_pass']]),
|
||||
CURLOPT_HTTPHEADER => ['Content-Type: application/json'],
|
||||
]);
|
||||
$body = curl_exec($ch);
|
||||
curl_close($ch);
|
||||
$resp = json_decode($body ?: '{}', true) ?: [];
|
||||
$token = $resp['token'] ?? '';
|
||||
if ($token) {
|
||||
$_SESSION['vv_lldap_token'] = $token;
|
||||
$_SESSION['vv_lldap_token_exp'] = time() + 3500;
|
||||
}
|
||||
return $token;
|
||||
}
|
||||
|
||||
function vv_lldap_gql(string $query, array $variables = []): array {
|
||||
$conf = vv_auth_conf();
|
||||
$token = vv_lldap_token();
|
||||
if (!$token) return ['errors' => [['message' => 'lldap auth failed — check credentials']]];
|
||||
|
||||
$ch = curl_init($conf['lldap_url'] . '/api/graphql');
|
||||
curl_setopt_array($ch, [
|
||||
CURLOPT_RETURNTRANSFER => true,
|
||||
CURLOPT_TIMEOUT => 10,
|
||||
CURLOPT_POST => true,
|
||||
CURLOPT_POSTFIELDS => json_encode(['query' => $query, 'variables' => $variables]),
|
||||
CURLOPT_HTTPHEADER => ['Content-Type: application/json', 'Authorization: Bearer ' . $token],
|
||||
]);
|
||||
$body = curl_exec($ch);
|
||||
curl_close($ch);
|
||||
return json_decode($body ?: '{}', true) ?: [];
|
||||
}
|
||||
|
||||
function vv_lldap_list_users(): array {
|
||||
$r = vv_lldap_gql('query { users { id displayName email creationDate groups { id displayName } } }');
|
||||
if (isset($r['errors'])) return ['ok' => false, 'error' => $r['errors'][0]['message'] ?? 'Query failed'];
|
||||
return ['ok' => true, 'users' => $r['data']['users'] ?? []];
|
||||
}
|
||||
|
||||
function vv_lldap_list_groups(): array {
|
||||
$r = vv_lldap_gql('query { groups { id displayName users { id displayName } } }');
|
||||
if (isset($r['errors'])) return ['ok' => false, 'error' => $r['errors'][0]['message'] ?? 'Query failed'];
|
||||
return ['ok' => true, 'groups' => $r['data']['groups'] ?? []];
|
||||
}
|
||||
|
||||
function vv_lldap_create_user(string $id, string $email, string $displayName, string $password): array {
|
||||
$r = vv_lldap_gql(
|
||||
'mutation CreateUser($user: CreateUserInput!) { createUser(user: $user) { id displayName email } }',
|
||||
['user' => ['id' => $id, 'email' => $email, 'displayName' => $displayName]]
|
||||
);
|
||||
if (isset($r['errors'])) return ['ok' => false, 'error' => $r['errors'][0]['message'] ?? 'Create failed'];
|
||||
if ($password) vv_lldap_set_password($id, $password);
|
||||
return ['ok' => true, 'user' => $r['data']['createUser'] ?? []];
|
||||
}
|
||||
|
||||
function vv_lldap_update_user(string $id, string $email, string $displayName): array {
|
||||
$r = vv_lldap_gql(
|
||||
'mutation UpdateUser($user: UpdateUserInput!) { updateUser(user: $user) { ok } }',
|
||||
['user' => ['id' => $id, 'email' => $email, 'displayName' => $displayName]]
|
||||
);
|
||||
if (isset($r['errors'])) return ['ok' => false, 'error' => $r['errors'][0]['message'] ?? 'Update failed'];
|
||||
return ['ok' => true];
|
||||
}
|
||||
|
||||
function vv_lldap_delete_user(string $id): array {
|
||||
$r = vv_lldap_gql(
|
||||
'mutation DeleteUser($userId: String!) { deleteUser(userId: $userId) { ok } }',
|
||||
['userId' => $id]
|
||||
);
|
||||
if (isset($r['errors'])) return ['ok' => false, 'error' => $r['errors'][0]['message'] ?? 'Delete failed'];
|
||||
return ['ok' => true];
|
||||
}
|
||||
|
||||
function vv_lldap_set_password(string $userId, string $password): array {
|
||||
$conf = vv_auth_conf();
|
||||
$token = vv_lldap_token();
|
||||
if (!$token) return ['ok' => false, 'error' => 'lldap auth failed'];
|
||||
|
||||
$ch = curl_init($conf['lldap_url'] . '/auth/admin/resetPassword');
|
||||
curl_setopt_array($ch, [
|
||||
CURLOPT_RETURNTRANSFER => true,
|
||||
CURLOPT_TIMEOUT => 10,
|
||||
CURLOPT_POST => true,
|
||||
CURLOPT_POSTFIELDS => json_encode(['userId' => $userId, 'password' => $password]),
|
||||
CURLOPT_HTTPHEADER => ['Content-Type: application/json', 'Authorization: Bearer ' . $token],
|
||||
]);
|
||||
$body = curl_exec($ch);
|
||||
$code = curl_getinfo($ch, CURLINFO_HTTP_CODE);
|
||||
curl_close($ch);
|
||||
if ($code >= 200 && $code < 300) return ['ok' => true];
|
||||
$err = json_decode($body ?: '{}', true)['message'] ?? "HTTP $code";
|
||||
return ['ok' => false, 'error' => $err];
|
||||
}
|
||||
|
||||
function vv_lldap_create_group(string $name): array {
|
||||
$r = vv_lldap_gql(
|
||||
'mutation CreateGroup($name: String!) { createGroup(name: $name) { id displayName } }',
|
||||
['name' => $name]
|
||||
);
|
||||
if (isset($r['errors'])) return ['ok' => false, 'error' => $r['errors'][0]['message'] ?? 'Create failed'];
|
||||
return ['ok' => true, 'group' => $r['data']['createGroup'] ?? []];
|
||||
}
|
||||
|
||||
function vv_lldap_delete_group(int $id): array {
|
||||
$r = vv_lldap_gql(
|
||||
'mutation DeleteGroup($groupId: Int!) { deleteGroup(groupId: $groupId) { ok } }',
|
||||
['groupId' => $id]
|
||||
);
|
||||
if (isset($r['errors'])) return ['ok' => false, 'error' => $r['errors'][0]['message'] ?? 'Delete failed'];
|
||||
return ['ok' => true];
|
||||
}
|
||||
|
||||
function vv_lldap_add_to_group(string $userId, int $groupId): array {
|
||||
$r = vv_lldap_gql(
|
||||
'mutation AddUserToGroup($userId: String!, $groupId: Int!) { addUserToGroup(userId: $userId, groupId: $groupId) { ok } }',
|
||||
['userId' => $userId, 'groupId' => $groupId]
|
||||
);
|
||||
if (isset($r['errors'])) return ['ok' => false, 'error' => $r['errors'][0]['message'] ?? 'Failed'];
|
||||
return ['ok' => true];
|
||||
}
|
||||
|
||||
function vv_lldap_remove_from_group(string $userId, int $groupId): array {
|
||||
$r = vv_lldap_gql(
|
||||
'mutation RemoveUserFromGroup($userId: String!, $groupId: Int!) { removeUserFromGroup(userId: $userId, groupId: $groupId) { ok } }',
|
||||
['userId' => $userId, 'groupId' => $groupId]
|
||||
);
|
||||
if (isset($r['errors'])) return ['ok' => false, 'error' => $r['errors'][0]['message'] ?? 'Failed'];
|
||||
return ['ok' => true];
|
||||
}
|
||||
|
||||
// ── Authelia ──────────────────────────────────────────────────────────────────
|
||||
|
||||
function vv_authelia_read_rules(): array {
|
||||
$conf = vv_auth_conf();
|
||||
$file = $conf['authelia_config'];
|
||||
if (!file_exists($file)) return ['ok' => false, 'error' => 'Config not found: ' . $file];
|
||||
|
||||
$py = "import yaml,json,sys\n"
|
||||
. "d=yaml.safe_load(open(sys.argv[1]))\n"
|
||||
. "ac=d.get('access_control',{})\n"
|
||||
. "print(json.dumps({'default_policy':ac.get('default_policy','deny'),'rules':ac.get('rules',[])}))\n";
|
||||
$tmp = '/tmp/vv_auth_rd_' . getmypid() . '.py';
|
||||
file_put_contents($tmp, $py);
|
||||
$out = shell_exec('python3 ' . escapeshellarg($tmp) . ' ' . escapeshellarg($file) . ' 2>/dev/null');
|
||||
@unlink($tmp);
|
||||
|
||||
if (!$out) return ['ok' => false, 'error' => 'Parse failed — python3 with PyYAML required'];
|
||||
$data = json_decode(trim($out), true);
|
||||
if (!$data) return ['ok' => false, 'error' => 'Invalid YAML response'];
|
||||
return ['ok' => true, 'default_policy' => $data['default_policy'], 'rules' => $data['rules']];
|
||||
}
|
||||
|
||||
function vv_authelia_write_rules(array $rules, string $defaultPolicy): array {
|
||||
$conf = vv_auth_conf();
|
||||
$file = $conf['authelia_config'];
|
||||
if (!file_exists($file)) return ['ok' => false, 'error' => 'Config not found: ' . $file];
|
||||
|
||||
$acJson = json_encode(['default_policy' => $defaultPolicy, 'rules' => $rules]);
|
||||
$py = <<<'PYEOF'
|
||||
import yaml, json, sys, re
|
||||
config_file = sys.argv[1]
|
||||
new_ac = json.loads(sys.argv[2])
|
||||
with open(config_file, 'r') as f:
|
||||
content = f.read()
|
||||
new_block = yaml.dump({'access_control': new_ac}, default_flow_style=False, allow_unicode=True, sort_keys=False)
|
||||
pattern = r'(?ms)^access_control:.*?(?=^[a-zA-Z#]|\Z)'
|
||||
if re.search(pattern, content):
|
||||
content = re.sub(pattern, new_block + '\n', content)
|
||||
else:
|
||||
content = content.rstrip('\n') + '\n\n' + new_block + '\n'
|
||||
with open(config_file, 'w') as f:
|
||||
f.write(content)
|
||||
print('ok')
|
||||
PYEOF;
|
||||
$tmp = '/tmp/vv_auth_wr_' . getmypid() . '.py';
|
||||
file_put_contents($tmp, $py);
|
||||
$out = shell_exec('python3 ' . escapeshellarg($tmp) . ' ' . escapeshellarg($file) . ' ' . escapeshellarg($acJson) . ' 2>&1');
|
||||
@unlink($tmp);
|
||||
|
||||
if (trim($out) !== 'ok') return ['ok' => false, 'error' => 'Write failed: ' . trim($out)];
|
||||
shell_exec('docker restart ' . escapeshellarg($conf['authelia_container']) . ' >/dev/null 2>&1 &');
|
||||
return ['ok' => true];
|
||||
}
|
||||
@@ -0,0 +1,418 @@
|
||||
<?php
|
||||
require_once __DIR__ . '/config.php';
|
||||
|
||||
// ── Config ────────────────────────────────────────────────────────────────────
|
||||
|
||||
function vv_auth_conf(): array {
|
||||
$v = vv_conf_vars();
|
||||
$host = strtoupper(vv_detect_host());
|
||||
return [
|
||||
'npm_url' => rtrim($v["{$host}_NPM_URL"] ?? 'http://localhost:81', '/'),
|
||||
'npm_user' => $v["{$host}_NPM_USER"] ?? '',
|
||||
'npm_pass' => $v["{$host}_NPM_PASS"] ?? '',
|
||||
'lldap_url' => rtrim($v["{$host}_LLDAP_URL"] ?? 'http://localhost:17170', '/'),
|
||||
'lldap_user' => $v["{$host}_LLDAP_USER"] ?? '',
|
||||
'lldap_pass' => $v["{$host}_LLDAP_PASS"] ?? '',
|
||||
'authelia_config' => $v["{$host}_AUTHELIA_CONFIG"] ?? '/mnt/user/appdata-Fallback/Critical-Data/Authelia/configuration.yml',
|
||||
'authelia_container' => $v["{$host}_AUTHELIA_CONTAINER"] ?? 'Authelia',
|
||||
'is_owner' => vv_is_owner(),
|
||||
];
|
||||
}
|
||||
|
||||
// ── NPM ───────────────────────────────────────────────────────────────────────
|
||||
|
||||
function vv_npm_token(): string {
|
||||
if (!session_id()) session_start();
|
||||
$conf = vv_auth_conf();
|
||||
$cached = $_SESSION['vv_npm_token'] ?? '';
|
||||
$expiry = $_SESSION['vv_npm_token_exp'] ?? 0;
|
||||
if ($cached && time() < $expiry) return $cached;
|
||||
|
||||
$resp = vv_npm_raw('POST', '/api/tokens', [
|
||||
'identity' => $conf['npm_user'],
|
||||
'secret' => $conf['npm_pass'],
|
||||
], '', $conf);
|
||||
$token = $resp['token'] ?? '';
|
||||
if ($token) {
|
||||
$_SESSION['vv_npm_token'] = $token;
|
||||
$_SESSION['vv_npm_token_exp'] = time() + 82800;
|
||||
}
|
||||
return $token;
|
||||
}
|
||||
|
||||
function vv_npm_raw(string $method, string $path, array $data, string $token, array $conf = []): array {
|
||||
if (!$conf) $conf = vv_auth_conf();
|
||||
$url = $conf['npm_url'] . $path;
|
||||
$headers = ['Content-Type: application/json', 'Accept: application/json'];
|
||||
if ($token) $headers[] = 'Authorization: Bearer ' . $token;
|
||||
|
||||
$ch = curl_init($url);
|
||||
curl_setopt_array($ch, [
|
||||
CURLOPT_RETURNTRANSFER => true,
|
||||
CURLOPT_TIMEOUT => 10,
|
||||
CURLOPT_HTTPHEADER => $headers,
|
||||
CURLOPT_CUSTOMREQUEST => $method,
|
||||
]);
|
||||
if ($data && in_array($method, ['POST', 'PUT'], true))
|
||||
curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($data));
|
||||
$body = curl_exec($ch);
|
||||
curl_close($ch);
|
||||
return json_decode($body ?: '{}', true) ?: [];
|
||||
}
|
||||
|
||||
function vv_npm_req(string $method, string $path, array $data = []): array {
|
||||
$token = vv_npm_token();
|
||||
if (!$token) return ['_err' => 'NPM auth failed — check credentials in host conf'];
|
||||
return vv_npm_raw($method, $path, $data, $token);
|
||||
}
|
||||
|
||||
function vv_npm_list_proxies(): array {
|
||||
$list = vv_npm_req('GET', '/api/nginx/proxy-hosts?expand=certificate');
|
||||
if (!is_array($list) || isset($list['_err']))
|
||||
return ['ok' => false, 'error' => $list['_err'] ?? 'Invalid response from NPM'];
|
||||
return ['ok' => true, 'proxies' => $list];
|
||||
}
|
||||
|
||||
function vv_npm_list_certs(): array {
|
||||
$list = vv_npm_req('GET', '/api/nginx/certificates');
|
||||
return is_array($list) ? $list : [];
|
||||
}
|
||||
|
||||
function vv_npm_create_proxy(array $data): array {
|
||||
$r = vv_npm_req('POST', '/api/nginx/proxy-hosts', $data);
|
||||
return isset($r['id']) ? ['ok' => true, 'proxy' => $r] : ['ok' => false, 'error' => $r['error'] ?? ($r['_err'] ?? 'Create failed')];
|
||||
}
|
||||
|
||||
function vv_npm_update_proxy(int $id, array $data): array {
|
||||
$r = vv_npm_req('PUT', "/api/nginx/proxy-hosts/$id", $data);
|
||||
return isset($r['id']) ? ['ok' => true, 'proxy' => $r] : ['ok' => false, 'error' => $r['error'] ?? ($r['_err'] ?? 'Update failed')];
|
||||
}
|
||||
|
||||
function vv_npm_delete_proxy(int $id): array {
|
||||
vv_npm_req('DELETE', "/api/nginx/proxy-hosts/$id");
|
||||
return ['ok' => true];
|
||||
}
|
||||
|
||||
function vv_npm_toggle_proxy(int $id, bool $enabled): array {
|
||||
vv_npm_req('POST', "/api/nginx/proxy-hosts/$id/" . ($enabled ? 'enable' : 'disable'));
|
||||
return ['ok' => true];
|
||||
}
|
||||
|
||||
// ── lldap ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
function vv_lldap_token(): string {
|
||||
if (!session_id()) session_start();
|
||||
$conf = vv_auth_conf();
|
||||
$cached = $_SESSION['vv_lldap_token'] ?? '';
|
||||
$expiry = $_SESSION['vv_lldap_token_exp'] ?? 0;
|
||||
if ($cached && time() < $expiry) return $cached;
|
||||
|
||||
$ch = curl_init($conf['lldap_url'] . '/auth/simple/login');
|
||||
curl_setopt_array($ch, [
|
||||
CURLOPT_RETURNTRANSFER => true,
|
||||
CURLOPT_TIMEOUT => 10,
|
||||
CURLOPT_POST => true,
|
||||
CURLOPT_POSTFIELDS => json_encode(['username' => $conf['lldap_user'], 'password' => $conf['lldap_pass']]),
|
||||
CURLOPT_HTTPHEADER => ['Content-Type: application/json'],
|
||||
]);
|
||||
$body = curl_exec($ch);
|
||||
curl_close($ch);
|
||||
$resp = json_decode($body ?: '{}', true) ?: [];
|
||||
$token = $resp['token'] ?? '';
|
||||
if ($token) {
|
||||
$_SESSION['vv_lldap_token'] = $token;
|
||||
$_SESSION['vv_lldap_token_exp'] = time() + 3500;
|
||||
}
|
||||
return $token;
|
||||
}
|
||||
|
||||
function vv_lldap_gql(string $query, array $variables = []): array {
|
||||
$conf = vv_auth_conf();
|
||||
$token = vv_lldap_token();
|
||||
if (!$token) return ['errors' => [['message' => 'lldap auth failed — check credentials']]];
|
||||
|
||||
$ch = curl_init($conf['lldap_url'] . '/api/graphql');
|
||||
curl_setopt_array($ch, [
|
||||
CURLOPT_RETURNTRANSFER => true,
|
||||
CURLOPT_TIMEOUT => 10,
|
||||
CURLOPT_POST => true,
|
||||
CURLOPT_POSTFIELDS => json_encode(['query' => $query, 'variables' => $variables]),
|
||||
CURLOPT_HTTPHEADER => ['Content-Type: application/json', 'Authorization: Bearer ' . $token],
|
||||
]);
|
||||
$body = curl_exec($ch);
|
||||
curl_close($ch);
|
||||
return json_decode($body ?: '{}', true) ?: [];
|
||||
}
|
||||
|
||||
function vv_lldap_list_users(): array {
|
||||
$r = vv_lldap_gql('query { users { id displayName email creationDate groups { id displayName } } }');
|
||||
if (isset($r['errors'])) return ['ok' => false, 'error' => $r['errors'][0]['message'] ?? 'Query failed'];
|
||||
return ['ok' => true, 'users' => $r['data']['users'] ?? []];
|
||||
}
|
||||
|
||||
function vv_lldap_list_groups(): array {
|
||||
$r = vv_lldap_gql('query { groups { id displayName users { id displayName } } }');
|
||||
if (isset($r['errors'])) return ['ok' => false, 'error' => $r['errors'][0]['message'] ?? 'Query failed'];
|
||||
return ['ok' => true, 'groups' => $r['data']['groups'] ?? []];
|
||||
}
|
||||
|
||||
function vv_lldap_create_user(string $id, string $email, string $displayName, string $password): array {
|
||||
$r = vv_lldap_gql(
|
||||
'mutation CreateUser($user: CreateUserInput!) { createUser(user: $user) { id displayName email } }',
|
||||
['user' => ['id' => $id, 'email' => $email, 'displayName' => $displayName]]
|
||||
);
|
||||
if (isset($r['errors'])) return ['ok' => false, 'error' => $r['errors'][0]['message'] ?? 'Create failed'];
|
||||
if ($password) vv_lldap_set_password($id, $password);
|
||||
return ['ok' => true, 'user' => $r['data']['createUser'] ?? []];
|
||||
}
|
||||
|
||||
function vv_lldap_update_user(string $id, string $email, string $displayName): array {
|
||||
$r = vv_lldap_gql(
|
||||
'mutation UpdateUser($user: UpdateUserInput!) { updateUser(user: $user) { ok } }',
|
||||
['user' => ['id' => $id, 'email' => $email, 'displayName' => $displayName]]
|
||||
);
|
||||
if (isset($r['errors'])) return ['ok' => false, 'error' => $r['errors'][0]['message'] ?? 'Update failed'];
|
||||
return ['ok' => true];
|
||||
}
|
||||
|
||||
function vv_lldap_delete_user(string $id): array {
|
||||
$r = vv_lldap_gql(
|
||||
'mutation DeleteUser($userId: String!) { deleteUser(userId: $userId) { ok } }',
|
||||
['userId' => $id]
|
||||
);
|
||||
if (isset($r['errors'])) return ['ok' => false, 'error' => $r['errors'][0]['message'] ?? 'Delete failed'];
|
||||
return ['ok' => true];
|
||||
}
|
||||
|
||||
function vv_lldap_set_password(string $userId, string $password): array {
|
||||
$conf = vv_auth_conf();
|
||||
$token = vv_lldap_token();
|
||||
if (!$token) return ['ok' => false, 'error' => 'lldap auth failed'];
|
||||
|
||||
$ch = curl_init($conf['lldap_url'] . '/auth/admin/resetPassword');
|
||||
curl_setopt_array($ch, [
|
||||
CURLOPT_RETURNTRANSFER => true,
|
||||
CURLOPT_TIMEOUT => 10,
|
||||
CURLOPT_POST => true,
|
||||
CURLOPT_POSTFIELDS => json_encode(['userId' => $userId, 'password' => $password]),
|
||||
CURLOPT_HTTPHEADER => ['Content-Type: application/json', 'Authorization: Bearer ' . $token],
|
||||
]);
|
||||
$body = curl_exec($ch);
|
||||
$code = curl_getinfo($ch, CURLINFO_HTTP_CODE);
|
||||
curl_close($ch);
|
||||
if ($code >= 200 && $code < 300) return ['ok' => true];
|
||||
$err = json_decode($body ?: '{}', true)['message'] ?? "HTTP $code";
|
||||
return ['ok' => false, 'error' => $err];
|
||||
}
|
||||
|
||||
function vv_lldap_create_group(string $name): array {
|
||||
$r = vv_lldap_gql(
|
||||
'mutation CreateGroup($name: String!) { createGroup(name: $name) { id displayName } }',
|
||||
['name' => $name]
|
||||
);
|
||||
if (isset($r['errors'])) return ['ok' => false, 'error' => $r['errors'][0]['message'] ?? 'Create failed'];
|
||||
return ['ok' => true, 'group' => $r['data']['createGroup'] ?? []];
|
||||
}
|
||||
|
||||
function vv_lldap_delete_group(int $id): array {
|
||||
$r = vv_lldap_gql(
|
||||
'mutation DeleteGroup($groupId: Int!) { deleteGroup(groupId: $groupId) { ok } }',
|
||||
['groupId' => $id]
|
||||
);
|
||||
if (isset($r['errors'])) return ['ok' => false, 'error' => $r['errors'][0]['message'] ?? 'Delete failed'];
|
||||
return ['ok' => true];
|
||||
}
|
||||
|
||||
function vv_lldap_add_to_group(string $userId, int $groupId): array {
|
||||
$r = vv_lldap_gql(
|
||||
'mutation AddUserToGroup($userId: String!, $groupId: Int!) { addUserToGroup(userId: $userId, groupId: $groupId) { ok } }',
|
||||
['userId' => $userId, 'groupId' => $groupId]
|
||||
);
|
||||
if (isset($r['errors'])) return ['ok' => false, 'error' => $r['errors'][0]['message'] ?? 'Failed'];
|
||||
return ['ok' => true];
|
||||
}
|
||||
|
||||
function vv_lldap_remove_from_group(string $userId, int $groupId): array {
|
||||
$r = vv_lldap_gql(
|
||||
'mutation RemoveUserFromGroup($userId: String!, $groupId: Int!) { removeUserFromGroup(userId: $userId, groupId: $groupId) { ok } }',
|
||||
['userId' => $userId, 'groupId' => $groupId]
|
||||
);
|
||||
if (isset($r['errors'])) return ['ok' => false, 'error' => $r['errors'][0]['message'] ?? 'Failed'];
|
||||
return ['ok' => true];
|
||||
}
|
||||
|
||||
// ── Authelia ──────────────────────────────────────────────────────────────────
|
||||
|
||||
function vv_authelia_read_rules(): array {
|
||||
$conf = vv_auth_conf();
|
||||
$file = $conf['authelia_config'];
|
||||
if (!file_exists($file)) return ['ok' => false, 'error' => 'Config not found: ' . $file];
|
||||
|
||||
$content = file_get_contents($file);
|
||||
if ($content === false) return ['ok' => false, 'error' => 'Cannot read config file'];
|
||||
|
||||
// Extract default_policy (strip inline comments)
|
||||
$defaultPolicy = 'deny';
|
||||
if (preg_match('/^[ \t]+default_policy:[ \t]+([a-z_]+)/m', $content, $m))
|
||||
$defaultPolicy = $m[1];
|
||||
|
||||
// Extract the indented block under access_control:
|
||||
if (!preg_match('/^access_control:[ \t]*\n((?:[ \t][^\n]*\n?)*)/m', $content, $m))
|
||||
return ['ok' => false, 'error' => 'access_control section not found'];
|
||||
|
||||
$acBlock = $m[1];
|
||||
|
||||
// Extract the indented block under rules: (3+ space indent = rule list items)
|
||||
if (!preg_match('/^ rules:[ \t]*\n((?:[ \t]{3,}[^\n]*\n?)*)/m', $acBlock, $m))
|
||||
return ['ok' => true, 'default_policy' => $defaultPolicy, 'rules' => []];
|
||||
|
||||
// Split into individual rule chunks at " - " (indent-4 rule starts)
|
||||
$chunks = preg_split('/(?=^ - )/m', $m[1]);
|
||||
$rules = [];
|
||||
foreach ($chunks as $chunk) {
|
||||
if (!preg_match('/^ - /', $chunk)) continue;
|
||||
$rule = vv_authelia_parse_rule_chunk($chunk);
|
||||
if (!empty($rule)) $rules[] = $rule;
|
||||
}
|
||||
|
||||
return ['ok' => true, 'default_policy' => $defaultPolicy, 'rules' => $rules];
|
||||
}
|
||||
|
||||
function vv_authelia_parse_rule_chunk(string $chunk): array {
|
||||
$rule = [];
|
||||
$field = null;
|
||||
$list = [];
|
||||
|
||||
$save = function () use (&$rule, &$field, &$list) {
|
||||
if ($field === null) return;
|
||||
if (!empty($list))
|
||||
$rule[$field] = count($list) === 1 ? $list[0] : $list;
|
||||
$field = null;
|
||||
$list = [];
|
||||
};
|
||||
|
||||
foreach (explode("\n", $chunk) as $line) {
|
||||
$raw = rtrim($line);
|
||||
$trim = trim($raw);
|
||||
if ($trim === '' || preg_match('/^#+/', $trim)) continue;
|
||||
$indent = strlen($raw) - strlen(ltrim($raw, ' '));
|
||||
|
||||
// indent=4, starts with "- " → first field of this rule block
|
||||
if ($indent === 4 && str_starts_with($trim, '- ')) {
|
||||
$rest = ltrim(substr($trim, 2));
|
||||
if (preg_match('/^([a-z_]+):[ \t]*(.*)$/', $rest, $m)) {
|
||||
$save();
|
||||
$field = $m[1];
|
||||
$val = trim($m[2]);
|
||||
if ($val !== '' && !str_starts_with($val, '#')) {
|
||||
$rule[$field] = vv_authelia_unquote($val);
|
||||
$field = null;
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// indent=6 → named field (scalar or list header)
|
||||
if ($indent === 6 && preg_match('/^([a-z_]+):[ \t]*(.*)$/', $trim, $m)) {
|
||||
$save();
|
||||
$field = $m[1];
|
||||
$val = trim($m[2]);
|
||||
if ($val !== '' && !str_starts_with($val, '#')) {
|
||||
$rule[$field] = vv_authelia_unquote($val);
|
||||
$field = null;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// indent=8, starts with "- " → list item under current field
|
||||
if ($indent === 8 && str_starts_with($trim, '- ')) {
|
||||
$list[] = vv_authelia_parse_list_item(trim(substr($trim, 2)));
|
||||
}
|
||||
}
|
||||
$save();
|
||||
return $rule;
|
||||
}
|
||||
|
||||
// Strip surrounding quotes and inline comments from a YAML scalar.
|
||||
function vv_authelia_unquote(string $val): string {
|
||||
$val = trim($val);
|
||||
$val = preg_replace('/\s+#[^"\']*$/', '', $val); // strip trailing comment
|
||||
if (preg_match('/^(["\'])(.+)\1$/', $val, $m)) return $m[2];
|
||||
return $val;
|
||||
}
|
||||
|
||||
// Parse a YAML list item: flow sequence ['group:name'] or plain/quoted scalar.
|
||||
function vv_authelia_parse_list_item(string $val): string {
|
||||
$val = trim($val);
|
||||
// Flow sequence: ['value'] or ["value"] or [value]
|
||||
if (preg_match('/^\[[\'""]?([^\]\'""]+)[\'""]?\]$/', $val, $m)) return trim($m[1]);
|
||||
return vv_authelia_unquote($val);
|
||||
}
|
||||
|
||||
function vv_authelia_write_rules(array $rules, string $defaultPolicy): array {
|
||||
$conf = vv_auth_conf();
|
||||
$file = $conf['authelia_config'];
|
||||
if (!file_exists($file)) return ['ok' => false, 'error' => 'Config not found: ' . $file];
|
||||
|
||||
$content = file_get_contents($file);
|
||||
if ($content === false) return ['ok' => false, 'error' => 'Cannot read config file'];
|
||||
|
||||
// Build the new access_control block
|
||||
$block = "access_control:\n";
|
||||
$block .= " default_policy: $defaultPolicy\n";
|
||||
$block .= " rules:\n";
|
||||
|
||||
// Preferred field output order
|
||||
$fieldOrder = ['domain', 'policy', 'subject', 'networks', 'resources'];
|
||||
|
||||
foreach ($rules as $rule) {
|
||||
$keys = array_merge(
|
||||
array_filter($fieldOrder, fn($k) => array_key_exists($k, $rule)),
|
||||
array_diff(array_keys($rule), $fieldOrder)
|
||||
);
|
||||
$first = true;
|
||||
foreach ($keys as $key) {
|
||||
if (!array_key_exists($key, $rule)) continue;
|
||||
$val = $rule[$key];
|
||||
$prefix = $first ? ' - ' : ' ';
|
||||
$first = false;
|
||||
|
||||
// domain, subject, resources, networks → always output as list
|
||||
$isList = in_array($key, ['domain', 'subject', 'resources', 'networks'], true);
|
||||
if ($isList) {
|
||||
$items = is_array($val) ? $val : [$val];
|
||||
$block .= $prefix . $key . ":\n";
|
||||
foreach ($items as $item) {
|
||||
$out = $key === 'subject'
|
||||
? "['" . $item . "']"
|
||||
: vv_authelia_yaml_scalar((string) $item);
|
||||
$block .= ' - ' . $out . "\n";
|
||||
}
|
||||
} else {
|
||||
$block .= $prefix . $key . ': ' . vv_authelia_yaml_scalar((string) $val) . "\n";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Replace existing access_control: block (from its line to next top-level key or EOF)
|
||||
$pattern = '/^access_control:[ \t]*\n(?:[ \t][^\n]*\n?)*/m';
|
||||
$new = preg_match($pattern, $content)
|
||||
? preg_replace($pattern, $block, $content, 1)
|
||||
: rtrim($content) . "\n\n" . $block;
|
||||
|
||||
if ($new === null) return ['ok' => false, 'error' => 'Regex replace failed'];
|
||||
|
||||
$tmp = $file . '.vv.tmp';
|
||||
if (file_put_contents($tmp, $new) === false) return ['ok' => false, 'error' => 'Write failed'];
|
||||
if (!rename($tmp, $file)) { @unlink($tmp); return ['ok' => false, 'error' => 'Atomic rename failed']; }
|
||||
|
||||
shell_exec('docker restart ' . escapeshellarg($conf['authelia_container']) . ' >/dev/null 2>&1 &');
|
||||
return ['ok' => true];
|
||||
}
|
||||
|
||||
// Quote a YAML scalar value if it contains characters that require quoting.
|
||||
function vv_authelia_yaml_scalar(string $val): string {
|
||||
if ($val === '' || preg_match('/[:#\[\]{},|>&*?!%@`\'"]/', $val) || preg_match('/^\s|\s$/', $val))
|
||||
return '"' . str_replace(['\\', '"'], ['\\\\', '\\"'], $val) . '"';
|
||||
return $val;
|
||||
}
|
||||
+586
@@ -0,0 +1,586 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ================================= Radarr Cleanup =============================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Delete orphaned movie files not tracked by Radarr. Queries the API for all
|
||||
# tracked movie file paths, walks the library on disk, and removes anything
|
||||
# untracked that is old enough to be past the import window. Triggers an Emby
|
||||
# library clean after each deletion run so ghost entries disappear immediately.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Every file encountered on disk is classified into one of five categories:
|
||||
#
|
||||
# TRACKED — Radarr API knows this exact path → leave it alone
|
||||
# PROTECTED — matches RADARR_PROTECTED_PATTERNS → never delete
|
||||
# ORPHAN — video file, not tracked, older than RADARR_ORPHAN_AGE → delete
|
||||
# JUNK — not a video extension, not protected → delete regardless of age
|
||||
# RECENT — not tracked, under RADARR_ORPHAN_AGE → skip (may be mid-import)
|
||||
#
|
||||
# Radarr generates movie artwork (*.jpg), metadata (*.nfo), and manages subtitles
|
||||
# (*.srt, *.sub, *.ass) but does NOT include these in its tracked file API response.
|
||||
# Without PROTECTED classification these would be deleted — breaking Radarr and
|
||||
# Emby metadata display.
|
||||
#
|
||||
# After deletions: notify_emby_scan() triggers Emby "Clean Missing Files" task.
|
||||
# Emby removes ghost entries immediately — no user-facing file-not-found errors.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Six gates — ALL must pass before any file is touched:
|
||||
# 1. Container running and not starting/unhealthy
|
||||
# 2. API reachable
|
||||
# 3. API version matches RADARR_VERSION_MAJOR in master.conf
|
||||
# 4. Movie count > 0
|
||||
# 5. Tracked file count > 0
|
||||
# 6. Deletion size < RADARR_MAX_DELETE_GB — or --i-know-what-im-doing required
|
||||
#
|
||||
# acquire_lock "wait" — large scans take time, wait for previous run to finish
|
||||
# jq + curl validation — exits if either tool missing
|
||||
# DOCKER_TIMEOUT — container checks protected against daemon hangs
|
||||
# notify_emby_scan() — triggers Emby clean after deletion
|
||||
# platform_require_cmd — notify script validated before use
|
||||
# Silent by default — orphans/junk warn(), clean library logs silently
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# host*.conf
|
||||
#
|
||||
# HOST*_RADARR_URL / HOST*_RADARR_API_KEY / HOST*_RADARR_MOVIES_ROOT
|
||||
# HOST*_RADARR_PATH_MAP — container path → host path translation
|
||||
# All aliased by detect_hosts() — script uses unprefixed names
|
||||
#
|
||||
# master.conf
|
||||
#
|
||||
# RADARR_ORPHAN_AGE — days before untracked file eligible for deletion
|
||||
# RADARR_MAX_DELETE_GB — require --i-know-what-im-doing above this
|
||||
# RADARR_EXTENSIONS — video file extensions for orphan classification
|
||||
# RADARR_PROTECTED_PATTERNS — file patterns never deleted
|
||||
# RADARR_VERSION_MAJOR — expected Radarr major version for API safety check
|
||||
# RADARR_IMPORT_SCAN_TIMEOUT — seconds to wait for pre-flight import scan (default 600)
|
||||
# ARR_CLEANUP_STATS — stats file path (read by coffee report)
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# radarr_cleanup.sh — normal run
|
||||
# radarr_cleanup.sh --dry-run — preview, no deletions
|
||||
# radarr_cleanup.sh --log — verbose output
|
||||
# radarr_cleanup.sh --status — show config and exit
|
||||
# radarr_cleanup.sh --i-know-what-im-doing — bypass size threshold
|
||||
# radarr_cleanup.sh --i-know-what-im-doing --skip-strike-list — NUCLEAR MODE
|
||||
#
|
||||
# NUCLEAR MODE: both flags bypass age check AND size threshold. User accepts full
|
||||
# responsibility — the flag name is long and annoying by design.
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
# ── Special flag pre-processing ───────────────────────────────────────────────────────────────
|
||||
I_KNOW=false
|
||||
SKIP_STRIKES=false
|
||||
FILTERED_ARGS=()
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--i-know-what-im-doing) I_KNOW=true ;;
|
||||
--skip-strike-list) SKIP_STRIKES=true ;;
|
||||
*) FILTERED_ARGS+=("$arg") ;;
|
||||
esac
|
||||
done
|
||||
|
||||
parse_args "${FILTERED_ARGS[@]}"
|
||||
|
||||
# ── Nuclear mode warning ──────────────────────────────────────────────────────────────────────
|
||||
if [[ "$I_KNOW" == true ]] && [[ "$SKIP_STRIKES" == true ]] && [[ "$DRY_RUN" != true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo "⚠️ WARNING — NUCLEAR MODE ACTIVE"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo " Flags: --i-know-what-im-doing --skip-strike-list"
|
||||
echo " Strike system: BYPASSED — deletes on first pass"
|
||||
echo " Size threshold: BYPASSED — no GB limit"
|
||||
echo " Data recovery: NOT POSSIBLE after deletion"
|
||||
echo ""
|
||||
echo " Review --dry-run output before proceeding."
|
||||
echo " You have 10 seconds to cancel (Ctrl+C)..."
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
sleep 10
|
||||
echo " Proceeding..."
|
||||
echo ""
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v curl >/dev/null 2>&1; then
|
||||
error "curl not found — required for Radarr API calls"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v jq >/dev/null 2>&1; then
|
||||
error "jq not found — required for JSON parsing"
|
||||
notify "Radarr cleanup failed on $(hostname) — jq not installed" "Radarr Cleanup" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
platform_require_cmd \
|
||||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||||
"" "" \
|
||||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||
|
||||
acquire_lock "wait"
|
||||
TMP_DIR="/tmp/radarr_cleanup_$$"
|
||||
mkdir -p "$TMP_DIR"
|
||||
trap "_release_all_locks; rm -rf $TMP_DIR" EXIT
|
||||
|
||||
if ! command -v docker &>/dev/null; then
|
||||
error "Docker command not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# detect_hosts() sets MY_ID and aliases RADARR_URL, RADARR_API_KEY, RADARR_MOVIES_ROOT
|
||||
detect_hosts
|
||||
|
||||
DOCKER_TIMEOUT=15
|
||||
RADARR_CONTAINER="Radarr"
|
||||
|
||||
# Build path map from MY_ID's Radarr path map
|
||||
declare -A ARR_PATH_MAP
|
||||
local_path_map_var="${MY_ID}_RADARR_PATH_MAP"
|
||||
eval "for key in \"\${!${local_path_map_var}[@]}\"; do
|
||||
ARR_PATH_MAP[\"\$key\"]=\"\${${local_path_map_var}[\$key]}\"
|
||||
done"
|
||||
|
||||
require_var RADARR_URL
|
||||
require_var RADARR_API_KEY
|
||||
require_var RADARR_MOVIES_ROOT
|
||||
|
||||
if [[ ! -d "$RADARR_MOVIES_ROOT" ]]; then
|
||||
error "Movies root not found: $RADARR_MOVIES_ROOT"
|
||||
notify "Radarr cleanup failed on $(hostname) — movies root not found: $RADARR_MOVIES_ROOT" \
|
||||
"Radarr Cleanup" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log "$ICON_GEAR Config: url=${RADARR_URL} root=${RADARR_MOVIES_ROOT}"
|
||||
echo " $MY_ID ($LOCAL_SERVER_NAME) — $RADARR_URL"
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be deleted"
|
||||
[[ "$I_KNOW" == true ]] && warn "OVERRIDE — --i-know-what-im-doing active"
|
||||
[[ "$SKIP_STRIKES" == true ]] && warn "OVERRIDE — --skip-strike-list active — age check bypassed"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_GEAR Radarr URL: $RADARR_URL"
|
||||
echo "$ICON_GEAR Movies root: $RADARR_MOVIES_ROOT"
|
||||
echo "$ICON_TIME Orphan age: ${RADARR_ORPHAN_AGE} days"
|
||||
echo "$ICON_GEAR Max delete: ${RADARR_MAX_DELETE_GB}GB (requires --i-know-what-im-doing)"
|
||||
echo "$ICON_GEAR Radarr ver: v${RADARR_VERSION_MAJOR} expected"
|
||||
echo "$ICON_GEAR Extensions: ${RADARR_EXTENSIONS[*]}"
|
||||
echo "$ICON_GEAR Protected patterns: ${RADARR_PROTECTED_PATTERNS[*]}"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "$ICON_GEAR I know: $I_KNOW"
|
||||
echo "$ICON_GEAR Skip strikes: $SKIP_STRIKES"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Safety Layer 1 — Container Health ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_SHIELD Safety Checks ━━━"
|
||||
|
||||
CONTAINER_RUNNING=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
|
||||
'{{.State.Running}}' "$RADARR_CONTAINER" 2>/dev/null)
|
||||
if [[ "$CONTAINER_RUNNING" != "true" ]]; then
|
||||
error "$RADARR_CONTAINER is not running — aborting"
|
||||
notify "Radarr cleanup aborted on $(hostname) — container not running" \
|
||||
"Radarr Cleanup" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
CONTAINER_HEALTH=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
|
||||
'{{.State.Health.Status}}' "$RADARR_CONTAINER" 2>/dev/null)
|
||||
case "$CONTAINER_HEALTH" in
|
||||
healthy) info "$RADARR_CONTAINER is healthy" ;;
|
||||
"") info "$RADARR_CONTAINER has no health check — proceeding" ;;
|
||||
starting)
|
||||
error "$RADARR_CONTAINER is still starting — aborting"
|
||||
notify "Radarr cleanup aborted on $(hostname) — container still starting" \
|
||||
"Radarr Cleanup" "warning"
|
||||
exit 1 ;;
|
||||
unhealthy)
|
||||
error "$RADARR_CONTAINER is unhealthy — aborting"
|
||||
notify "Radarr cleanup aborted on $(hostname) — container unhealthy" \
|
||||
"Radarr Cleanup" "warning"
|
||||
exit 1 ;;
|
||||
*) warn "$RADARR_CONTAINER health: $CONTAINER_HEALTH — proceeding with caution" ;;
|
||||
esac
|
||||
|
||||
info "Safety layer 1 passed — container healthy"
|
||||
|
||||
# ==============================================================================================
|
||||
# ── HELPER FUNCTIONS ──────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
radarr_api() {
|
||||
local endpoint="$1"
|
||||
local response http_code body
|
||||
|
||||
response=$(curl -sf \
|
||||
--max-time 30 \
|
||||
-H "X-Api-Key: $RADARR_API_KEY" \
|
||||
-w "\n%{http_code}" \
|
||||
"${RADARR_URL}/api/v3/${endpoint}" 2>/dev/null)
|
||||
|
||||
http_code=$(echo "$response" | tail -1)
|
||||
body=$(echo "$response" | head -n -1)
|
||||
|
||||
if [[ "$http_code" != "200" ]]; then
|
||||
error "Radarr API HTTP $http_code for: $endpoint"
|
||||
return 1
|
||||
fi
|
||||
echo "$body"
|
||||
}
|
||||
|
||||
is_video_file() {
|
||||
local ext="${1##*.}"
|
||||
ext="${ext,,}"
|
||||
for valid_ext in "${RADARR_EXTENSIONS[@]}"; do
|
||||
[[ "$ext" == "$valid_ext" ]] && return 0
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
is_protected_file() {
|
||||
local filename
|
||||
filename=$(basename "$1")
|
||||
for pattern in "${RADARR_PROTECTED_PATTERNS[@]}"; do
|
||||
# shellcheck disable=SC2254
|
||||
case "$filename" in
|
||||
$pattern) return 0 ;;
|
||||
esac
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
format_bytes() {
|
||||
local bytes=$1
|
||||
if (( bytes > 1073741824 )); then
|
||||
awk "BEGIN {printf \"%.1fGB\", $bytes / 1073741824}"
|
||||
elif (( bytes > 1048576 )); then
|
||||
awk "BEGIN {printf \"%.1fMB\", $bytes / 1048576}"
|
||||
else
|
||||
echo "${bytes}B"
|
||||
fi
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Pre-flight: Radarr Import Scan ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Pre-flight: Radarr Import Scan ━━━"
|
||||
|
||||
# Reverse-lookup container path from path map so Radarr gets its own path, not the host path
|
||||
RADARR_CONTAINER_ROOT=""
|
||||
for _cp in "${!ARR_PATH_MAP[@]}"; do
|
||||
if [[ "${ARR_PATH_MAP[$_cp]}" == "$RADARR_MOVIES_ROOT" ]]; then
|
||||
RADARR_CONTAINER_ROOT="$_cp"
|
||||
break
|
||||
fi
|
||||
done
|
||||
unset _cp
|
||||
|
||||
if [[ -n "$RADARR_CONTAINER_ROOT" ]]; then
|
||||
info "Triggering DownloadedMoviesScan on: $RADARR_CONTAINER_ROOT"
|
||||
SCAN_PAYLOAD="{\"name\": \"DownloadedMoviesScan\", \"path\": \"$RADARR_CONTAINER_ROOT\"}"
|
||||
else
|
||||
info "No path map match — triggering DownloadedMoviesScan (all root folders)"
|
||||
SCAN_PAYLOAD='{"name": "DownloadedMoviesScan"}'
|
||||
fi
|
||||
|
||||
SCAN_RESPONSE=$(curl -sf --max-time 30 -X POST \
|
||||
-H "X-Api-Key: $RADARR_API_KEY" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "$SCAN_PAYLOAD" \
|
||||
"${RADARR_URL}/api/v3/command" 2>/dev/null)
|
||||
|
||||
SCAN_CMD_ID=$(echo "$SCAN_RESPONSE" | jq -r '.id // empty' 2>/dev/null)
|
||||
|
||||
if [[ -z "$SCAN_CMD_ID" ]]; then
|
||||
warn "Could not trigger import scan — proceeding without pre-flight"
|
||||
else
|
||||
info "Import scan queued (command ID: $SCAN_CMD_ID) — waiting for completion..."
|
||||
POLL_TIMEOUT=${RADARR_IMPORT_SCAN_TIMEOUT:-600}
|
||||
POLLED=0
|
||||
while [[ "$POLLED" -lt "$POLL_TIMEOUT" ]]; do
|
||||
SCAN_STATUS=$(curl -sf --max-time 10 \
|
||||
-H "X-Api-Key: $RADARR_API_KEY" \
|
||||
"${RADARR_URL}/api/v3/command/${SCAN_CMD_ID}" 2>/dev/null | \
|
||||
jq -r '.status // empty' 2>/dev/null)
|
||||
case "$SCAN_STATUS" in
|
||||
completed) info "Import scan complete ✅"; break ;;
|
||||
failed) warn "Import scan reported failed — proceeding anyway"; break ;;
|
||||
esac
|
||||
sleep 10
|
||||
(( POLLED += 10 ))
|
||||
[[ $(( POLLED % 60 )) -eq 0 ]] && log " Still scanning... (${POLLED}s elapsed)"
|
||||
done
|
||||
[[ "$POLLED" -ge "$POLL_TIMEOUT" ]] && \
|
||||
warn "Import scan timed out after ${POLL_TIMEOUT}s — proceeding anyway"
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Fetch Radarr Tracked Files ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Fetching Radarr Tracked Files ━━━"
|
||||
|
||||
# Safety Layer 2 — API reachability
|
||||
if ! check_api "$RADARR_URL" "Radarr" 10; then
|
||||
notify "Radarr cleanup aborted on $(hostname) — API unreachable" "Radarr Cleanup" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Safety Layer 3 — API version check
|
||||
check_arr_version "$RADARR_URL" "$RADARR_API_KEY" "v3" "$RADARR_VERSION_MAJOR" "Radarr" || exit 1
|
||||
|
||||
info "Querying Radarr API..."
|
||||
|
||||
# Fetch all movies
|
||||
MOVIES_RESPONSE=$(radarr_api "movie") || {
|
||||
error "Failed to fetch movies from Radarr"
|
||||
notify "Radarr cleanup failed on $(hostname) — could not fetch movies" \
|
||||
"Radarr Cleanup" "warning"
|
||||
exit 1
|
||||
}
|
||||
|
||||
MOVIE_IDS=$(echo "$MOVIES_RESPONSE" | jq -r '.[].id' 2>/dev/null)
|
||||
MOVIE_COUNT=$(echo "$MOVIE_IDS" | grep -c "." 2>/dev/null || echo 0)
|
||||
|
||||
# Safety Layer 4 — movie count > 0
|
||||
if [[ "$MOVIE_COUNT" -eq 0 ]]; then
|
||||
error "API returned 0 movies — aborting to prevent mass deletion"
|
||||
notify "Radarr cleanup aborted on $(hostname) — 0 movies returned" \
|
||||
"Radarr Cleanup" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
info "Found $MOVIE_COUNT movies — fetching movie files..."
|
||||
|
||||
TRACKED_FILE="$TMP_DIR/tracked_paths.txt"
|
||||
> "$TRACKED_FILE"
|
||||
|
||||
MOVIE_INDEX=0
|
||||
while IFS= read -r movie_id; do
|
||||
[[ -z "$movie_id" ]] && continue
|
||||
(( MOVIE_INDEX++ ))
|
||||
[[ $(( MOVIE_INDEX % 100 )) -eq 0 ]] && \
|
||||
log "Fetching files: $MOVIE_INDEX/$MOVIE_COUNT movies..."
|
||||
MOVIE_FILES=$(radarr_api "moviefile?movieId=${movie_id}" 2>/dev/null)
|
||||
if [[ -n "$MOVIE_FILES" ]]; then
|
||||
while IFS= read -r api_path; do
|
||||
[[ -z "$api_path" ]] && continue
|
||||
translate_path "$api_path" >> "$TRACKED_FILE"
|
||||
done < <(echo "$MOVIE_FILES" | jq -r '.[].path // .path' 2>/dev/null)
|
||||
fi
|
||||
done <<< "$MOVIE_IDS"
|
||||
|
||||
sort -u "$TRACKED_FILE" -o "$TRACKED_FILE"
|
||||
|
||||
# Build in-memory lookup map — O(1) per lookup vs O(n) grep per file
|
||||
# Eliminates the main performance bottleneck for large libraries
|
||||
declare -A TRACKED_MAP
|
||||
while IFS= read -r _tracked_path; do
|
||||
[[ -n "$_tracked_path" ]] && TRACKED_MAP["$_tracked_path"]=1
|
||||
done < "$TRACKED_FILE"
|
||||
unset _tracked_path
|
||||
info "Built in-memory lookup map: ${#TRACKED_MAP[@]} tracked paths"
|
||||
TRACKED_COUNT=$(wc -l < "$TRACKED_FILE")
|
||||
|
||||
# Safety Layer 5 — tracked count > 0
|
||||
if [[ "$TRACKED_COUNT" -eq 0 ]]; then
|
||||
error "API returned 0 tracked files — aborting to prevent mass deletion"
|
||||
notify "Radarr cleanup aborted on $(hostname) — 0 tracked files returned" \
|
||||
"Radarr Cleanup" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
info "$MOVIE_COUNT movies | $TRACKED_COUNT tracked movie files"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Scan Movies Root ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_CLEAN Scanning Movies Root ━━━"
|
||||
info "Root: $RADARR_MOVIES_ROOT | Orphan age: ${RADARR_ORPHAN_AGE} days"
|
||||
|
||||
START=$(date +%s)
|
||||
ORPHAN_COUNT=0
|
||||
JUNK_COUNT=0
|
||||
RECENT_COUNT=0
|
||||
PROTECTED_COUNT=0
|
||||
ORPHAN_BYTES=0
|
||||
JUNK_BYTES=0
|
||||
|
||||
AGE_SECONDS=$(( RADARR_ORPHAN_AGE * 86400 ))
|
||||
NOW=$(date +%s)
|
||||
MAX_DELETE_BYTES=$(awk "BEGIN {printf \"%d\", $RADARR_MAX_DELETE_GB * 1073741824}")
|
||||
|
||||
while IFS= read -r filepath; do
|
||||
[[ -z "$filepath" ]] && continue
|
||||
|
||||
if [[ -n "${TRACKED_MAP[$filepath]:-}" ]]; then
|
||||
log "TRACKED: $filepath"
|
||||
continue
|
||||
fi
|
||||
|
||||
if is_protected_file "$filepath"; then
|
||||
log "$ICON_PROTECTED PROTECTED: $filepath"
|
||||
(( PROTECTED_COUNT++ ))
|
||||
continue
|
||||
fi
|
||||
|
||||
FILE_SIZE=$(stat -c%s "$filepath" 2>/dev/null || echo 0)
|
||||
|
||||
if is_video_file "$filepath"; then
|
||||
FILE_MTIME=$(stat -c %Y "$filepath" 2>/dev/null || echo 0)
|
||||
FILE_AGE=$(( NOW - FILE_MTIME ))
|
||||
|
||||
if [[ "$FILE_AGE" -lt "$AGE_SECONDS" ]] && [[ "$SKIP_STRIKES" != true ]]; then
|
||||
log "RECENT (skipping): $filepath"
|
||||
(( RECENT_COUNT++ ))
|
||||
continue
|
||||
fi
|
||||
|
||||
warn "$ICON_TRASH ORPHAN: $filepath"
|
||||
(( ORPHAN_COUNT++ ))
|
||||
ORPHAN_BYTES=$(( ORPHAN_BYTES + FILE_SIZE ))
|
||||
else
|
||||
log "JUNK: $filepath"
|
||||
(( JUNK_COUNT++ ))
|
||||
JUNK_BYTES=$(( JUNK_BYTES + FILE_SIZE ))
|
||||
fi
|
||||
|
||||
done < <(
|
||||
for host_path in "${ARR_PATH_MAP[@]}" "$RADARR_MOVIES_ROOT"; do
|
||||
[[ -d "$host_path" ]] && find "$host_path" -type f 2>/dev/null
|
||||
done | sort -u
|
||||
)
|
||||
|
||||
TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES ))
|
||||
TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT ))
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Safety Layer 6 — Deletion Size Threshold ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$TOTAL_DELETE_BYTES" -gt "$MAX_DELETE_BYTES" ]]; then
|
||||
TOTAL_HUMAN=$(awk "BEGIN {printf \"%.1fGB\", $TOTAL_DELETE_BYTES / 1073741824}")
|
||||
if [[ "$I_KNOW" != true ]]; then
|
||||
echo ""
|
||||
error "Deletion would exceed ${RADARR_MAX_DELETE_GB}GB — $TOTAL_HUMAN would be deleted"
|
||||
error "Review ORPHAN lines above carefully before proceeding"
|
||||
error "Rerun with: --i-know-what-im-doing"
|
||||
error "To also bypass age check: add --skip-strike-list"
|
||||
notify "Radarr cleanup halted on $(hostname) — ${TOTAL_HUMAN} requires --i-know-what-im-doing" \
|
||||
"Radarr Cleanup" "warning"
|
||||
exit 1
|
||||
else
|
||||
warn "OVERRIDE — deletion is $TOTAL_HUMAN — proceeding with --i-know-what-im-doing"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Execute Deletions ─────────────────────────────────────────────────────────────────────────
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
while IFS= read -r filepath; do
|
||||
[[ -z "$filepath" ]] && continue
|
||||
[[ -n "${TRACKED_MAP[$filepath]:-}" ]] && continue
|
||||
is_protected_file "$filepath" && continue
|
||||
|
||||
FILE_MTIME=$(stat -c %Y "$filepath" 2>/dev/null || echo 0)
|
||||
FILE_AGE=$(( NOW - FILE_MTIME ))
|
||||
|
||||
if is_video_file "$filepath"; then
|
||||
[[ "$FILE_AGE" -lt "$AGE_SECONDS" ]] && \
|
||||
[[ "$SKIP_STRIKES" != true ]] && continue
|
||||
fi
|
||||
|
||||
rm -f "$filepath" 2>/dev/null || error "Failed to delete: $filepath"
|
||||
|
||||
done < <(
|
||||
for host_path in "${ARR_PATH_MAP[@]}" "$RADARR_MOVIES_ROOT"; do
|
||||
[[ -d "$host_path" ]] && find "$host_path" -type f 2>/dev/null
|
||||
done | sort -u
|
||||
)
|
||||
|
||||
info "Cleaning up empty folders..."
|
||||
for host_path in "${ARR_PATH_MAP[@]}" "$RADARR_MOVIES_ROOT"; do
|
||||
[[ -d "$host_path" ]] && \
|
||||
find "$host_path" -mindepth 1 -type d -empty -delete 2>/dev/null
|
||||
done
|
||||
info "Empty folders removed"
|
||||
fi
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
ORPHAN_HUMAN=$(format_bytes "$ORPHAN_BYTES")
|
||||
JUNK_HUMAN=$(format_bytes "$JUNK_BYTES")
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY RADARR CLEANUP SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_SYNC Tracked: $TRACKED_COUNT files ($MOVIE_COUNT movies)"
|
||||
echo "$ICON_SHIELD Protected: $PROTECTED_COUNT files (artwork, subtitles, metadata)"
|
||||
echo "$ICON_TRASH Orphans: $ORPHAN_COUNT files ($ORPHAN_HUMAN)"
|
||||
echo "$ICON_TRASH Junk: $JUNK_COUNT files ($JUNK_HUMAN)"
|
||||
echo "$ICON_SKIP Recent skipped: $RECENT_COUNT files (under ${RADARR_ORPHAN_AGE} days)"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no files deleted"
|
||||
elif [[ "$TOTAL_REMOVED" -eq 0 ]]; then
|
||||
echo "$ICON_DONE Clean — nothing to remove"
|
||||
else
|
||||
warn "$ICON_DONE Removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)"
|
||||
notify "Radarr cleanup on $(hostname) — removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" \
|
||||
"Radarr Cleanup" "warning"
|
||||
# Notify Emby to clean missing files — removes ghost entries immediately
|
||||
notify_emby_scan
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
# Write stats for sunday_morning_coffee_report.sh
|
||||
if [[ "$DRY_RUN" == false ]] && [[ -n "${ARR_CLEANUP_STATS:-}" ]]; then
|
||||
echo "$(date '+%Y-%m-%d')|radarr|${ORPHAN_COUNT}|${ORPHAN_BYTES}|${JUNK_COUNT}|${JUNK_BYTES}|${RECENT_COUNT}|${TRACKED_COUNT}" \
|
||||
>> "$ARR_CLEANUP_STATS" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
exit 0
|
||||
+587
@@ -0,0 +1,587 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ================================= Radarr Cleanup =============================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Delete orphaned movie files not tracked by Radarr. Queries the API for all
|
||||
# tracked movie file paths, walks the library on disk, and removes anything
|
||||
# untracked that is old enough to be past the import window. Triggers an Emby
|
||||
# library clean after each deletion run so ghost entries disappear immediately.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Every file encountered on disk is classified into one of five categories:
|
||||
#
|
||||
# TRACKED — Radarr API knows this exact path → leave it alone
|
||||
# PROTECTED — matches RADARR_PROTECTED_PATTERNS → never delete
|
||||
# ORPHAN — video file, not tracked, older than RADARR_ORPHAN_AGE → delete
|
||||
# JUNK — not a video extension, not protected → delete regardless of age
|
||||
# RECENT — not tracked, under RADARR_ORPHAN_AGE → skip (may be mid-import)
|
||||
#
|
||||
# Radarr generates movie artwork (*.jpg), metadata (*.nfo), and manages subtitles
|
||||
# (*.srt, *.sub, *.ass) but does NOT include these in its tracked file API response.
|
||||
# Without PROTECTED classification these would be deleted — breaking Radarr and
|
||||
# Emby metadata display.
|
||||
#
|
||||
# After deletions: notify_emby_scan() triggers Emby "Clean Missing Files" task.
|
||||
# Emby removes ghost entries immediately — no user-facing file-not-found errors.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Six gates — ALL must pass before any file is touched:
|
||||
# 1. Container running and not starting/unhealthy
|
||||
# 2. API reachable
|
||||
# 3. API version matches RADARR_VERSION_MAJOR in master.conf
|
||||
# 4. Movie count > 0
|
||||
# 5. Tracked file count > 0
|
||||
# 6. Deletion size < RADARR_MAX_DELETE_GB — or --i-know-what-im-doing required
|
||||
#
|
||||
# acquire_lock "wait" — large scans take time, wait for previous run to finish
|
||||
# jq + curl validation — exits if either tool missing
|
||||
# DOCKER_TIMEOUT — container checks protected against daemon hangs
|
||||
# notify_emby_scan() — triggers Emby clean after deletion
|
||||
# platform_require_cmd — notify script validated before use
|
||||
# Silent by default — orphans/junk warn(), clean library logs silently
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# host*.conf
|
||||
#
|
||||
# HOST*_RADARR_URL / HOST*_RADARR_API_KEY / HOST*_RADARR_MOVIES_ROOT
|
||||
# HOST*_RADARR_PATH_MAP — container path → host path translation
|
||||
# All aliased by detect_hosts() — script uses unprefixed names
|
||||
#
|
||||
# master.conf
|
||||
#
|
||||
# RADARR_ORPHAN_AGE — days before untracked file eligible for deletion
|
||||
# RADARR_MAX_DELETE_GB — require --i-know-what-im-doing above this
|
||||
# RADARR_EXTENSIONS — video file extensions for orphan classification
|
||||
# RADARR_PROTECTED_PATTERNS — file patterns never deleted
|
||||
# RADARR_VERSION_MAJOR — expected Radarr major version for API safety check
|
||||
# RADARR_IMPORT_SCAN_TIMEOUT — seconds to wait for pre-flight import scan (default 600)
|
||||
# ARR_CLEANUP_STATS — stats file path (read by coffee report)
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# radarr_cleanup.sh — normal run
|
||||
# radarr_cleanup.sh --dry-run — preview, no deletions
|
||||
# radarr_cleanup.sh --log — verbose output
|
||||
# radarr_cleanup.sh --status — show config and exit
|
||||
# radarr_cleanup.sh --i-know-what-im-doing — bypass size threshold
|
||||
# radarr_cleanup.sh --i-know-what-im-doing --skip-strike-list — NUCLEAR MODE
|
||||
#
|
||||
# NUCLEAR MODE: both flags bypass age check AND size threshold. User accepts full
|
||||
# responsibility — the flag name is long and annoying by design.
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
# ── Special flag pre-processing ───────────────────────────────────────────────────────────────
|
||||
I_KNOW=false
|
||||
SKIP_STRIKES=false
|
||||
FILTERED_ARGS=()
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--i-know-what-im-doing) I_KNOW=true ;;
|
||||
--skip-strike-list) SKIP_STRIKES=true ;;
|
||||
*) FILTERED_ARGS+=("$arg") ;;
|
||||
esac
|
||||
done
|
||||
|
||||
parse_args "${FILTERED_ARGS[@]}"
|
||||
|
||||
# ── Nuclear mode warning ──────────────────────────────────────────────────────────────────────
|
||||
if [[ "$I_KNOW" == true ]] && [[ "$SKIP_STRIKES" == true ]] && [[ "$DRY_RUN" != true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo "⚠️ WARNING — NUCLEAR MODE ACTIVE"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo " Flags: --i-know-what-im-doing --skip-strike-list"
|
||||
echo " Strike system: BYPASSED — deletes on first pass"
|
||||
echo " Size threshold: BYPASSED — no GB limit"
|
||||
echo " Data recovery: NOT POSSIBLE after deletion"
|
||||
echo ""
|
||||
echo " Review --dry-run output before proceeding."
|
||||
echo " You have 10 seconds to cancel (Ctrl+C)..."
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
sleep 10
|
||||
echo " Proceeding..."
|
||||
echo ""
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v curl >/dev/null 2>&1; then
|
||||
error "curl not found — required for Radarr API calls"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v jq >/dev/null 2>&1; then
|
||||
error "jq not found — required for JSON parsing"
|
||||
notify "Radarr cleanup failed on $(hostname) — jq not installed" "Radarr Cleanup" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
platform_require_cmd \
|
||||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||||
"" "" \
|
||||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||
|
||||
acquire_lock "wait"
|
||||
TMP_DIR="/tmp/radarr_cleanup_$$"
|
||||
mkdir -p "$TMP_DIR"
|
||||
trap "_release_all_locks; rm -rf $TMP_DIR" EXIT
|
||||
|
||||
if ! command -v docker &>/dev/null; then
|
||||
error "Docker command not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# detect_hosts() sets MY_ID and aliases RADARR_URL, RADARR_API_KEY, RADARR_MOVIES_ROOT
|
||||
detect_hosts
|
||||
|
||||
DOCKER_TIMEOUT=15
|
||||
RADARR_CONTAINER="Radarr"
|
||||
|
||||
# Build path map from MY_ID's Radarr path map
|
||||
declare -A ARR_PATH_MAP
|
||||
local_path_map_var="${MY_ID}_RADARR_PATH_MAP"
|
||||
eval "for key in \"\${!${local_path_map_var}[@]}\"; do
|
||||
ARR_PATH_MAP[\"\$key\"]=\"\${${local_path_map_var}[\$key]}\"
|
||||
done"
|
||||
|
||||
require_var RADARR_URL
|
||||
require_var RADARR_API_KEY
|
||||
require_var RADARR_MOVIES_ROOT
|
||||
|
||||
if [[ ! -d "$RADARR_MOVIES_ROOT" ]]; then
|
||||
error "Movies root not found: $RADARR_MOVIES_ROOT"
|
||||
notify "Radarr cleanup failed on $(hostname) — movies root not found: $RADARR_MOVIES_ROOT" \
|
||||
"Radarr Cleanup" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log "$ICON_GEAR Config: url=${RADARR_URL} root=${RADARR_MOVIES_ROOT}"
|
||||
echo " $MY_ID ($LOCAL_SERVER_NAME) — $RADARR_URL"
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be deleted"
|
||||
[[ "$I_KNOW" == true ]] && warn "OVERRIDE — --i-know-what-im-doing active"
|
||||
[[ "$SKIP_STRIKES" == true ]] && warn "OVERRIDE — --skip-strike-list active — age check bypassed"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_GEAR Radarr URL: $RADARR_URL"
|
||||
echo "$ICON_GEAR Movies root: $RADARR_MOVIES_ROOT"
|
||||
echo "$ICON_TIME Orphan age: ${RADARR_ORPHAN_AGE} days"
|
||||
echo "$ICON_GEAR Max delete: ${RADARR_MAX_DELETE_GB}GB (requires --i-know-what-im-doing)"
|
||||
echo "$ICON_GEAR Radarr ver: v${RADARR_VERSION_MAJOR} expected"
|
||||
echo "$ICON_GEAR Extensions: ${RADARR_EXTENSIONS[*]}"
|
||||
echo "$ICON_GEAR Protected patterns: ${RADARR_PROTECTED_PATTERNS[*]}"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "$ICON_GEAR I know: $I_KNOW"
|
||||
echo "$ICON_GEAR Skip strikes: $SKIP_STRIKES"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Safety Layer 1 — Container Health ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_SHIELD Safety Checks ━━━"
|
||||
|
||||
CONTAINER_RUNNING=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
|
||||
'{{.State.Running}}' "$RADARR_CONTAINER" 2>/dev/null)
|
||||
if [[ "$CONTAINER_RUNNING" != "true" ]]; then
|
||||
error "$RADARR_CONTAINER is not running — aborting"
|
||||
notify "Radarr cleanup aborted on $(hostname) — container not running" \
|
||||
"Radarr Cleanup" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
CONTAINER_HEALTH=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
|
||||
'{{.State.Health.Status}}' "$RADARR_CONTAINER" 2>/dev/null)
|
||||
case "$CONTAINER_HEALTH" in
|
||||
healthy) info "$RADARR_CONTAINER is healthy" ;;
|
||||
"") info "$RADARR_CONTAINER has no health check — proceeding" ;;
|
||||
starting)
|
||||
error "$RADARR_CONTAINER is still starting — aborting"
|
||||
notify "Radarr cleanup aborted on $(hostname) — container still starting" \
|
||||
"Radarr Cleanup" "warning"
|
||||
exit 1 ;;
|
||||
unhealthy)
|
||||
error "$RADARR_CONTAINER is unhealthy — aborting"
|
||||
notify "Radarr cleanup aborted on $(hostname) — container unhealthy" \
|
||||
"Radarr Cleanup" "warning"
|
||||
exit 1 ;;
|
||||
*) warn "$RADARR_CONTAINER health: $CONTAINER_HEALTH — proceeding with caution" ;;
|
||||
esac
|
||||
|
||||
info "Safety layer 1 passed — container healthy"
|
||||
|
||||
# ==============================================================================================
|
||||
# ── HELPER FUNCTIONS ──────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
radarr_api() {
|
||||
local endpoint="$1"
|
||||
local response http_code body
|
||||
|
||||
response=$(curl -sf \
|
||||
--max-time 30 \
|
||||
-H "X-Api-Key: $RADARR_API_KEY" \
|
||||
-w "\n%{http_code}" \
|
||||
"${RADARR_URL}/api/v3/${endpoint}" 2>/dev/null)
|
||||
|
||||
http_code=$(echo "$response" | tail -1)
|
||||
body=$(echo "$response" | head -n -1)
|
||||
|
||||
if [[ "$http_code" != "200" ]]; then
|
||||
error "Radarr API HTTP $http_code for: $endpoint"
|
||||
return 1
|
||||
fi
|
||||
echo "$body"
|
||||
}
|
||||
|
||||
is_video_file() {
|
||||
local ext="${1##*.}"
|
||||
ext="${ext,,}"
|
||||
for valid_ext in "${RADARR_EXTENSIONS[@]}"; do
|
||||
[[ "$ext" == "$valid_ext" ]] && return 0
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
is_protected_file() {
|
||||
local filename
|
||||
filename=$(basename "$1")
|
||||
for pattern in "${RADARR_PROTECTED_PATTERNS[@]}"; do
|
||||
# shellcheck disable=SC2254
|
||||
case "$filename" in
|
||||
$pattern) return 0 ;;
|
||||
esac
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
format_bytes() {
|
||||
local bytes=$1
|
||||
if (( bytes > 1073741824 )); then
|
||||
awk "BEGIN {printf \"%.1fGB\", $bytes / 1073741824}"
|
||||
elif (( bytes > 1048576 )); then
|
||||
awk "BEGIN {printf \"%.1fMB\", $bytes / 1048576}"
|
||||
else
|
||||
echo "${bytes}B"
|
||||
fi
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Pre-flight: Radarr Import Scan ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Pre-flight: Radarr Import Scan ━━━"
|
||||
|
||||
# Fetch root folders from Radarr API and translate container paths to host paths
|
||||
mapfile -t SCAN_ROOTS < <(
|
||||
radarr_api "rootfolder" | \
|
||||
jq -r '.[].path' 2>/dev/null | \
|
||||
while IFS= read -r cp; do translate_path "$cp"; done
|
||||
)
|
||||
|
||||
if [[ "${#SCAN_ROOTS[@]}" -eq 0 ]]; then
|
||||
error "No root folders returned from Radarr API — aborting"
|
||||
notify "Radarr cleanup aborted on $(hostname) — no root folders from API" \
|
||||
"Radarr Cleanup" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
info "Scan targets (${#SCAN_ROOTS[@]}): ${SCAN_ROOTS[*]}"
|
||||
|
||||
info "Triggering DownloadedMoviesScan (all root folders)"
|
||||
SCAN_PAYLOAD='{"name": "DownloadedMoviesScan"}'
|
||||
|
||||
SCAN_RESPONSE=$(curl -sf --max-time 30 -X POST \
|
||||
-H "X-Api-Key: $RADARR_API_KEY" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "$SCAN_PAYLOAD" \
|
||||
"${RADARR_URL}/api/v3/command" 2>/dev/null)
|
||||
|
||||
SCAN_CMD_ID=$(echo "$SCAN_RESPONSE" | jq -r '.id // empty' 2>/dev/null)
|
||||
|
||||
if [[ -z "$SCAN_CMD_ID" ]]; then
|
||||
warn "Could not trigger import scan — proceeding without pre-flight"
|
||||
else
|
||||
info "Import scan queued (command ID: $SCAN_CMD_ID) — waiting for completion..."
|
||||
POLL_TIMEOUT=${RADARR_IMPORT_SCAN_TIMEOUT:-600}
|
||||
POLLED=0
|
||||
while [[ "$POLLED" -lt "$POLL_TIMEOUT" ]]; do
|
||||
SCAN_STATUS=$(curl -sf --max-time 10 \
|
||||
-H "X-Api-Key: $RADARR_API_KEY" \
|
||||
"${RADARR_URL}/api/v3/command/${SCAN_CMD_ID}" 2>/dev/null | \
|
||||
jq -r '.status // empty' 2>/dev/null)
|
||||
case "$SCAN_STATUS" in
|
||||
completed) info "Import scan complete ✅"; break ;;
|
||||
failed) warn "Import scan reported failed — proceeding anyway"; break ;;
|
||||
esac
|
||||
sleep 10
|
||||
(( POLLED += 10 ))
|
||||
[[ $(( POLLED % 60 )) -eq 0 ]] && log " Still scanning... (${POLLED}s elapsed)"
|
||||
done
|
||||
[[ "$POLLED" -ge "$POLL_TIMEOUT" ]] && \
|
||||
warn "Import scan timed out after ${POLL_TIMEOUT}s — proceeding anyway"
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Fetch Radarr Tracked Files ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Fetching Radarr Tracked Files ━━━"
|
||||
|
||||
# Safety Layer 2 — API reachability
|
||||
if ! check_api "$RADARR_URL" "Radarr" 10; then
|
||||
notify "Radarr cleanup aborted on $(hostname) — API unreachable" "Radarr Cleanup" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Safety Layer 3 — API version check
|
||||
check_arr_version "$RADARR_URL" "$RADARR_API_KEY" "v3" "$RADARR_VERSION_MAJOR" "Radarr" || exit 1
|
||||
|
||||
info "Querying Radarr API..."
|
||||
|
||||
# Fetch all movies
|
||||
MOVIES_RESPONSE=$(radarr_api "movie") || {
|
||||
error "Failed to fetch movies from Radarr"
|
||||
notify "Radarr cleanup failed on $(hostname) — could not fetch movies" \
|
||||
"Radarr Cleanup" "warning"
|
||||
exit 1
|
||||
}
|
||||
|
||||
MOVIE_IDS=$(echo "$MOVIES_RESPONSE" | jq -r '.[].id' 2>/dev/null)
|
||||
MOVIE_COUNT=$(echo "$MOVIE_IDS" | grep -c "." 2>/dev/null || echo 0)
|
||||
|
||||
# Safety Layer 4 — movie count > 0
|
||||
if [[ "$MOVIE_COUNT" -eq 0 ]]; then
|
||||
error "API returned 0 movies — aborting to prevent mass deletion"
|
||||
notify "Radarr cleanup aborted on $(hostname) — 0 movies returned" \
|
||||
"Radarr Cleanup" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
info "Found $MOVIE_COUNT movies — fetching movie files..."
|
||||
|
||||
TRACKED_FILE="$TMP_DIR/tracked_paths.txt"
|
||||
> "$TRACKED_FILE"
|
||||
|
||||
MOVIE_INDEX=0
|
||||
while IFS= read -r movie_id; do
|
||||
[[ -z "$movie_id" ]] && continue
|
||||
(( MOVIE_INDEX++ ))
|
||||
[[ $(( MOVIE_INDEX % 100 )) -eq 0 ]] && \
|
||||
log "Fetching files: $MOVIE_INDEX/$MOVIE_COUNT movies..."
|
||||
MOVIE_FILES=$(radarr_api "moviefile?movieId=${movie_id}" 2>/dev/null)
|
||||
if [[ -n "$MOVIE_FILES" ]]; then
|
||||
while IFS= read -r api_path; do
|
||||
[[ -z "$api_path" ]] && continue
|
||||
translate_path "$api_path" >> "$TRACKED_FILE"
|
||||
done < <(echo "$MOVIE_FILES" | jq -r '.[].path // .path' 2>/dev/null)
|
||||
fi
|
||||
done <<< "$MOVIE_IDS"
|
||||
|
||||
sort -u "$TRACKED_FILE" -o "$TRACKED_FILE"
|
||||
|
||||
# Build in-memory lookup map — O(1) per lookup vs O(n) grep per file
|
||||
# Eliminates the main performance bottleneck for large libraries
|
||||
declare -A TRACKED_MAP
|
||||
while IFS= read -r _tracked_path; do
|
||||
[[ -n "$_tracked_path" ]] && TRACKED_MAP["$_tracked_path"]=1
|
||||
done < "$TRACKED_FILE"
|
||||
unset _tracked_path
|
||||
info "Built in-memory lookup map: ${#TRACKED_MAP[@]} tracked paths"
|
||||
TRACKED_COUNT=$(wc -l < "$TRACKED_FILE")
|
||||
|
||||
# Safety Layer 5 — tracked count > 0
|
||||
if [[ "$TRACKED_COUNT" -eq 0 ]]; then
|
||||
error "API returned 0 tracked files — aborting to prevent mass deletion"
|
||||
notify "Radarr cleanup aborted on $(hostname) — 0 tracked files returned" \
|
||||
"Radarr Cleanup" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
info "$MOVIE_COUNT movies | $TRACKED_COUNT tracked movie files"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Scan Movies Root ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_CLEAN Scanning Movies Root ━━━"
|
||||
info "Root: $RADARR_MOVIES_ROOT | Orphan age: ${RADARR_ORPHAN_AGE} days"
|
||||
|
||||
START=$(date +%s)
|
||||
ORPHAN_COUNT=0
|
||||
JUNK_COUNT=0
|
||||
RECENT_COUNT=0
|
||||
PROTECTED_COUNT=0
|
||||
ORPHAN_BYTES=0
|
||||
JUNK_BYTES=0
|
||||
|
||||
AGE_SECONDS=$(( RADARR_ORPHAN_AGE * 86400 ))
|
||||
NOW=$(date +%s)
|
||||
MAX_DELETE_BYTES=$(awk "BEGIN {printf \"%d\", $RADARR_MAX_DELETE_GB * 1073741824}")
|
||||
|
||||
while IFS= read -r filepath; do
|
||||
[[ -z "$filepath" ]] && continue
|
||||
|
||||
if [[ -n "${TRACKED_MAP[$filepath]:-}" ]]; then
|
||||
log "TRACKED: $filepath"
|
||||
continue
|
||||
fi
|
||||
|
||||
if is_protected_file "$filepath"; then
|
||||
log "$ICON_PROTECTED PROTECTED: $filepath"
|
||||
(( PROTECTED_COUNT++ ))
|
||||
continue
|
||||
fi
|
||||
|
||||
FILE_SIZE=$(stat -c%s "$filepath" 2>/dev/null || echo 0)
|
||||
|
||||
if is_video_file "$filepath"; then
|
||||
FILE_MTIME=$(stat -c %Y "$filepath" 2>/dev/null || echo 0)
|
||||
FILE_AGE=$(( NOW - FILE_MTIME ))
|
||||
|
||||
if [[ "$FILE_AGE" -lt "$AGE_SECONDS" ]] && [[ "$SKIP_STRIKES" != true ]]; then
|
||||
log "RECENT (skipping): $filepath"
|
||||
(( RECENT_COUNT++ ))
|
||||
continue
|
||||
fi
|
||||
|
||||
warn "$ICON_TRASH ORPHAN: $filepath"
|
||||
(( ORPHAN_COUNT++ ))
|
||||
ORPHAN_BYTES=$(( ORPHAN_BYTES + FILE_SIZE ))
|
||||
else
|
||||
log "JUNK: $filepath"
|
||||
(( JUNK_COUNT++ ))
|
||||
JUNK_BYTES=$(( JUNK_BYTES + FILE_SIZE ))
|
||||
fi
|
||||
|
||||
done < <(
|
||||
for host_path in "${SCAN_ROOTS[@]}"; do
|
||||
[[ -d "$host_path" ]] && find "$host_path" -type f 2>/dev/null
|
||||
done | sort -u
|
||||
)
|
||||
|
||||
TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES ))
|
||||
TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT ))
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Safety Layer 6 — Deletion Size Threshold ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$TOTAL_DELETE_BYTES" -gt "$MAX_DELETE_BYTES" ]]; then
|
||||
TOTAL_HUMAN=$(awk "BEGIN {printf \"%.1fGB\", $TOTAL_DELETE_BYTES / 1073741824}")
|
||||
if [[ "$I_KNOW" != true ]]; then
|
||||
echo ""
|
||||
error "Deletion would exceed ${RADARR_MAX_DELETE_GB}GB — $TOTAL_HUMAN would be deleted"
|
||||
error "Review ORPHAN lines above carefully before proceeding"
|
||||
error "Rerun with: --i-know-what-im-doing"
|
||||
error "To also bypass age check: add --skip-strike-list"
|
||||
notify "Radarr cleanup halted on $(hostname) — ${TOTAL_HUMAN} requires --i-know-what-im-doing" \
|
||||
"Radarr Cleanup" "warning"
|
||||
exit 1
|
||||
else
|
||||
warn "OVERRIDE — deletion is $TOTAL_HUMAN — proceeding with --i-know-what-im-doing"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Execute Deletions ─────────────────────────────────────────────────────────────────────────
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
while IFS= read -r filepath; do
|
||||
[[ -z "$filepath" ]] && continue
|
||||
[[ -n "${TRACKED_MAP[$filepath]:-}" ]] && continue
|
||||
is_protected_file "$filepath" && continue
|
||||
|
||||
FILE_MTIME=$(stat -c %Y "$filepath" 2>/dev/null || echo 0)
|
||||
FILE_AGE=$(( NOW - FILE_MTIME ))
|
||||
|
||||
if is_video_file "$filepath"; then
|
||||
[[ "$FILE_AGE" -lt "$AGE_SECONDS" ]] && \
|
||||
[[ "$SKIP_STRIKES" != true ]] && continue
|
||||
fi
|
||||
|
||||
rm -f "$filepath" 2>/dev/null || error "Failed to delete: $filepath"
|
||||
|
||||
done < <(
|
||||
for host_path in "${SCAN_ROOTS[@]}"; do
|
||||
[[ -d "$host_path" ]] && find "$host_path" -type f 2>/dev/null
|
||||
done | sort -u
|
||||
)
|
||||
|
||||
info "Cleaning up empty folders..."
|
||||
for host_path in "${SCAN_ROOTS[@]}"; do
|
||||
[[ -d "$host_path" ]] && \
|
||||
find "$host_path" -mindepth 1 -type d -empty -delete 2>/dev/null
|
||||
done
|
||||
info "Empty folders removed"
|
||||
fi
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
ORPHAN_HUMAN=$(format_bytes "$ORPHAN_BYTES")
|
||||
JUNK_HUMAN=$(format_bytes "$JUNK_BYTES")
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY RADARR CLEANUP SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_SYNC Tracked: $TRACKED_COUNT files ($MOVIE_COUNT movies)"
|
||||
echo "$ICON_SHIELD Protected: $PROTECTED_COUNT files (artwork, subtitles, metadata)"
|
||||
echo "$ICON_TRASH Orphans: $ORPHAN_COUNT files ($ORPHAN_HUMAN)"
|
||||
echo "$ICON_TRASH Junk: $JUNK_COUNT files ($JUNK_HUMAN)"
|
||||
echo "$ICON_SKIP Recent skipped: $RECENT_COUNT files (under ${RADARR_ORPHAN_AGE} days)"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no files deleted"
|
||||
elif [[ "$TOTAL_REMOVED" -eq 0 ]]; then
|
||||
echo "$ICON_DONE Clean — nothing to remove"
|
||||
else
|
||||
warn "$ICON_DONE Removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)"
|
||||
notify "Radarr cleanup on $(hostname) — removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" \
|
||||
"Radarr Cleanup" "warning"
|
||||
# Notify Emby to clean missing files — removes ghost entries immediately
|
||||
notify_emby_scan
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
# Write stats for sunday_morning_coffee_report.sh
|
||||
if [[ "$DRY_RUN" == false ]] && [[ -n "${ARR_CLEANUP_STATS:-}" ]]; then
|
||||
echo "$(date '+%Y-%m-%d')|radarr|${ORPHAN_COUNT}|${ORPHAN_BYTES}|${JUNK_COUNT}|${JUNK_BYTES}|${RECENT_COUNT}|${TRACKED_COUNT}" \
|
||||
>> "$ARR_CLEANUP_STATS" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
exit 0
|
||||
@@ -0,0 +1,771 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ========================== HOST1 CONFIGURATION — unRAID-Gmer4Lfe ============================
|
||||
# ==============================================================================================
|
||||
# HOST1-specific variables — credentials, container names, share paths, failover lists.
|
||||
# Sourced after master.conf — values here extend shared profile arrays and add HOST1-specific
|
||||
# identity, credentials, and container configuration.
|
||||
#
|
||||
# Sparse checkout (git) ensures HOST2 never receives this file.
|
||||
# HOST2 never sees HOST1 credentials — clean separation at the file level.
|
||||
#
|
||||
# DO NOT put shared config here — thresholds, toggles, profiles belong in master.conf.
|
||||
# DO NOT put HOST2 variables here — they belong in host2.conf.
|
||||
#
|
||||
# ── INDEX ─────────────────────────────────────────────────────────────────────────────────────
|
||||
#
|
||||
# ── IDENTITY & CONNECTIVITY ────────────────────────────────────────────────────────────────
|
||||
# IDENTITY hostname, SSH key
|
||||
# EMBY container name, URL, API key
|
||||
# NOTIFICATIONS Discord webhook
|
||||
# PARTNERSHIP auth containers, backup paths
|
||||
#
|
||||
# ── RSYNC ──────────────────────────────────────────────────────────────────────────────────
|
||||
# DAILY SYNC SHARES media shares HOST1 owns and pushes to HOST2
|
||||
# WEEKLY SYNC SHARES appdata shares synced weekly (Sunday 2:30am)
|
||||
# CRITICAL SYNC SHARES appdata shares synced every 30 minutes
|
||||
# BACKUP VERIFY shares for checksum verification against remote
|
||||
# HOST1 RSYNC PROFILE host1-appdata profile for HOST1-specific appdata syncs
|
||||
#
|
||||
# ── DOCKER ─────────────────────────────────────────────────────────────────────────────────
|
||||
# DOCKER DAILY RESTART containers restarted daily
|
||||
# DOCKER WEEKLY RESTART containers restarted weekly
|
||||
# DOCKER WATCHDOG memory limits, health URLs, required containers, ignore list
|
||||
# DOCKER NETWORK CONNECT networks and containers for docker_network_connect.sh
|
||||
#
|
||||
# ── FALLBACK ───────────────────────────────────────────────────────────────────────────────
|
||||
# DDNS DDNS containers managed by HOST1
|
||||
# INTERNET LOSS containers stopped when internet is lost
|
||||
# FALLBACK TIERS what HOST1 runs for HOST2 per tier
|
||||
# TIER DELAYS how long HOST1 must be down before each tier activates on HOST2
|
||||
# RSYNC WRITEBACK HOST1 appdata synced back on handback
|
||||
#
|
||||
# ── MEDIA ──────────────────────────────────────────────────────────────────────────────────
|
||||
# MEDIA PERMISSIONS share list for media_shares_permissions.sh
|
||||
# MEDIA CLEANER folder lists for media_cleaner.sh
|
||||
#
|
||||
# ── MONITORS ───────────────────────────────────────────────────────────────────────────────
|
||||
# CERTIFICATE MONITOR domains checked for SSL expiry
|
||||
# SMART HEALTH drives to skip in SMART monitoring
|
||||
# ZFS REPORT pools to exclude from ZFS health report
|
||||
#
|
||||
# ── TRANSCODES ─────────────────────────────────────────────────────────────────────────────
|
||||
# TRANSCODES ramdisk size, thresholds, SSD path, server array
|
||||
#
|
||||
# ── ARR STACK ──────────────────────────────────────────────────────────────────────────────
|
||||
# DOWNLOADERS slskd, SABnzbd, qBittorrent credentials and URLs
|
||||
# LIDARR URL, API key, path map
|
||||
# SONARR URL, API key, path map
|
||||
# RADARR URL, API key, path map
|
||||
# ARR RECOVERY per-arr recovery toggles
|
||||
#
|
||||
# ── SYSTEM WATCHDOG ────────────────────────────────────────────────────────────────────────
|
||||
# SYSTEM WATCHDOG per-host check toggles and NIC configuration
|
||||
#
|
||||
# ── RESOURCE MANAGER ───────────────────────────────────────────────────────────────────────
|
||||
# RESOURCE MANAGER containers paused/stopped under memory pressure
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
# ==============================================================================================
|
||||
# ── IDENTITY & CONNECTIVITY ───────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Identity ━━━
|
||||
# HOST1 hostname lives in master.conf (not a credential — safe for all servers).
|
||||
# SSH key used for all server-to-server operations — rsync, failover container commands.
|
||||
# Must be in /root/.ssh/ and authorised in HOST2's /root/.ssh/authorized_keys.
|
||||
HOST1_SSH_KEY="/root/.ssh/gmer4lfe_rsync_automation"
|
||||
HOST1_OWNER="gmer4lfe"
|
||||
HOST1_OWNER_EMAIL="gmer4lfe@gmail.com"
|
||||
|
||||
# ━━━ Emby ━━━
|
||||
# Referenced by transcode_manager.sh, emby_session_report.sh, emby_database_repair.sh,
|
||||
# weekly_sync_maintenance.sh, and HOST1_TRANSCODE_SERVERS below.
|
||||
# API key: Emby Dashboard → API Keys → + New Key
|
||||
HOST1_EMBY_CONTAINER="Emby"
|
||||
HOST1_EMBY_URL="http://localhost:8096"
|
||||
HOST1_EMBY_API_KEY="0c27448d93a7431f9ac63569f7655829"
|
||||
|
||||
# ━━━ Jellyfin ━━━
|
||||
# API key: Jellyfin Dashboard → Administration → API Keys → + New Key
|
||||
HOST1_JELLYFIN_CONTAINER="Jellyfin"
|
||||
HOST1_JELLYFIN_URL="http://localhost:8095"
|
||||
HOST1_JELLYFIN_API_KEY="4e820e7df74c4933acec212b1996314e"
|
||||
|
||||
# ━━━ Gitea ━━━
|
||||
# Personal access token for gitea_ssh_setup.sh — registers this server's SSH public key
|
||||
# with Gitea so git operations use key auth instead of passwords.
|
||||
# Create in Gitea: Settings → Applications → Generate Token → scope: write:user
|
||||
HOST1_GITEA_API_TOKEN=""
|
||||
|
||||
# ━━━ Notifications ━━━
|
||||
# Discord webhook — leave blank to disable.
|
||||
# Per-host so HOST1 and HOST2 can post to different channels or only one server notifies.
|
||||
HOST1_DISCORD_WEBHOOK=""
|
||||
|
||||
# ━━━ Partnership ━━━
|
||||
# HOST1 is always the owner (source of truth) unless --transfer has been run.
|
||||
# See README-Partnership.md and master.conf PARTNERSHIP section for full lifecycle docs.
|
||||
|
||||
# Auth containers reconfigured on onboard/offboard.
|
||||
# Format: "ContainerName|WebUIPort"
|
||||
# On onboard → WebUI pointed at owner's Tailscale IP (mirror clicks NPM, gets owner's auth)
|
||||
# On offboard → WebUI pointed back at localhost
|
||||
HOST1_PARTNERSHIP_AUTH_WEBUIS=(
|
||||
"NginxProxyManager|81"
|
||||
"Lldap-Gmer4Lfe|17170"
|
||||
"Authelia|9091"
|
||||
"Authelia-Secondary|9092"
|
||||
)
|
||||
|
||||
# XML templates (from this server's templates-user/) pushed to mirror during onboard.
|
||||
# These become the mirror's active auth stack, backed by the rsync-synced appdata.
|
||||
# Update filename if Lldap is renamed to drop the host suffix.
|
||||
HOST1_PARTNERSHIP_AUTH_STACK=(
|
||||
# Dependencies first — Mariadb/Redis must be healthy before Authelia starts
|
||||
"my-Mariadb-Authelia.xml"
|
||||
"my-Mariadb-Authelia-Secondary.xml"
|
||||
"my-Redis-Authelia.xml"
|
||||
"my-Redis-Authelia-Secondary.xml"
|
||||
# Auth apps — deployed after their deps are confirmed healthy
|
||||
"my-Authelia.xml"
|
||||
"my-Authelia-Secondary.xml"
|
||||
"my-NginxProxyManager.xml"
|
||||
"my-Lldap-Gmer4Lfe.xml"
|
||||
# Source of truth — must be available on HOST2 independently of the auth stack
|
||||
"my-Gitea.xml"
|
||||
)
|
||||
|
||||
# XML templates pushed to mirror for the arr stack during onboard.
|
||||
# Deps (e.g. databases) first if any — same ordering rule as auth stack.
|
||||
HOST1_PARTNERSHIP_ARR_STACK=(
|
||||
# "my-Sonarr.xml"
|
||||
# "my-Radarr.xml"
|
||||
# "my-Lidarr.xml"
|
||||
# "my-Prowlarr.xml"
|
||||
# "my-Bazarr.xml"
|
||||
)
|
||||
|
||||
# Paths HOST2 should collect during the grace window after offboard.
|
||||
# Notified on offboard — no auto-deletion, HOST2 must collect manually within PARTNERSHIP_GRACE_HOURS.
|
||||
HOST1_PARTNERSHIP_MIRROR_BACKUPS=(
|
||||
# "/mnt/user/appdata-Fallback/Jayred365-Emby"
|
||||
)
|
||||
|
||||
# Containers parked on this server when partnership is active.
|
||||
# Stopped on onboard (owner deploys its stack instead), restarted on offboard.
|
||||
HOST1_PARTNERSHIP_OWN_CONTAINERS=(
|
||||
# "Emby"
|
||||
# "NginxProxyManager"
|
||||
)
|
||||
|
||||
# Emby admin provisioning — toggle is owner-only, credentials are per-host.
|
||||
# Owner enables/disables the feature. Each host sets the account they want on the shared Emby.
|
||||
# On onboard: owner reads mirror's HOST*_PARTNERSHIP_EMBY_ADMIN_* and creates that account.
|
||||
# On offboard: account is deleted. Username collision → onboard exits with error.
|
||||
HOST1_PARTNERSHIP_PROVISION_EMBY_ADMIN=false # owner controls whether Emby is shared
|
||||
HOST1_PARTNERSHIP_EMBY_PORT=8096
|
||||
HOST1_PARTNERSHIP_EMBY_ADMIN_USER="" # this server's desired Emby username
|
||||
HOST1_PARTNERSHIP_EMBY_ADMIN_PASS="" # this server's desired Emby password
|
||||
|
||||
# ==============================================================================================
|
||||
# ── RSYNC ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Daily Sync Shares ━━━
|
||||
# Shares HOST1 pushes to all other nodes every night (1am via daily_sync_maintenance.sh).
|
||||
# Mesh model: every node pushes every media share — no ownership, no mirrors.
|
||||
# arr_sync ensures all arr libraries converge (union). rsync spreads files (additive, no --delete).
|
||||
# arr_cleanup removes true orphans based on local arr state.
|
||||
# Any node can download content to any share — it propagates to all nodes on the next cycle.
|
||||
# Nextcloud is intentionally one-directional (HOST1→HOST2 offsite backup — not arr-managed).
|
||||
# Uses DEFAULT_RSYNC_OPTS from master.conf — no profile needed.
|
||||
# For shares needing container stops or custom options — add a profile in master.conf.
|
||||
HOST1_DAILY_SYNC_SHARES=(
|
||||
/mnt/user/Books
|
||||
/mnt/user/Intros
|
||||
/mnt/user/Kids_Movies
|
||||
/mnt/user/Kids_Tv_Shows
|
||||
/mnt/user/Movies
|
||||
/mnt/user/Music
|
||||
/mnt/user/Music_Videos
|
||||
/mnt/user/Nextcloud
|
||||
/mnt/user/stand-up_comedy
|
||||
/mnt/user/Sports
|
||||
/mnt/user/Tv_Shows
|
||||
/mnt/user/Anime_Shows-Old
|
||||
/mnt/user/Anime_Movies-Old
|
||||
/mnt/user/Anime_Movies
|
||||
/mnt/user/Anime_Shows
|
||||
)
|
||||
|
||||
# Personal encrypted shares — synced for offsite backup, independent of media shares.
|
||||
# ZFS encrypted at dataset level — remote receives encrypted blocks, cannot read content.
|
||||
# See README-Rsync_Setup.md for ZFS encryption setup before uncommenting.
|
||||
HOST1_PERSONAL_SHARES=(
|
||||
# /mnt/user/HOST1-Personal # uncomment after creating encrypted dataset
|
||||
)
|
||||
|
||||
# ━━━ Weekly Sync Shares ━━━
|
||||
# Appdata shares synced during the weekly maintenance window (Sunday 2:30am).
|
||||
# Containers stopped both sides before sync — full clean state guaranteed.
|
||||
# Profiles drive container stops, excludes, and options — configured in master.conf RSYNC section.
|
||||
# Order matters — Emby first (larger transfer), then Critical-Data (auth stack).
|
||||
HOST1_WEEKLY_SYNC_SHARES=(
|
||||
"/mnt/user/Media_Server/Emby" # emby profile — full clean mirror
|
||||
"/mnt/user/appdata-Fallback/Critical-Data" # critical-data profile — auth stack
|
||||
)
|
||||
|
||||
# ━━━ Intermediate Sync Shares ━━━
|
||||
# Shares synced every 4 hours by intermediate_sync_maintenance.sh.
|
||||
# Uses DEFAULT_RSYNC_OPTS (no --delete) — for sub-daily propagation of metadata or watch state.
|
||||
# Full media share sync stays in the daily window. Leave empty to skip mid-day rsync entirely.
|
||||
HOST1_INTERMEDIATE_SYNC_SHARES=(
|
||||
# Add shares here to enable mid-day rsync
|
||||
# Example: "/mnt/user/Emby_Metadata"
|
||||
)
|
||||
|
||||
# ━━━ Critical Sync Shares ━━━
|
||||
# Appdata shares synced every 30 minutes by critical_sync_maintenance.sh.
|
||||
# Format: "/path/to/share" or "/path/to/share|profile-name"
|
||||
# Order matters — Critical-Data first (auth stack), then Emby dirty sync.
|
||||
HOST1_CRITICAL_SYNC_SHARES=(
|
||||
"/mnt/user/appdata-Fallback/Critical-Data|critical-fallback" # auth dirty sync — stays running
|
||||
"/mnt/user/Media_Server/Emby|emby-fallback" # Emby dirty sync — stays running
|
||||
)
|
||||
|
||||
# ━━━ Backup Verify ━━━
|
||||
# Shares verified by backup_verify.sh — random file checksum comparison against remote.
|
||||
# Leave empty to use HOST1_DAILY_SYNC_SHARES automatically.
|
||||
# Sample size and minimum file size defined in master.conf.
|
||||
HOST1_BACKUP_VERIFY_SHARES=(
|
||||
# leave empty to use HOST1_DAILY_SYNC_SHARES automatically
|
||||
)
|
||||
|
||||
# ━━━ HOST1 Rsync Profile — host1-appdata ━━━
|
||||
# HOST1-specific appdata sync profile — extends the shared PROFILE_* arrays in master.conf.
|
||||
# Use for appdata unique to HOST1 (Organizrv2, VaultWarden, UptimeKuma etc.)
|
||||
# Shared appdata (auth stack, Emby) use dedicated profiles defined in master.conf.
|
||||
# Run manually: bash Rsync/rsync.sh /mnt/user/appdata-Fallback/HOST1-Appdata --profile=host1-appdata
|
||||
PROFILE_RSYNC_OPTS[host1-appdata]="-av --info=progress2 --bwlimit=${PROFILE_BW_LIMIT[host1-appdata]:-8000}"
|
||||
PROFILE_BW_LIMIT[host1-appdata]=8000
|
||||
PROFILE_RETRY_COUNT[host1-appdata]=3
|
||||
PROFILE_SLEEP[host1-appdata]=300
|
||||
PROFILE_CRITICAL_CONTAINER_NAMES[host1-appdata]="Organizrv2-Gmer4Lfe UptimeKuma-Gmer4Lfe VaultWarden-Gmer4Lfe"
|
||||
PROFILE_DELAYED_CONTAINERS[host1-appdata]=""
|
||||
PROFILE_CONTAINER_DELAY[host1-appdata]=5
|
||||
PROFILE_EXCLUDE_DIRS[host1-appdata]="logs *.tmp"
|
||||
|
||||
# ==============================================================================================
|
||||
# ── DOCKER ────────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Docker Daily Restart ━━━
|
||||
# Containers restarted every day via DAILY_MAINTENANCE_SCRIPTS.
|
||||
# Dispatcharr degrades over time without restart — daily is intentional, not just housekeeping.
|
||||
# Order matters — auth stack first, then media services.
|
||||
HOST1_DAILY_RESTART_CONTAINERS=(
|
||||
"NginxProxyManager"
|
||||
"Lldap-Gmer4Lfe"
|
||||
"Authelia"
|
||||
"Authelia-Secondary"
|
||||
"Dispatcharr-Iptv-Users"
|
||||
"Dispatcharr" # Live TV scheduler — degrades without daily restart
|
||||
"Dispatcharr-Basic"
|
||||
"ErsatzTV-Emby"
|
||||
"Slskd" # Soulseek connection drops after extended uptime; restart refreshes share index
|
||||
)
|
||||
|
||||
# ━━━ Docker Weekly Restart ━━━
|
||||
# Less critical services restarted weekly via WEEKLY_MAINTENANCE_SCRIPTS (Sunday 2:30am).
|
||||
# Containers already stopped for weekly sync — restart adds zero extra downtime.
|
||||
HOST1_WEEKLY_RESTART_CONTAINERS=(
|
||||
"NextCloud"
|
||||
"Organizrv2-Gmer4Lfe"
|
||||
"AdGuard-Home"
|
||||
"Immich-Gmer4Lfe"
|
||||
)
|
||||
|
||||
# ━━━ Docker Watchdog ━━━
|
||||
# Per-HOST1 container configuration for docker_watchdog.sh.
|
||||
# Shared thresholds and toggles live in master.conf.
|
||||
|
||||
# Memory hard limits in MB — immediate restart if exceeded.
|
||||
# Set at "container is clearly broken" not "container is busy".
|
||||
# 20GB=20480 18GB=18432 16GB=16384 12GB=12288 8GB=8192 4GB=4096 2GB=2048 1GB=1024
|
||||
declare -A HOST1_WATCHDOG_CONTAINERS=(
|
||||
["Emby"]=20480 # 20GB — large library + active transcodes
|
||||
["LidaTube"]=6144 # 6GB — memory leak over time
|
||||
["Tdarr"]=6144 # 6GB — encoding is memory intensive
|
||||
["Code-Server"]=1024 # 1GB — should never need more
|
||||
)
|
||||
|
||||
# HTTP health check URLs — checked every cycle, strike system before restart.
|
||||
# Only add containers with a meaningful web interface to check.
|
||||
declare -A HOST1_WATCHDOG_CONTAINER_URLS=(
|
||||
["Emby"]="http://localhost:8096"
|
||||
)
|
||||
|
||||
# Required containers — must always be running on HOST1.
|
||||
# Strike system before restart — repeated failures go on skip list, auto-clears on recovery.
|
||||
# Listed in dependency order — dependencies before dependents.
|
||||
HOST1_WATCHDOG_REQUIRED_CONTAINERS=(
|
||||
"NginxProxyManager"
|
||||
"Lldap-Gmer4Lfe"
|
||||
"Mariadb-Authelia"
|
||||
"Mariadb-Authelia-Secondary"
|
||||
"Redis-Authelia"
|
||||
"Redis-Authelia-Secondary"
|
||||
"Authelia"
|
||||
"Authelia-Secondary"
|
||||
)
|
||||
|
||||
# Containers to skip in Tier 2 global scan — legitimately stopped or frequently restarting.
|
||||
# Watchdog leaves these alone entirely — no restart attempts, no crash loop tracking.
|
||||
HOST1_WATCHDOG_SCAN_IGNORE=(
|
||||
"DashGate"
|
||||
"PIA-WG-Config-Generator"
|
||||
"Aperture"
|
||||
"Aperture-Kids"
|
||||
"pgvector-18-Apeture-Kids"
|
||||
"Pgvector18-Aperture"
|
||||
"emby-test" # broken test container (exit 127 — bad image)
|
||||
)
|
||||
|
||||
# Dependency ordering — skip restarting a container if its dependency is also down.
|
||||
# Prevents watchdog from restarting Authelia before Mariadb is back up.
|
||||
# SPACE-SEPARATED STRINGS — converted to array at runtime.
|
||||
declare -A HOST1_WATCHDOG_DEPENDENCIES=(
|
||||
["Authelia"]="Mariadb-Authelia Redis-Authelia"
|
||||
["Authelia-Secondary"]="Mariadb-Authelia Redis-Authelia-Secondary"
|
||||
["NextCloud"]="Postgres-NextCloud"
|
||||
)
|
||||
|
||||
# Per-container appdata growth suppress ceilings in MB.
|
||||
# ONLY needed in specific cases — growth rate detection covers all containers automatically.
|
||||
# Use this when a container legitimately has large stable data and you want to guarantee
|
||||
# it never triggers a false-positive growth alert. Growth warnings are suppressed while the
|
||||
# container's dir stays below this ceiling; above it, warnings resume as normal.
|
||||
# 50GB=51200 25GB=25600 20GB=20480 15GB=15360 10GB=10240 5GB=5120
|
||||
declare -A HOST1_WATCHDOG_APPDATA_SIZES=(
|
||||
["Tdarr"]="25600" # 25GB — transcode cache grows legitimately during active jobs
|
||||
["7dtd"]="20480" # 20GB — game server world data, expected to be large
|
||||
)
|
||||
|
||||
# ━━━ Network Watchdog ━━━
|
||||
# Host-specific connectivity config for Watchdogs/System/network_watchdog.sh.
|
||||
HOST1_NETWORK_WATCHDOG_DDNS_DOMAIN="gmer4lfe.com"
|
||||
HOST1_NETWORK_WATCHDOG_DDNS_CONTAINER="Gmer4Lfe.com"
|
||||
HOST1_NETWORK_WATCHDOG_NPM_URL="https://gmer4lfe.com"
|
||||
|
||||
# ━━━ Docker Network Connect ━━━
|
||||
# Containers connected to custom networks at array start by docker_network_connect.sh.
|
||||
# Networks created if they don't exist — idempotent, safe to re-run.
|
||||
HOST1_NETWORK_CONNECT_CONTAINERS=(
|
||||
"memcached"
|
||||
"Npm-CrowdSec"
|
||||
)
|
||||
|
||||
HOST1_NETWORK_CONNECT_NETWORKS=(
|
||||
"high-availability"
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── FALLBACK ──────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ DDNS ━━━
|
||||
# DDNS containers HOST1 manages — started/stopped by fallback.sh per DDNS absolute rules:
|
||||
# Internet loss → stop immediately
|
||||
# Failover → HOST2 starts HOST1's DDNS as Tier 1 (before any other containers)
|
||||
# Handback → stop HOST1's DDNS on HOST2 → rsync → start containers → start local DDNS last
|
||||
HOST1_DDNS_CONTAINERS=(
|
||||
"Gmer4Lfe.com"
|
||||
)
|
||||
|
||||
# ━━━ Internet Loss ━━━
|
||||
# Containers stopped immediately on HOST1 when internet connection is lost.
|
||||
# Prevents external-facing services from operating without connectivity.
|
||||
FALLBACK_HOST1_STOP_ON_NO_NET=(
|
||||
"Gmer4Lfe.com"
|
||||
)
|
||||
|
||||
# ━━━ Fallback Tiers — HOST1 Runs for HOST2 ━━━
|
||||
# Containers HOST1 starts when HOST2 goes down.
|
||||
# Tier 1 is always immediate — vital services cannot wait.
|
||||
# Higher tiers activate after HOST2_TIER*_DELAY minutes (set in host2.conf).
|
||||
FALLBACK_HOST1_COVERS_HOST2_TIER1=(
|
||||
"Gmer4Lfe.us"
|
||||
"VaultWarden-Jayred365"
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_COVERS_HOST2_TIER2=(
|
||||
# "container-placeholder"
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_COVERS_HOST2_TIER3=(
|
||||
# "container-placeholder"
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_COVERS_HOST2_TIER4=(
|
||||
# "container-placeholder"
|
||||
)
|
||||
|
||||
# ━━━ Tier Delays — HOST1's Containers on HOST2 ━━━
|
||||
# How long HOST1 must be down before each tier activates on HOST2 — in minutes.
|
||||
# Tier 1 is always immediate — no delay var needed.
|
||||
HOST1_TIER2_DELAY=240 # 4 hours — NextCloud, Immich
|
||||
HOST1_TIER3_DELAY=720 # 12 hours — secondary services
|
||||
HOST1_TIER4_DELAY=1440 # 24 hours — arrs + downloaders
|
||||
|
||||
# ━━━ Rsync Writeback — HOST1 Appdata Back on Handback ━━━
|
||||
# Syncs HOST1 appdata BACK to HOST1 when it comes back online after a failover.
|
||||
# Containers stopped before writeback — clean source, no competing writes.
|
||||
#
|
||||
# HOST1_TIER1_WRITEBACK_DELAY: short outages skip Tier 1 writeback — primary state
|
||||
# is more reliable than dirty sync data for brief outages.
|
||||
HOST1_TIER1_WRITEBACK_DELAY=60 # skip Emby writeback if outage under 1hr
|
||||
|
||||
# Tier 4 automatically syncs HOST1_DAILY_SYNC_SHARES — only list paths NOT in that array.
|
||||
FALLBACK_HOST1_WRITEBACK_TIER1=(
|
||||
"/mnt/user/Media_Server/Emby" # watch states built up during outage
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_WRITEBACK_TIER2=(
|
||||
"/mnt/user/appdata-Fallback/Important-Data" # NextCloud + Postgres — files added during outage
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_WRITEBACK_TIER3=(
|
||||
# "location-placeholder"
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_WRITEBACK_TIER4=(
|
||||
"/mnt/user/appdata-Fallback/Arrs_Stack" # arr databases — downloads queued during outage
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── MEDIA ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Media Permissions ━━━
|
||||
# Shares that media_shares_permissions.sh applies PERMISSIONS_MODE and PERMISSIONS_OWNER to.
|
||||
# Runs first in DAILY_MAINTENANCE_SCRIPTS — arr cleanup depends on correct ownership.
|
||||
HOST1_MEDIA_PERMISSION_SHARES=(
|
||||
/mnt/user/Anime_Movies
|
||||
/mnt/user/Anime_Movies-Old
|
||||
/mnt/user/Anime_Shows
|
||||
/mnt/user/Anime_Shows-Old
|
||||
/mnt/user/appcache
|
||||
/mnt/user/Books
|
||||
/mnt/user/Downloads
|
||||
/mnt/user/Games
|
||||
/mnt/user/Intros
|
||||
/mnt/user/Kids_Movies
|
||||
/mnt/user/Kids_Tv_Shows
|
||||
/mnt/user/Movie_Recordings
|
||||
/mnt/user/Movies
|
||||
/mnt/user/Music
|
||||
/mnt/user/Music_Videos
|
||||
/mnt/user/Photo
|
||||
/mnt/user/Sports
|
||||
/mnt/user/stand-up_comedy
|
||||
/mnt/user/Temp_Storage
|
||||
/mnt/user/Tv_Recordings
|
||||
/mnt/user/Tv_Shows
|
||||
/mnt/user/YouTube
|
||||
)
|
||||
|
||||
# ━━━ Media Cleaner ━━━
|
||||
# Folder lists for media_cleaner.sh — two profiles: anime and media.
|
||||
# File patterns shared across all servers — defined in master.conf.
|
||||
# Called via DAILY_MAINTENANCE_SCRIPTS. Run manually: Media/media_cleaner.sh anime|media
|
||||
HOST1_ANIME_CLEAN_FOLDERS=(
|
||||
/mnt/user/Anime_Movies
|
||||
/mnt/user/Anime_Movies-Old
|
||||
/mnt/user/Anime_Shows
|
||||
/mnt/user/Anime_Shows-Old
|
||||
)
|
||||
|
||||
HOST1_MEDIA_CLEAN_FOLDERS=(
|
||||
/mnt/user/Kids_Movies
|
||||
/mnt/user/Kids_Tv_Shows
|
||||
/mnt/user/Movies
|
||||
/mnt/user/Music
|
||||
/mnt/user/Sports
|
||||
/mnt/user/stand-up_comedy
|
||||
/mnt/user/Tv_Shows
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── MONITORS ──────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Certificate Monitor ━━━
|
||||
# Domains checked via direct openssl connection — not relying on NPM's certificate state.
|
||||
# Checks the actual certificate served, not what NPM thinks it has.
|
||||
# Thresholds (CERT_WARN_DAYS, CERT_CRIT_DAYS) defined in master.conf.
|
||||
HOST1_CERT_MONITOR_DOMAINS=(
|
||||
"Gmer4Lfe.com"
|
||||
"Gmer4Lfe.us"
|
||||
)
|
||||
|
||||
# ━━━ SMART Health ━━━
|
||||
# Drives skipped in SMART attribute monitoring — hardware is server-specific.
|
||||
# Thresholds read from dynamix.cfg at runtime — fallbacks in master.conf.
|
||||
HOST1_SMART_IGNORE_DRIVES=(
|
||||
"sda" # boot USB — SMART not meaningful on flash drives
|
||||
)
|
||||
|
||||
# ━━━ ZFS Report ━━━
|
||||
# Pools excluded from the weekly ZFS health report — reduces noise from single-disk array pools.
|
||||
# These are individual array disks formatted as ZFS — converting to XFS over time via unBalance.
|
||||
# Pool health thresholds defined in master.conf.
|
||||
HOST1_ZFS_REPORT_IGNORE_POOLS=(
|
||||
"disk5"
|
||||
"disk6"
|
||||
"disk8"
|
||||
"disk9"
|
||||
"disk10"
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── TRANSCODES ────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# Ramdisk size ceiling — tmpfs only uses RAM actually needed, not the full size upfront.
|
||||
# Real-world: 9 streams peaked at ~5.5GB — 10G gives generous headroom on 128GB RAM.
|
||||
HOST1_RAMDISK_SIZE="10G"
|
||||
|
||||
# Usage thresholds — coupled to HOST1_RAMDISK_SIZE, adjust all three together if size changes.
|
||||
# Hysteresis gap (8.5 - 7 = 1.5GB) prevents flip-flop between ramdisk and SSD.
|
||||
HOST1_RAMDISK_WARN_GB=8.5 # flip to SSD when ramdisk usage reaches this
|
||||
HOST1_RAMDISK_LOW_GB=7 # flip back to ramdisk when usage drops to this
|
||||
|
||||
# SSD fallback path — where transcodes land when ramdisk exceeds HOST1_RAMDISK_WARN_GB.
|
||||
# Must be on cache pool — array disks too slow for active transcode writes.
|
||||
HOST1_TRANSCODE_SSD="/mnt/cache/Temp_Storage/Emby/Transcodes/"
|
||||
|
||||
# Media servers sharing the ramdisk transcode space on HOST1.
|
||||
# Format: "ContainerName|URL|APIKey|Type" — Type: emby | jellyfin | plex
|
||||
# Entries with placeholder API keys are skipped automatically.
|
||||
# ⚠️ Tdarr does NOT belong here — keep Tdarr on SSD, not ramdisk.
|
||||
HOST1_TRANSCODE_SERVERS=(
|
||||
"${HOST1_EMBY_CONTAINER}|${HOST1_EMBY_URL}|${HOST1_EMBY_API_KEY}|emby"
|
||||
"${HOST1_JELLYFIN_CONTAINER}|${HOST1_JELLYFIN_URL}|${HOST1_JELLYFIN_API_KEY}|jellyfin"
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── ARR STACK ─────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Used by arr cleanup scripts and arrs_failed_stalled_recovery.sh.
|
||||
# detect_hosts() selects HOST1 vars when running on HOST1.
|
||||
#
|
||||
# PATH MAPS — container path → host path translation.
|
||||
# Arr stores file paths using container-internal paths — scripts need host paths to scan.
|
||||
# Add one entry per root folder in arr Settings → Media Management → Root Folders.
|
||||
|
||||
# ━━━ Downloaders ━━━
|
||||
# Used by downloaders_reset.sh — runs every 30min via CRITICAL_MAINTENANCE_SCRIPTS.
|
||||
# Clears stuck states, purges old history, prepares each client for a clean cycle.
|
||||
|
||||
# slskd — clears stuck searches, dead transfers, purges expired failed imports.
|
||||
# SLSKD_FAILED_IMPORTS_DIR: where Soularr moves albums Lidarr rejected.
|
||||
HOST1_SLSKD_URL="http://localhost:8980"
|
||||
HOST1_SLSKD_API_KEY="4bF9kL2mNpQrT7vWxYz1A3dEgHjKoRsU"
|
||||
HOST1_SLSKD_FAILED_IMPORTS_DIR="/mnt/user/Temp_Storage/Slskd/completed/failed_imports"
|
||||
|
||||
# SABnzbd
|
||||
HOST1_SABNZBD_URL="http://localhost:8180"
|
||||
HOST1_SABNZBD_API_KEY="8bfefe41d83b4d50883e32859b55ca9a"
|
||||
|
||||
# qBittorrent — deleteFiles=false removes torrent from qBit but leaves files on disk.
|
||||
# Radarr/Sonarr manage actual files independently.
|
||||
HOST1_QBIT_URL="http://localhost:8080"
|
||||
HOST1_QBIT_USERNAME="root"
|
||||
HOST1_QBIT_PASSWORD="Stay0utD!ck"
|
||||
|
||||
# ━━━ Lidarr — HOST1 only ━━━
|
||||
# HOST2 does not run Lidarr — HOST1_LIDARR_RECOVERY flag handles the exit cleanly.
|
||||
HOST1_LIDARR_URL="http://localhost:8686"
|
||||
HOST1_LIDARR_API_KEY="b2977e71ef074bc0a0529d9fcce3b2dc"
|
||||
HOST1_LIDARR_MUSIC_ROOT="/mnt/user/Music-New"
|
||||
HOST1_FANART_API_KEY="Yd7147a43b692df0b364b94dc47efb81"
|
||||
HOST1_LASTFM_API_KEY="be6dc169c33ae263e690c30d18b7491d"
|
||||
|
||||
declare -A HOST1_LIDARR_PATH_MAP=(
|
||||
["/ext-music"]="/mnt/user/Music-New"
|
||||
)
|
||||
|
||||
# ━━━ Sonarr ━━━
|
||||
HOST1_SONARR_URL="http://localhost:8989"
|
||||
HOST1_SONARR_API_KEY="130decd3db5b4c25afad64864cd03f9f"
|
||||
HOST1_SONARR_TV_ROOT="/mnt/user/Tv_Shows"
|
||||
|
||||
# Note: stand-up_comedy in both Sonarr + Radarr — TV specials and movie specials, one folder
|
||||
declare -A HOST1_SONARR_PATH_MAP=(
|
||||
["/tv"]="/mnt/user/Tv_Shows"
|
||||
["/ext-standup-comedy"]="/mnt/user/stand-up_comedy"
|
||||
["/kids tv"]="/mnt/user/Kids_Tv_Shows"
|
||||
["/ext-anime-shows"]="/mnt/user/Anime_Shows-Old"
|
||||
)
|
||||
|
||||
# ━━━ Radarr ━━━
|
||||
HOST1_RADARR_URL="http://localhost:7878"
|
||||
HOST1_RADARR_API_KEY="d43a3ec6cf1549edb4af0cc63f98b2a9"
|
||||
HOST1_TMDB_API_KEY="3dac5e2e49b5540472d2eafec4f01260"
|
||||
HOST1_RADARR_MOVIES_ROOT="/mnt/user/Movies"
|
||||
|
||||
# Note: stand-up_comedy in both Radarr + Sonarr — movie specials and TV specials, one folder
|
||||
declare -A HOST1_RADARR_PATH_MAP=(
|
||||
["/movies"]="/mnt/user/Movies"
|
||||
["/kids movies"]="/mnt/user/Kids_Movies"
|
||||
["/ext-stand-up-comedy"]="/mnt/user/stand-up_comedy"
|
||||
["/anime-movies"]="/mnt/user/Anime_Movies-Old"
|
||||
)
|
||||
|
||||
# ━━━ Arr Recovery Toggles ━━━
|
||||
# false = skip that arr on this host — exits cleanly without error
|
||||
HOST1_SONARR_RECOVERY=true
|
||||
HOST1_RADARR_RECOVERY=true
|
||||
HOST1_LIDARR_RECOVERY=true # HOST1 only — exits cleanly on HOST2
|
||||
|
||||
# ==============================================================================================
|
||||
# ── SYSTEM WATCHDOG ───────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Per-host check toggles and NIC config for system_watchdog.sh.
|
||||
# Aliased by detect_hosts() — script uses unprefixed SYS_WATCHDOG_* names.
|
||||
# HOST1: TR1950X 128GB — full media server, active transcoding, ZFS cache pools.
|
||||
#
|
||||
# Three-tier response — all critical checks enabled by default on HOST1:
|
||||
# Tier 1 (bypass strikes, reboot now): docker daemon, rootfs full, kernel oops, FD, /boot
|
||||
# Tier 2 (bypass strikes with OOM): RAM critical + OOM kills in cycle
|
||||
# Tier 3 (standard strike system): everything else
|
||||
#
|
||||
# RAM tiers, OOM limits, and reboot loop settings in master.conf System Watchdog section.
|
||||
|
||||
# ━━━ Primary NIC ━━━
|
||||
# Network interface for NIC state check — verify with: ip link show | grep "^[0-9]"
|
||||
# Common values: eth0, bond0, br0, eno1
|
||||
HOST1_SYS_WATCHDOG_NIC="eth0"
|
||||
|
||||
# ━━━ Tier 1 — Critical Checks ━━━
|
||||
# These bypass the strike system — a single hit triggers immediate reboot.
|
||||
# Disabling any of these is not recommended — they protect against acute system failure.
|
||||
|
||||
# Docker daemon unresponsive → try restart, reboot if restart fails.
|
||||
# Without a working daemon docker_watchdog.sh is blind and containers cannot be managed.
|
||||
HOST1_SYS_WATCHDOG_CHECK_DOCKER_DAEMON=true
|
||||
|
||||
# rootfs at critical threshold (ROOTFS_CRITICAL_PCT=99) → reboot immediately.
|
||||
# At 99% rootfs writes fail silently — logs stop, Docker errors out, SSH may stop working.
|
||||
# Standard 95% threshold still uses strike system — only 99%+ is critical tier.
|
||||
HOST1_SYS_WATCHDOG_CHECK_ROOTFS=true
|
||||
|
||||
# Kernel BUG/Oops in dmesg delta since last cycle → reboot immediately.
|
||||
# A kernel oops means the kernel ran with a corrupted state — stability is not guaranteed.
|
||||
HOST1_SYS_WATCHDOG_CHECK_KERNEL_OOPS=true
|
||||
|
||||
# File descriptor exhaustion at FD_CRITICAL_PCT (95%) → reboot immediately.
|
||||
# At 95% FD: new connections fail, Docker can't spawn processes, SSH drops.
|
||||
HOST1_SYS_WATCHDOG_CHECK_FD=true
|
||||
|
||||
# /boot read-only detected → reboot immediately.
|
||||
# Unexpected read-only /boot means state files and config writes are silently failing.
|
||||
# Fallback state, watchdog reboot log, and lock files all go stale silently.
|
||||
HOST1_SYS_WATCHDOG_CHECK_BOOT=true
|
||||
|
||||
# ━━━ Tier 2 — Urgent OOM Check ━━━
|
||||
# Bypass strikes when RAM is critically low AND OOM kill rate confirms active crisis.
|
||||
# Both must be enabled for Tier 2 bypass to function — disable either to always use strikes.
|
||||
|
||||
# Track kernel OOM kills each cycle via /proc/vmstat oom_kill delta.
|
||||
# Also provides diagnostic context in reboot messages (which processes were killed).
|
||||
HOST1_SYS_WATCHDOG_CHECK_OOM=true
|
||||
|
||||
# Free RAM check — required for both Tier 2 bypass and RAM tier logic.
|
||||
# Tiers: MEM_WARN_GB(10) → notify | MEM_SHUTDOWN_GB(6) → stop containers | MEM_GB(4) → strikes
|
||||
HOST1_SYS_WATCHDOG_CHECK_RAM=true
|
||||
|
||||
# ━━━ Tier 3 — Standard Checks (strike system) ━━━
|
||||
# Each check must fail SYS_WATCHDOG_STRIKE_LIMIT consecutive cycles before action is taken.
|
||||
# Single spikes are ignored — sustained problems trigger reboot.
|
||||
|
||||
# /var/log filesystem usage above SYS_WATCHDOG_LOG_PCT.
|
||||
# Log spam (Docker log storms, syslog loops) fills rootfs — indicates something broken.
|
||||
HOST1_SYS_WATCHDOG_CHECK_LOG=true
|
||||
|
||||
# ZFS ARC memory pinned above SYS_WATCHDOG_ARC_PINNED_PCT after cache drop.
|
||||
# Enabled on HOST1 — ZFS cache pools actively used. Disable on hosts without ZFS.
|
||||
HOST1_SYS_WATCHDOG_CHECK_ARC=true
|
||||
|
||||
# CPU temperature above SYS_WATCHDOG_CPU_TEMP_MAX (95°C).
|
||||
# Sustained high temp causes kernel throttling or panic. Requires lm-sensors.
|
||||
HOST1_SYS_WATCHDOG_CHECK_CPU_TEMP=true
|
||||
|
||||
# Load average above SYS_WATCHDOG_LOAD_MULTIPLIER × core count.
|
||||
# DISABLED on HOST1 — Tdarr and Emby cause legitimate sustained load spikes during encoding.
|
||||
# Enable on idle servers or adjust SYS_WATCHDOG_LOAD_MULTIPLIER if load is always high.
|
||||
HOST1_SYS_WATCHDOG_CHECK_LOAD=false
|
||||
|
||||
# Zombie process count above SYS_WATCHDOG_ZOMBIE_LIMIT (50).
|
||||
# Large zombie counts indicate serious process management failure — something is stuck.
|
||||
HOST1_SYS_WATCHDOG_CHECK_ZOMBIES=true
|
||||
|
||||
# Check docker_watchdog.sh persistent skip list — required containers on skip list.
|
||||
# Cross-watchdog coordination: if docker_watchdog gave up, system_watchdog escalates.
|
||||
# ENABLED — HOST1 fully built and operational, skip list is meaningful.
|
||||
HOST1_SYS_WATCHDOG_CHECK_CONTAINERS=true
|
||||
|
||||
# /tmp filesystem usage above SYS_WATCHDOG_TMP_PCT with auto-clear attempt.
|
||||
# Script tries to clear aged /tmp files first — only strikes if clear fails.
|
||||
# Lock files, rsync temp files, and Docker ops use /tmp — 100% means lock failures.
|
||||
HOST1_SYS_WATCHDOG_CHECK_TMP=true
|
||||
|
||||
# Array disk error count delta in /proc/mdstat — accumulating errors = disk failing now.
|
||||
# Triggers on SYS_WATCHDOG_MDSTAT_ERROR_LIMIT new errors in one cycle.
|
||||
HOST1_SYS_WATCHDOG_CHECK_MDSTAT=true
|
||||
|
||||
# Primary NIC operstate — detects NIC going down (physical or driver failure).
|
||||
# Uses HOST1_SYS_WATCHDOG_NIC above. Strike system — brief flaps don't trigger reboot.
|
||||
HOST1_SYS_WATCHDOG_CHECK_NETWORK=true
|
||||
|
||||
# sshd running check — attempts restart before escalating.
|
||||
# sshd down = no remote access. Script tries rc.sshd start, notifies, strikes on failure.
|
||||
HOST1_SYS_WATCHDOG_CHECK_SSHD=true
|
||||
|
||||
# Runaway process detection — single process above SYS_WATCHDOG_RUNAWAY_CPU_PCT sustained.
|
||||
# DISABLED — Tdarr encoding and Emby transcoding legitimately peg CPU for extended periods.
|
||||
# Enable only if HOST1 has no CPU-intensive workloads.
|
||||
HOST1_SYS_WATCHDOG_CHECK_RUNAWAY=false
|
||||
|
||||
# ==============================================================================================
|
||||
# ── RESOURCE MANAGER ──────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Containers to manage under pressure — see master.conf RW_CRITICAL_CONTAINERS for exclusions.
|
||||
|
||||
# docker pause at medium pressure (RAM < RW_RAM_MEDIUM_GB or load > medium threshold)
|
||||
# Suspended in-place — instant to pause/unpause, no state lost, no restart delay.
|
||||
HOST1_RW_PAUSE_CONTAINERS=(
|
||||
"Huntarr" # arr search automation — safe to suspend
|
||||
"Cleanuparr" # download cleanup — safe to suspend
|
||||
"Healarr" # arr health checks — safe to suspend
|
||||
"Soularr" # Slskd automation — background only
|
||||
"ChannelTube" # YouTube archiver — background only
|
||||
"Pinchflat" # YouTube archiver — background only
|
||||
)
|
||||
|
||||
# docker stop at hard pressure (RAM < RW_RAM_HARD_GB)
|
||||
# Full stop — these are optional/heavy services that free significant RAM when stopped.
|
||||
# resource_watchdog.sh restarts them when pressure fully clears (RAM >= RW_RAM_RECOVER_GB).
|
||||
HOST1_RW_STOP_CONTAINERS=(
|
||||
"LocalAI" # GPU/CPU heavy — largest RAM consumer when idle
|
||||
"7DaysToDie" # game server — optional
|
||||
"V-Rising" # game server — optional
|
||||
"Code-Server" # IDE — not needed during pressure events
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ──────────────────────── End Of HOST1 Variables ──────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
HOST1_UNRAID_API_KEY="1825c3a2e03ea5089974f4da2e171aa2d5907a1dea23cc479c33e492c8ff4dbb"
|
||||
@@ -0,0 +1,790 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ========================== HOST1 CONFIGURATION — unRAID-Gmer4Lfe ============================
|
||||
# ==============================================================================================
|
||||
# HOST1-specific variables — credentials, container names, share paths, failover lists.
|
||||
# Sourced after master.conf — values here extend shared profile arrays and add HOST1-specific
|
||||
# identity, credentials, and container configuration.
|
||||
#
|
||||
# Sparse checkout (git) ensures HOST2 never receives this file.
|
||||
# HOST2 never sees HOST1 credentials — clean separation at the file level.
|
||||
#
|
||||
# DO NOT put shared config here — thresholds, toggles, profiles belong in master.conf.
|
||||
# DO NOT put HOST2 variables here — they belong in host2.conf.
|
||||
#
|
||||
# ── INDEX ─────────────────────────────────────────────────────────────────────────────────────
|
||||
#
|
||||
# ── IDENTITY & CONNECTIVITY ────────────────────────────────────────────────────────────────
|
||||
# IDENTITY hostname, SSH key
|
||||
# EMBY container name, URL, API key
|
||||
# NOTIFICATIONS Discord webhook
|
||||
# PARTNERSHIP auth containers, backup paths
|
||||
#
|
||||
# ── RSYNC ──────────────────────────────────────────────────────────────────────────────────
|
||||
# DAILY SYNC SHARES media shares HOST1 owns and pushes to HOST2
|
||||
# WEEKLY SYNC SHARES appdata shares synced weekly (Sunday 2:30am)
|
||||
# CRITICAL SYNC SHARES appdata shares synced every 30 minutes
|
||||
# BACKUP VERIFY shares for checksum verification against remote
|
||||
# HOST1 RSYNC PROFILE host1-appdata profile for HOST1-specific appdata syncs
|
||||
#
|
||||
# ── DOCKER ─────────────────────────────────────────────────────────────────────────────────
|
||||
# DOCKER DAILY RESTART containers restarted daily
|
||||
# DOCKER WEEKLY RESTART containers restarted weekly
|
||||
# DOCKER WATCHDOG memory limits, health URLs, required containers, ignore list
|
||||
# DOCKER NETWORK CONNECT networks and containers for docker_network_connect.sh
|
||||
#
|
||||
# ── FALLBACK ───────────────────────────────────────────────────────────────────────────────
|
||||
# DDNS DDNS containers managed by HOST1
|
||||
# INTERNET LOSS containers stopped when internet is lost
|
||||
# FALLBACK TIERS what HOST1 runs for HOST2 per tier
|
||||
# TIER DELAYS how long HOST1 must be down before each tier activates on HOST2
|
||||
# RSYNC WRITEBACK HOST1 appdata synced back on handback
|
||||
#
|
||||
# ── MEDIA ──────────────────────────────────────────────────────────────────────────────────
|
||||
# MEDIA PERMISSIONS share list for media_shares_permissions.sh
|
||||
# MEDIA CLEANER folder lists for media_cleaner.sh
|
||||
#
|
||||
# ── MONITORS ───────────────────────────────────────────────────────────────────────────────
|
||||
# CERTIFICATE MONITOR domains checked for SSL expiry
|
||||
# SMART HEALTH drives to skip in SMART monitoring
|
||||
# ZFS REPORT pools to exclude from ZFS health report
|
||||
#
|
||||
# ── TRANSCODES ─────────────────────────────────────────────────────────────────────────────
|
||||
# TRANSCODES ramdisk size, thresholds, SSD path, server array
|
||||
#
|
||||
# ── ARR STACK ──────────────────────────────────────────────────────────────────────────────
|
||||
# DOWNLOADERS slskd, SABnzbd, qBittorrent credentials and URLs
|
||||
# LIDARR URL, API key, path map
|
||||
# SONARR URL, API key, path map
|
||||
# RADARR URL, API key, path map
|
||||
# ARR RECOVERY per-arr recovery toggles
|
||||
#
|
||||
# ── SYSTEM WATCHDOG ────────────────────────────────────────────────────────────────────────
|
||||
# SYSTEM WATCHDOG per-host check toggles and NIC configuration
|
||||
#
|
||||
# ── RESOURCE MANAGER ───────────────────────────────────────────────────────────────────────
|
||||
# RESOURCE MANAGER containers paused/stopped under memory pressure
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
# ==============================================================================================
|
||||
# ── IDENTITY & CONNECTIVITY ───────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Identity ━━━
|
||||
# HOST1 hostname lives in master.conf (not a credential — safe for all servers).
|
||||
# SSH key used for all server-to-server operations — rsync, failover container commands.
|
||||
# Must be in /root/.ssh/ and authorised in HOST2's /root/.ssh/authorized_keys.
|
||||
HOST1_SSH_KEY="/root/.ssh/gmer4lfe_rsync_automation"
|
||||
HOST1_OWNER="gmer4lfe"
|
||||
HOST1_OWNER_EMAIL="gmer4lfe@gmail.com"
|
||||
|
||||
# ━━━ Emby ━━━
|
||||
# Referenced by transcode_manager.sh, emby_session_report.sh, emby_database_repair.sh,
|
||||
# weekly_sync_maintenance.sh, and HOST1_TRANSCODE_SERVERS below.
|
||||
# API key: Emby Dashboard → API Keys → + New Key
|
||||
HOST1_EMBY_CONTAINER="Emby"
|
||||
HOST1_EMBY_URL="http://localhost:8096"
|
||||
HOST1_EMBY_API_KEY="0c27448d93a7431f9ac63569f7655829"
|
||||
|
||||
# ━━━ Jellyfin ━━━
|
||||
# API key: Jellyfin Dashboard → Administration → API Keys → + New Key
|
||||
HOST1_JELLYFIN_CONTAINER="Jellyfin"
|
||||
HOST1_JELLYFIN_URL="http://localhost:8095"
|
||||
HOST1_JELLYFIN_API_KEY="4e820e7df74c4933acec212b1996314e"
|
||||
|
||||
# ━━━ Gitea ━━━
|
||||
# Personal access token for gitea_ssh_setup.sh — registers this server's SSH public key
|
||||
# with Gitea so git operations use key auth instead of passwords.
|
||||
# Create in Gitea: Settings → Applications → Generate Token → scope: write:user
|
||||
HOST1_GITEA_API_TOKEN=""
|
||||
|
||||
# ━━━ Notifications ━━━
|
||||
# Discord webhook — leave blank to disable.
|
||||
# Per-host so HOST1 and HOST2 can post to different channels or only one server notifies.
|
||||
HOST1_DISCORD_WEBHOOK=""
|
||||
|
||||
# ━━━ Partnership ━━━
|
||||
# HOST1 is always the owner (source of truth) unless --transfer has been run.
|
||||
# See README-Partnership.md and master.conf PARTNERSHIP section for full lifecycle docs.
|
||||
|
||||
# Auth containers reconfigured on onboard/offboard.
|
||||
# Format: "ContainerName|WebUIPort"
|
||||
# On onboard → WebUI pointed at owner's Tailscale IP (mirror clicks NPM, gets owner's auth)
|
||||
# On offboard → WebUI pointed back at localhost
|
||||
HOST1_PARTNERSHIP_AUTH_WEBUIS=(
|
||||
"NginxProxyManager|81"
|
||||
"Lldap-Gmer4Lfe|17170"
|
||||
"Authelia|9091"
|
||||
"Authelia-Secondary|9092"
|
||||
)
|
||||
|
||||
# XML templates (from this server's templates-user/) pushed to mirror during onboard.
|
||||
# These become the mirror's active auth stack, backed by the rsync-synced appdata.
|
||||
# Update filename if Lldap is renamed to drop the host suffix.
|
||||
HOST1_PARTNERSHIP_AUTH_STACK=(
|
||||
# Dependencies first — Mariadb/Redis must be healthy before Authelia starts
|
||||
"my-Mariadb-Authelia.xml"
|
||||
"my-Mariadb-Authelia-Secondary.xml"
|
||||
"my-Redis-Authelia.xml"
|
||||
"my-Redis-Authelia-Secondary.xml"
|
||||
# Auth apps — deployed after their deps are confirmed healthy
|
||||
"my-Authelia.xml"
|
||||
"my-Authelia-Secondary.xml"
|
||||
"my-NginxProxyManager.xml"
|
||||
"my-Lldap-Gmer4Lfe.xml"
|
||||
# Source of truth — must be available on HOST2 independently of the auth stack
|
||||
"my-Gitea.xml"
|
||||
)
|
||||
|
||||
# XML templates pushed to mirror for the arr stack during onboard.
|
||||
# Deps (e.g. databases) first if any — same ordering rule as auth stack.
|
||||
HOST1_PARTNERSHIP_ARR_STACK=(
|
||||
# "my-Sonarr.xml"
|
||||
# "my-Radarr.xml"
|
||||
# "my-Lidarr.xml"
|
||||
# "my-Prowlarr.xml"
|
||||
# "my-Bazarr.xml"
|
||||
)
|
||||
|
||||
# Paths HOST2 should collect during the grace window after offboard.
|
||||
# Notified on offboard — no auto-deletion, HOST2 must collect manually within PARTNERSHIP_GRACE_HOURS.
|
||||
HOST1_PARTNERSHIP_MIRROR_BACKUPS=(
|
||||
# "/mnt/user/appdata-Fallback/Jayred365-Emby"
|
||||
)
|
||||
|
||||
# Containers parked on this server when partnership is active.
|
||||
# Stopped on onboard (owner deploys its stack instead), restarted on offboard.
|
||||
HOST1_PARTNERSHIP_OWN_CONTAINERS=(
|
||||
# "Emby"
|
||||
# "NginxProxyManager"
|
||||
)
|
||||
|
||||
# Emby admin provisioning — toggle is owner-only, credentials are per-host.
|
||||
# Owner enables/disables the feature. Each host sets the account they want on the shared Emby.
|
||||
# On onboard: owner reads mirror's HOST*_PARTNERSHIP_EMBY_ADMIN_* and creates that account.
|
||||
# On offboard: account is deleted. Username collision → onboard exits with error.
|
||||
HOST1_PARTNERSHIP_PROVISION_EMBY_ADMIN=false # owner controls whether Emby is shared
|
||||
HOST1_PARTNERSHIP_EMBY_PORT=8096
|
||||
HOST1_PARTNERSHIP_EMBY_ADMIN_USER="" # this server's desired Emby username
|
||||
HOST1_PARTNERSHIP_EMBY_ADMIN_PASS="" # this server's desired Emby password
|
||||
|
||||
# ==============================================================================================
|
||||
# ── RSYNC ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Daily Sync Shares ━━━
|
||||
# Shares HOST1 pushes to all other nodes every night (1am via daily_sync_maintenance.sh).
|
||||
# Mesh model: every node pushes every media share — no ownership, no mirrors.
|
||||
# arr_sync ensures all arr libraries converge (union). rsync spreads files (additive, no --delete).
|
||||
# arr_cleanup removes true orphans based on local arr state.
|
||||
# Any node can download content to any share — it propagates to all nodes on the next cycle.
|
||||
# Nextcloud is intentionally one-directional (HOST1→HOST2 offsite backup — not arr-managed).
|
||||
# Uses DEFAULT_RSYNC_OPTS from master.conf — no profile needed.
|
||||
# For shares needing container stops or custom options — add a profile in master.conf.
|
||||
HOST1_DAILY_SYNC_SHARES=(
|
||||
/mnt/user/Books
|
||||
/mnt/user/Intros
|
||||
/mnt/user/Kids_Movies
|
||||
/mnt/user/Kids_Tv_Shows
|
||||
/mnt/user/Movies
|
||||
/mnt/user/Music
|
||||
/mnt/user/Music_Videos
|
||||
/mnt/user/Nextcloud
|
||||
/mnt/user/stand-up_comedy
|
||||
/mnt/user/Sports
|
||||
/mnt/user/Tv_Shows
|
||||
/mnt/user/Anime_Shows-Old
|
||||
/mnt/user/Anime_Movies-Old
|
||||
/mnt/user/Anime_Movies
|
||||
/mnt/user/Anime_Shows
|
||||
)
|
||||
|
||||
# Personal encrypted shares — synced for offsite backup, independent of media shares.
|
||||
# ZFS encrypted at dataset level — remote receives encrypted blocks, cannot read content.
|
||||
# See README-Rsync_Setup.md for ZFS encryption setup before uncommenting.
|
||||
HOST1_PERSONAL_SHARES=(
|
||||
# /mnt/user/HOST1-Personal # uncomment after creating encrypted dataset
|
||||
)
|
||||
|
||||
# ━━━ Weekly Sync Shares ━━━
|
||||
# Appdata shares synced during the weekly maintenance window (Sunday 2:30am).
|
||||
# Containers stopped both sides before sync — full clean state guaranteed.
|
||||
# Profiles drive container stops, excludes, and options — configured in master.conf RSYNC section.
|
||||
# Order matters — Emby first (larger transfer), then Critical-Data (auth stack).
|
||||
HOST1_WEEKLY_SYNC_SHARES=(
|
||||
"/mnt/user/Media_Server/Emby" # emby profile — full clean mirror
|
||||
"/mnt/user/appdata-Fallback/Critical-Data" # critical-data profile — auth stack
|
||||
)
|
||||
|
||||
# ━━━ Intermediate Sync Shares ━━━
|
||||
# Shares synced every 4 hours by intermediate_sync_maintenance.sh.
|
||||
# Uses DEFAULT_RSYNC_OPTS (no --delete) — for sub-daily propagation of metadata or watch state.
|
||||
# Full media share sync stays in the daily window. Leave empty to skip mid-day rsync entirely.
|
||||
HOST1_INTERMEDIATE_SYNC_SHARES=(
|
||||
# Add shares here to enable mid-day rsync
|
||||
# Example: "/mnt/user/Emby_Metadata"
|
||||
)
|
||||
|
||||
# ━━━ Critical Sync Shares ━━━
|
||||
# Appdata shares synced every 30 minutes by critical_sync_maintenance.sh.
|
||||
# Format: "/path/to/share" or "/path/to/share|profile-name"
|
||||
# Order matters — Critical-Data first (auth stack), then Emby dirty sync.
|
||||
HOST1_CRITICAL_SYNC_SHARES=(
|
||||
"/mnt/user/appdata-Fallback/Critical-Data|critical-fallback" # auth dirty sync — stays running
|
||||
"/mnt/user/Media_Server/Emby|emby-fallback" # Emby dirty sync — stays running
|
||||
)
|
||||
|
||||
# ━━━ Backup Verify ━━━
|
||||
# Shares verified by backup_verify.sh — random file checksum comparison against remote.
|
||||
# Leave empty to use HOST1_DAILY_SYNC_SHARES automatically.
|
||||
# Sample size and minimum file size defined in master.conf.
|
||||
HOST1_BACKUP_VERIFY_SHARES=(
|
||||
# leave empty to use HOST1_DAILY_SYNC_SHARES automatically
|
||||
)
|
||||
|
||||
# ━━━ HOST1 Rsync Profile — host1-appdata ━━━
|
||||
# HOST1-specific appdata sync profile — extends the shared PROFILE_* arrays in master.conf.
|
||||
# Use for appdata unique to HOST1 (Organizrv2, VaultWarden, UptimeKuma etc.)
|
||||
# Shared appdata (auth stack, Emby) use dedicated profiles defined in master.conf.
|
||||
# Run manually: bash Rsync/rsync.sh /mnt/user/appdata-Fallback/HOST1-Appdata --profile=host1-appdata
|
||||
PROFILE_RSYNC_OPTS[host1-appdata]="-av --info=progress2 --bwlimit=${PROFILE_BW_LIMIT[host1-appdata]:-8000}"
|
||||
PROFILE_BW_LIMIT[host1-appdata]=8000
|
||||
PROFILE_RETRY_COUNT[host1-appdata]=3
|
||||
PROFILE_SLEEP[host1-appdata]=300
|
||||
PROFILE_CRITICAL_CONTAINER_NAMES[host1-appdata]="Organizrv2-Gmer4Lfe UptimeKuma-Gmer4Lfe VaultWarden-Gmer4Lfe"
|
||||
PROFILE_DELAYED_CONTAINERS[host1-appdata]=""
|
||||
PROFILE_CONTAINER_DELAY[host1-appdata]=5
|
||||
PROFILE_EXCLUDE_DIRS[host1-appdata]="logs *.tmp"
|
||||
|
||||
# ==============================================================================================
|
||||
# ── DOCKER ────────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Docker Daily Restart ━━━
|
||||
# Containers restarted every day via DAILY_MAINTENANCE_SCRIPTS.
|
||||
# Dispatcharr degrades over time without restart — daily is intentional, not just housekeeping.
|
||||
# Order matters — auth stack first, then media services.
|
||||
HOST1_DAILY_RESTART_CONTAINERS=(
|
||||
"NginxProxyManager"
|
||||
"Lldap-Gmer4Lfe"
|
||||
"Authelia"
|
||||
"Authelia-Secondary"
|
||||
"Dispatcharr-Iptv-Users"
|
||||
"Dispatcharr" # Live TV scheduler — degrades without daily restart
|
||||
"Dispatcharr-Basic"
|
||||
"ErsatzTV-Emby"
|
||||
"Slskd" # Soulseek connection drops after extended uptime; restart refreshes share index
|
||||
)
|
||||
|
||||
# ━━━ Docker Weekly Restart ━━━
|
||||
# Less critical services restarted weekly via WEEKLY_MAINTENANCE_SCRIPTS (Sunday 2:30am).
|
||||
# Containers already stopped for weekly sync — restart adds zero extra downtime.
|
||||
HOST1_WEEKLY_RESTART_CONTAINERS=(
|
||||
"NextCloud"
|
||||
"Organizrv2-Gmer4Lfe"
|
||||
"AdGuard-Home"
|
||||
"Immich-Gmer4Lfe"
|
||||
)
|
||||
|
||||
# ━━━ Docker Watchdog ━━━
|
||||
# Per-HOST1 container configuration for docker_watchdog.sh.
|
||||
# Shared thresholds and toggles live in master.conf.
|
||||
|
||||
# Memory hard limits in MB — immediate restart if exceeded.
|
||||
# Set at "container is clearly broken" not "container is busy".
|
||||
# 20GB=20480 18GB=18432 16GB=16384 12GB=12288 8GB=8192 4GB=4096 2GB=2048 1GB=1024
|
||||
declare -A HOST1_WATCHDOG_CONTAINERS=(
|
||||
["Emby"]=20480 # 20GB — large library + active transcodes
|
||||
["LidaTube"]=6144 # 6GB — memory leak over time
|
||||
["Tdarr"]=6144 # 6GB — encoding is memory intensive
|
||||
["Code-Server"]=1024 # 1GB — should never need more
|
||||
)
|
||||
|
||||
# HTTP health check URLs — checked every cycle, strike system before restart.
|
||||
# Only add containers with a meaningful web interface to check.
|
||||
declare -A HOST1_WATCHDOG_CONTAINER_URLS=(
|
||||
["Emby"]="http://localhost:8096"
|
||||
)
|
||||
|
||||
# Required containers — must always be running on HOST1.
|
||||
# Strike system before restart — repeated failures go on skip list, auto-clears on recovery.
|
||||
# Listed in dependency order — dependencies before dependents.
|
||||
HOST1_WATCHDOG_REQUIRED_CONTAINERS=(
|
||||
"NginxProxyManager"
|
||||
"Lldap-Gmer4Lfe"
|
||||
"Mariadb-Authelia"
|
||||
"Mariadb-Authelia-Secondary"
|
||||
"Redis-Authelia"
|
||||
"Redis-Authelia-Secondary"
|
||||
"Authelia"
|
||||
"Authelia-Secondary"
|
||||
)
|
||||
|
||||
# Containers to skip in Tier 2 global scan — legitimately stopped or frequently restarting.
|
||||
# Watchdog leaves these alone entirely — no restart attempts, no crash loop tracking.
|
||||
HOST1_WATCHDOG_SCAN_IGNORE=(
|
||||
"DashGate"
|
||||
"PIA-WG-Config-Generator"
|
||||
"Aperture"
|
||||
"Aperture-Kids"
|
||||
"pgvector-18-Apeture-Kids"
|
||||
"Pgvector18-Aperture"
|
||||
"emby-test" # broken test container (exit 127 — bad image)
|
||||
)
|
||||
|
||||
# Dependency ordering — skip restarting a container if its dependency is also down.
|
||||
# Prevents watchdog from restarting Authelia before Mariadb is back up.
|
||||
# SPACE-SEPARATED STRINGS — converted to array at runtime.
|
||||
declare -A HOST1_WATCHDOG_DEPENDENCIES=(
|
||||
["Authelia"]="Mariadb-Authelia Redis-Authelia"
|
||||
["Authelia-Secondary"]="Mariadb-Authelia Redis-Authelia-Secondary"
|
||||
["NextCloud"]="Postgres-NextCloud"
|
||||
)
|
||||
|
||||
# Per-container appdata growth suppress ceilings in MB.
|
||||
# ONLY needed in specific cases — growth rate detection covers all containers automatically.
|
||||
# Use this when a container legitimately has large stable data and you want to guarantee
|
||||
# it never triggers a false-positive growth alert. Growth warnings are suppressed while the
|
||||
# container's dir stays below this ceiling; above it, warnings resume as normal.
|
||||
# 50GB=51200 25GB=25600 20GB=20480 15GB=15360 10GB=10240 5GB=5120
|
||||
declare -A HOST1_WATCHDOG_APPDATA_SIZES=(
|
||||
["Tdarr"]="25600" # 25GB — transcode cache grows legitimately during active jobs
|
||||
["7dtd"]="20480" # 20GB — game server world data, expected to be large
|
||||
)
|
||||
|
||||
# ━━━ Network Watchdog ━━━
|
||||
# Host-specific connectivity config for Watchdogs/System/network_watchdog.sh.
|
||||
HOST1_NETWORK_WATCHDOG_DDNS_DOMAIN="gmer4lfe.com"
|
||||
HOST1_NETWORK_WATCHDOG_DDNS_CONTAINER="Gmer4Lfe.com"
|
||||
HOST1_NETWORK_WATCHDOG_NPM_URL="https://gmer4lfe.com"
|
||||
|
||||
# ━━━ Docker Network Connect ━━━
|
||||
# Containers connected to custom networks at array start by docker_network_connect.sh.
|
||||
# Networks created if they don't exist — idempotent, safe to re-run.
|
||||
HOST1_NETWORK_CONNECT_CONTAINERS=(
|
||||
"memcached"
|
||||
"Npm-CrowdSec"
|
||||
)
|
||||
|
||||
HOST1_NETWORK_CONNECT_NETWORKS=(
|
||||
"high-availability"
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── FALLBACK ──────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ DDNS ━━━
|
||||
# DDNS containers HOST1 manages — started/stopped by fallback.sh per DDNS absolute rules:
|
||||
# Internet loss → stop immediately
|
||||
# Failover → HOST2 starts HOST1's DDNS as Tier 1 (before any other containers)
|
||||
# Handback → stop HOST1's DDNS on HOST2 → rsync → start containers → start local DDNS last
|
||||
HOST1_DDNS_CONTAINERS=(
|
||||
"Gmer4Lfe.com"
|
||||
)
|
||||
|
||||
# ━━━ Internet Loss ━━━
|
||||
# Containers stopped immediately on HOST1 when internet connection is lost.
|
||||
# Prevents external-facing services from operating without connectivity.
|
||||
FALLBACK_HOST1_STOP_ON_NO_NET=(
|
||||
"Gmer4Lfe.com"
|
||||
)
|
||||
|
||||
# ━━━ Fallback Tiers — HOST1 Runs for HOST2 ━━━
|
||||
# Containers HOST1 starts when HOST2 goes down.
|
||||
# Tier 1 is always immediate — vital services cannot wait.
|
||||
# Higher tiers activate after HOST2_TIER*_DELAY minutes (set in host2.conf).
|
||||
FALLBACK_HOST1_COVERS_HOST2_TIER1=(
|
||||
"Gmer4Lfe.us"
|
||||
"VaultWarden-Jayred365"
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_COVERS_HOST2_TIER2=(
|
||||
# "container-placeholder"
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_COVERS_HOST2_TIER3=(
|
||||
# "container-placeholder"
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_COVERS_HOST2_TIER4=(
|
||||
# "container-placeholder"
|
||||
)
|
||||
|
||||
# ━━━ Tier Delays — HOST1's Containers on HOST2 ━━━
|
||||
# How long HOST1 must be down before each tier activates on HOST2 — in minutes.
|
||||
# Tier 1 is always immediate — no delay var needed.
|
||||
HOST1_TIER2_DELAY=240 # 4 hours — NextCloud, Immich
|
||||
HOST1_TIER3_DELAY=720 # 12 hours — secondary services
|
||||
HOST1_TIER4_DELAY=1440 # 24 hours — arrs + downloaders
|
||||
|
||||
# ━━━ Rsync Writeback — HOST1 Appdata Back on Handback ━━━
|
||||
# Syncs HOST1 appdata BACK to HOST1 when it comes back online after a failover.
|
||||
# Containers stopped before writeback — clean source, no competing writes.
|
||||
#
|
||||
# HOST1_TIER1_WRITEBACK_DELAY: short outages skip Tier 1 writeback — primary state
|
||||
# is more reliable than dirty sync data for brief outages.
|
||||
HOST1_TIER1_WRITEBACK_DELAY=60 # skip Emby writeback if outage under 1hr
|
||||
|
||||
# Tier 4 automatically syncs HOST1_DAILY_SYNC_SHARES — only list paths NOT in that array.
|
||||
FALLBACK_HOST1_WRITEBACK_TIER1=(
|
||||
"/mnt/user/Media_Server/Emby" # watch states built up during outage
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_WRITEBACK_TIER2=(
|
||||
"/mnt/user/appdata-Fallback/Important-Data" # NextCloud + Postgres — files added during outage
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_WRITEBACK_TIER3=(
|
||||
# "location-placeholder"
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_WRITEBACK_TIER4=(
|
||||
"/mnt/user/appdata-Fallback/Arrs_Stack" # arr databases — downloads queued during outage
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── MEDIA ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Media Permissions ━━━
|
||||
# Shares that media_shares_permissions.sh applies PERMISSIONS_MODE and PERMISSIONS_OWNER to.
|
||||
# Runs first in DAILY_MAINTENANCE_SCRIPTS — arr cleanup depends on correct ownership.
|
||||
HOST1_MEDIA_PERMISSION_SHARES=(
|
||||
/mnt/user/Anime_Movies
|
||||
/mnt/user/Anime_Movies-Old
|
||||
/mnt/user/Anime_Shows
|
||||
/mnt/user/Anime_Shows-Old
|
||||
/mnt/user/appcache
|
||||
/mnt/user/Books
|
||||
/mnt/user/Downloads
|
||||
/mnt/user/Games
|
||||
/mnt/user/Intros
|
||||
/mnt/user/Kids_Movies
|
||||
/mnt/user/Kids_Tv_Shows
|
||||
/mnt/user/Movie_Recordings
|
||||
/mnt/user/Movies
|
||||
/mnt/user/Music
|
||||
/mnt/user/Music_Videos
|
||||
/mnt/user/Photo
|
||||
/mnt/user/Sports
|
||||
/mnt/user/stand-up_comedy
|
||||
/mnt/user/Temp_Storage
|
||||
/mnt/user/Tv_Recordings
|
||||
/mnt/user/Tv_Shows
|
||||
/mnt/user/YouTube
|
||||
)
|
||||
|
||||
# ━━━ Media Cleaner ━━━
|
||||
# Folder lists for media_cleaner.sh — two profiles: anime and media.
|
||||
# File patterns shared across all servers — defined in master.conf.
|
||||
# Called via DAILY_MAINTENANCE_SCRIPTS. Run manually: Media/media_cleaner.sh anime|media
|
||||
HOST1_ANIME_CLEAN_FOLDERS=(
|
||||
/mnt/user/Anime_Movies
|
||||
/mnt/user/Anime_Movies-Old
|
||||
/mnt/user/Anime_Shows
|
||||
/mnt/user/Anime_Shows-Old
|
||||
)
|
||||
|
||||
HOST1_MEDIA_CLEAN_FOLDERS=(
|
||||
/mnt/user/Kids_Movies
|
||||
/mnt/user/Kids_Tv_Shows
|
||||
/mnt/user/Movies
|
||||
/mnt/user/Music
|
||||
/mnt/user/Sports
|
||||
/mnt/user/stand-up_comedy
|
||||
/mnt/user/Tv_Shows
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── MONITORS ──────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Certificate Monitor ━━━
|
||||
# Domains checked via direct openssl connection — not relying on NPM's certificate state.
|
||||
# Checks the actual certificate served, not what NPM thinks it has.
|
||||
# Thresholds (CERT_WARN_DAYS, CERT_CRIT_DAYS) defined in master.conf.
|
||||
HOST1_CERT_MONITOR_DOMAINS=(
|
||||
"Gmer4Lfe.com"
|
||||
"Gmer4Lfe.us"
|
||||
)
|
||||
|
||||
# ━━━ SMART Health ━━━
|
||||
# Drives skipped in SMART attribute monitoring — hardware is server-specific.
|
||||
# Thresholds read from dynamix.cfg at runtime — fallbacks in master.conf.
|
||||
HOST1_SMART_IGNORE_DRIVES=(
|
||||
"sda" # boot USB — SMART not meaningful on flash drives
|
||||
)
|
||||
|
||||
# ━━━ ZFS Report ━━━
|
||||
# Pools excluded from the weekly ZFS health report — reduces noise from single-disk array pools.
|
||||
# These are individual array disks formatted as ZFS — converting to XFS over time via unBalance.
|
||||
# Pool health thresholds defined in master.conf.
|
||||
HOST1_ZFS_REPORT_IGNORE_POOLS=(
|
||||
"disk5"
|
||||
"disk6"
|
||||
"disk8"
|
||||
"disk9"
|
||||
"disk10"
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── TRANSCODES ────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# Ramdisk size ceiling — tmpfs only uses RAM actually needed, not the full size upfront.
|
||||
# Real-world: 9 streams peaked at ~5.5GB — 10G gives generous headroom on 128GB RAM.
|
||||
HOST1_RAMDISK_SIZE="10G"
|
||||
|
||||
# Usage thresholds — coupled to HOST1_RAMDISK_SIZE, adjust all three together if size changes.
|
||||
# Hysteresis gap (8.5 - 7 = 1.5GB) prevents flip-flop between ramdisk and SSD.
|
||||
HOST1_RAMDISK_WARN_GB=8.5 # flip to SSD when ramdisk usage reaches this
|
||||
HOST1_RAMDISK_LOW_GB=7 # flip back to ramdisk when usage drops to this
|
||||
|
||||
# SSD fallback path — where transcodes land when ramdisk exceeds HOST1_RAMDISK_WARN_GB.
|
||||
# Must be on cache pool — array disks too slow for active transcode writes.
|
||||
HOST1_TRANSCODE_SSD="/mnt/cache/Temp_Storage/Emby/Transcodes/"
|
||||
|
||||
# Media servers sharing the ramdisk transcode space on HOST1.
|
||||
# Format: "ContainerName|URL|APIKey|Type" — Type: emby | jellyfin | plex
|
||||
# Entries with placeholder API keys are skipped automatically.
|
||||
# ⚠️ Tdarr does NOT belong here — keep Tdarr on SSD, not ramdisk.
|
||||
HOST1_TRANSCODE_SERVERS=(
|
||||
"${HOST1_EMBY_CONTAINER}|${HOST1_EMBY_URL}|${HOST1_EMBY_API_KEY}|emby"
|
||||
"${HOST1_JELLYFIN_CONTAINER}|${HOST1_JELLYFIN_URL}|${HOST1_JELLYFIN_API_KEY}|jellyfin"
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── ARR STACK ─────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Used by arr cleanup scripts and arrs_failed_stalled_recovery.sh.
|
||||
# detect_hosts() selects HOST1 vars when running on HOST1.
|
||||
#
|
||||
# PATH MAPS — container path → host path translation.
|
||||
# Arr stores file paths using container-internal paths — scripts need host paths to scan.
|
||||
# Add one entry per root folder in arr Settings → Media Management → Root Folders.
|
||||
|
||||
# ━━━ Downloaders ━━━
|
||||
# Used by downloaders_reset.sh — runs every 30min via CRITICAL_MAINTENANCE_SCRIPTS.
|
||||
# Clears stuck states, purges old history, prepares each client for a clean cycle.
|
||||
|
||||
# slskd — clears stuck searches, dead transfers, purges expired failed imports.
|
||||
# SLSKD_FAILED_IMPORTS_DIR: where Soularr moves albums Lidarr rejected.
|
||||
HOST1_SLSKD_URL="http://localhost:8980"
|
||||
HOST1_SLSKD_API_KEY="4bF9kL2mNpQrT7vWxYz1A3dEgHjKoRsU"
|
||||
HOST1_SLSKD_FAILED_IMPORTS_DIR="/mnt/user/Temp_Storage/Slskd/completed/failed_imports"
|
||||
|
||||
# SABnzbd
|
||||
HOST1_SABNZBD_URL="http://localhost:8180"
|
||||
HOST1_SABNZBD_API_KEY="8bfefe41d83b4d50883e32859b55ca9a"
|
||||
|
||||
# qBittorrent — deleteFiles=false removes torrent from qBit but leaves files on disk.
|
||||
# Radarr/Sonarr manage actual files independently.
|
||||
HOST1_QBIT_URL="http://localhost:8080"
|
||||
HOST1_QBIT_USERNAME="root"
|
||||
HOST1_QBIT_PASSWORD="Stay0utD!ck"
|
||||
|
||||
# ━━━ Lidarr — HOST1 only ━━━
|
||||
# HOST2 does not run Lidarr — HOST1_LIDARR_RECOVERY flag handles the exit cleanly.
|
||||
HOST1_LIDARR_URL="http://localhost:8686"
|
||||
HOST1_LIDARR_API_KEY="b2977e71ef074bc0a0529d9fcce3b2dc"
|
||||
HOST1_LIDARR_MUSIC_ROOT="/mnt/user/Music-New"
|
||||
HOST1_FANART_API_KEY="Yd7147a43b692df0b364b94dc47efb81"
|
||||
HOST1_LASTFM_API_KEY="be6dc169c33ae263e690c30d18b7491d"
|
||||
|
||||
declare -A HOST1_LIDARR_PATH_MAP=(
|
||||
["/ext-music"]="/mnt/user/Music-New"
|
||||
)
|
||||
|
||||
# ━━━ Sonarr ━━━
|
||||
HOST1_SONARR_URL="http://localhost:8989"
|
||||
HOST1_SONARR_API_KEY="130decd3db5b4c25afad64864cd03f9f"
|
||||
HOST1_SONARR_TV_ROOT="/mnt/user/Tv_Shows"
|
||||
|
||||
# Note: stand-up_comedy in both Sonarr + Radarr — TV specials and movie specials, one folder
|
||||
declare -A HOST1_SONARR_PATH_MAP=(
|
||||
["/tv"]="/mnt/user/Tv_Shows"
|
||||
["/ext-standup-comedy"]="/mnt/user/stand-up_comedy"
|
||||
["/kids tv"]="/mnt/user/Kids_Tv_Shows"
|
||||
["/ext-anime-shows"]="/mnt/user/Anime_Shows-Old"
|
||||
)
|
||||
|
||||
# ━━━ Radarr ━━━
|
||||
HOST1_RADARR_URL="http://localhost:7878"
|
||||
HOST1_RADARR_API_KEY="d43a3ec6cf1549edb4af0cc63f98b2a9"
|
||||
HOST1_TMDB_API_KEY="3dac5e2e49b5540472d2eafec4f01260"
|
||||
HOST1_RADARR_MOVIES_ROOT="/mnt/user/Movies"
|
||||
|
||||
# Note: stand-up_comedy in both Radarr + Sonarr — movie specials and TV specials, one folder
|
||||
declare -A HOST1_RADARR_PATH_MAP=(
|
||||
["/movies"]="/mnt/user/Movies"
|
||||
["/kids movies"]="/mnt/user/Kids_Movies"
|
||||
["/ext-stand-up-comedy"]="/mnt/user/stand-up_comedy"
|
||||
["/anime-movies"]="/mnt/user/Anime_Movies-Old"
|
||||
)
|
||||
|
||||
# ━━━ Arr Recovery Toggles ━━━
|
||||
# false = skip that arr on this host — exits cleanly without error
|
||||
HOST1_SONARR_RECOVERY=true
|
||||
HOST1_RADARR_RECOVERY=true
|
||||
HOST1_LIDARR_RECOVERY=true # HOST1 only — exits cleanly on HOST2
|
||||
|
||||
# ==============================================================================================
|
||||
# ── SYSTEM WATCHDOG ───────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Per-host check toggles and NIC config for system_watchdog.sh.
|
||||
# Aliased by detect_hosts() — script uses unprefixed SYS_WATCHDOG_* names.
|
||||
# HOST1: TR1950X 128GB — full media server, active transcoding, ZFS cache pools.
|
||||
#
|
||||
# Three-tier response — all critical checks enabled by default on HOST1:
|
||||
# Tier 1 (bypass strikes, reboot now): docker daemon, rootfs full, kernel oops, FD, /boot
|
||||
# Tier 2 (bypass strikes with OOM): RAM critical + OOM kills in cycle
|
||||
# Tier 3 (standard strike system): everything else
|
||||
#
|
||||
# RAM tiers, OOM limits, and reboot loop settings in master.conf System Watchdog section.
|
||||
|
||||
# ━━━ Primary NIC ━━━
|
||||
# Network interface for NIC state check — verify with: ip link show | grep "^[0-9]"
|
||||
# Common values: eth0, bond0, br0, eno1
|
||||
HOST1_SYS_WATCHDOG_NIC="eth0"
|
||||
|
||||
# ━━━ Tier 1 — Critical Checks ━━━
|
||||
# These bypass the strike system — a single hit triggers immediate reboot.
|
||||
# Disabling any of these is not recommended — they protect against acute system failure.
|
||||
|
||||
# Docker daemon unresponsive → try restart, reboot if restart fails.
|
||||
# Without a working daemon docker_watchdog.sh is blind and containers cannot be managed.
|
||||
HOST1_SYS_WATCHDOG_CHECK_DOCKER_DAEMON=true
|
||||
|
||||
# rootfs at critical threshold (ROOTFS_CRITICAL_PCT=99) → reboot immediately.
|
||||
# At 99% rootfs writes fail silently — logs stop, Docker errors out, SSH may stop working.
|
||||
# Standard 95% threshold still uses strike system — only 99%+ is critical tier.
|
||||
HOST1_SYS_WATCHDOG_CHECK_ROOTFS=true
|
||||
|
||||
# Kernel BUG/Oops in dmesg delta since last cycle → reboot immediately.
|
||||
# A kernel oops means the kernel ran with a corrupted state — stability is not guaranteed.
|
||||
HOST1_SYS_WATCHDOG_CHECK_KERNEL_OOPS=true
|
||||
|
||||
# File descriptor exhaustion at FD_CRITICAL_PCT (95%) → reboot immediately.
|
||||
# At 95% FD: new connections fail, Docker can't spawn processes, SSH drops.
|
||||
HOST1_SYS_WATCHDOG_CHECK_FD=true
|
||||
|
||||
# /boot read-only detected → reboot immediately.
|
||||
# Unexpected read-only /boot means state files and config writes are silently failing.
|
||||
# Fallback state, watchdog reboot log, and lock files all go stale silently.
|
||||
HOST1_SYS_WATCHDOG_CHECK_BOOT=true
|
||||
|
||||
# ━━━ Tier 2 — Urgent OOM Check ━━━
|
||||
# Bypass strikes when RAM is critically low AND OOM kill rate confirms active crisis.
|
||||
# Both must be enabled for Tier 2 bypass to function — disable either to always use strikes.
|
||||
|
||||
# Track kernel OOM kills each cycle via /proc/vmstat oom_kill delta.
|
||||
# Also provides diagnostic context in reboot messages (which processes were killed).
|
||||
HOST1_SYS_WATCHDOG_CHECK_OOM=true
|
||||
|
||||
# Free RAM check — required for both Tier 2 bypass and RAM tier logic.
|
||||
# Tiers: MEM_WARN_GB(10) → notify | MEM_SHUTDOWN_GB(6) → stop containers | MEM_GB(4) → strikes
|
||||
HOST1_SYS_WATCHDOG_CHECK_RAM=true
|
||||
|
||||
# ━━━ Tier 3 — Standard Checks (strike system) ━━━
|
||||
# Each check must fail SYS_WATCHDOG_STRIKE_LIMIT consecutive cycles before action is taken.
|
||||
# Single spikes are ignored — sustained problems trigger reboot.
|
||||
|
||||
# /var/log filesystem usage above SYS_WATCHDOG_LOG_PCT.
|
||||
# Log spam (Docker log storms, syslog loops) fills rootfs — indicates something broken.
|
||||
HOST1_SYS_WATCHDOG_CHECK_LOG=true
|
||||
|
||||
# ZFS ARC memory pinned above SYS_WATCHDOG_ARC_PINNED_PCT after cache drop.
|
||||
# Enabled on HOST1 — ZFS cache pools actively used. Disable on hosts without ZFS.
|
||||
HOST1_SYS_WATCHDOG_CHECK_ARC=true
|
||||
|
||||
# CPU temperature above SYS_WATCHDOG_CPU_TEMP_MAX (95°C).
|
||||
# Sustained high temp causes kernel throttling or panic. Requires lm-sensors.
|
||||
HOST1_SYS_WATCHDOG_CHECK_CPU_TEMP=true
|
||||
|
||||
# Load average above SYS_WATCHDOG_LOAD_MULTIPLIER × core count.
|
||||
# DISABLED on HOST1 — Tdarr and Emby cause legitimate sustained load spikes during encoding.
|
||||
# Enable on idle servers or adjust SYS_WATCHDOG_LOAD_MULTIPLIER if load is always high.
|
||||
HOST1_SYS_WATCHDOG_CHECK_LOAD=false
|
||||
|
||||
# Zombie process count above SYS_WATCHDOG_ZOMBIE_LIMIT (50).
|
||||
# Large zombie counts indicate serious process management failure — something is stuck.
|
||||
HOST1_SYS_WATCHDOG_CHECK_ZOMBIES=true
|
||||
|
||||
# Check docker_watchdog.sh persistent skip list — required containers on skip list.
|
||||
# Cross-watchdog coordination: if docker_watchdog gave up, system_watchdog escalates.
|
||||
# ENABLED — HOST1 fully built and operational, skip list is meaningful.
|
||||
HOST1_SYS_WATCHDOG_CHECK_CONTAINERS=true
|
||||
|
||||
# /tmp filesystem usage above SYS_WATCHDOG_TMP_PCT with auto-clear attempt.
|
||||
# Script tries to clear aged /tmp files first — only strikes if clear fails.
|
||||
# Lock files, rsync temp files, and Docker ops use /tmp — 100% means lock failures.
|
||||
HOST1_SYS_WATCHDOG_CHECK_TMP=true
|
||||
|
||||
# Array disk error count delta in /proc/mdstat — accumulating errors = disk failing now.
|
||||
# Triggers on SYS_WATCHDOG_MDSTAT_ERROR_LIMIT new errors in one cycle.
|
||||
HOST1_SYS_WATCHDOG_CHECK_MDSTAT=true
|
||||
|
||||
# Primary NIC operstate — detects NIC going down (physical or driver failure).
|
||||
# Uses HOST1_SYS_WATCHDOG_NIC above. Strike system — brief flaps don't trigger reboot.
|
||||
HOST1_SYS_WATCHDOG_CHECK_NETWORK=true
|
||||
|
||||
# sshd running check — attempts restart before escalating.
|
||||
# sshd down = no remote access. Script tries rc.sshd start, notifies, strikes on failure.
|
||||
HOST1_SYS_WATCHDOG_CHECK_SSHD=true
|
||||
|
||||
# Runaway process detection — single process above SYS_WATCHDOG_RUNAWAY_CPU_PCT sustained.
|
||||
# DISABLED — Tdarr encoding and Emby transcoding legitimately peg CPU for extended periods.
|
||||
# Enable only if HOST1 has no CPU-intensive workloads.
|
||||
HOST1_SYS_WATCHDOG_CHECK_RUNAWAY=false
|
||||
|
||||
# ==============================================================================================
|
||||
# ── RESOURCE MANAGER ──────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Containers to manage under pressure — see master.conf RW_CRITICAL_CONTAINERS for exclusions.
|
||||
|
||||
# docker pause at medium pressure (RAM < RW_RAM_MEDIUM_GB or load > medium threshold)
|
||||
# Suspended in-place — instant to pause/unpause, no state lost, no restart delay.
|
||||
HOST1_RW_PAUSE_CONTAINERS=(
|
||||
"Huntarr" # arr search automation — safe to suspend
|
||||
"Cleanuparr" # download cleanup — safe to suspend
|
||||
"Healarr" # arr health checks — safe to suspend
|
||||
"Soularr" # Slskd automation — background only
|
||||
"ChannelTube" # YouTube archiver — background only
|
||||
"Pinchflat" # YouTube archiver — background only
|
||||
)
|
||||
|
||||
# docker stop at hard pressure (RAM < RW_RAM_HARD_GB)
|
||||
# Full stop — these are optional/heavy services that free significant RAM when stopped.
|
||||
# resource_watchdog.sh restarts them when pressure fully clears (RAM >= RW_RAM_RECOVER_GB).
|
||||
HOST1_RW_STOP_CONTAINERS=(
|
||||
"LocalAI" # GPU/CPU heavy — largest RAM consumer when idle
|
||||
"7DaysToDie" # game server — optional
|
||||
"V-Rising" # game server — optional
|
||||
"Code-Server" # IDE — not needed during pressure events
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── AUTH STACK ────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Credentials for the Varaverk Auth Stack page (NPM, lldap, Authelia).
|
||||
|
||||
# ━━━ NginxProxyManager ━━━
|
||||
HOST1_NPM_URL="http://localhost:81"
|
||||
HOST1_NPM_USER="failedproxy2@gmail.com"
|
||||
HOST1_NPM_PASS="183134\$eanHess"
|
||||
|
||||
# ━━━ lldap ━━━
|
||||
HOST1_LLDAP_URL="http://localhost:17170"
|
||||
HOST1_LLDAP_USER="admin"
|
||||
HOST1_LLDAP_PASS="183134\$eanHess"
|
||||
|
||||
# ━━━ Authelia ━━━
|
||||
HOST1_AUTHELIA_CONFIG="/mnt/user/appdata-Fallback/Critical-Data/Authelia/configuration.yml"
|
||||
HOST1_AUTHELIA_CONTAINER="Authelia"
|
||||
|
||||
# ==============================================================================================
|
||||
# ──────────────────────── End Of HOST1 Variables ──────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
HOST1_UNRAID_API_KEY="1825c3a2e03ea5089974f4da2e171aa2d5907a1dea23cc479c33e492c8ff4dbb"
|
||||
@@ -0,0 +1,790 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ========================== HOST1 CONFIGURATION — unRAID-Gmer4Lfe ============================
|
||||
# ==============================================================================================
|
||||
# HOST1-specific variables — credentials, container names, share paths, failover lists.
|
||||
# Sourced after master.conf — values here extend shared profile arrays and add HOST1-specific
|
||||
# identity, credentials, and container configuration.
|
||||
#
|
||||
# Sparse checkout (git) ensures HOST2 never receives this file.
|
||||
# HOST2 never sees HOST1 credentials — clean separation at the file level.
|
||||
#
|
||||
# DO NOT put shared config here — thresholds, toggles, profiles belong in master.conf.
|
||||
# DO NOT put HOST2 variables here — they belong in host2.conf.
|
||||
#
|
||||
# ── INDEX ─────────────────────────────────────────────────────────────────────────────────────
|
||||
#
|
||||
# ── IDENTITY & CONNECTIVITY ────────────────────────────────────────────────────────────────
|
||||
# IDENTITY hostname, SSH key
|
||||
# EMBY container name, URL, API key
|
||||
# NOTIFICATIONS Discord webhook
|
||||
# PARTNERSHIP auth containers, backup paths
|
||||
#
|
||||
# ── RSYNC ──────────────────────────────────────────────────────────────────────────────────
|
||||
# DAILY SYNC SHARES media shares HOST1 owns and pushes to HOST2
|
||||
# WEEKLY SYNC SHARES appdata shares synced weekly (Sunday 2:30am)
|
||||
# CRITICAL SYNC SHARES appdata shares synced every 30 minutes
|
||||
# BACKUP VERIFY shares for checksum verification against remote
|
||||
# HOST1 RSYNC PROFILE host1-appdata profile for HOST1-specific appdata syncs
|
||||
#
|
||||
# ── DOCKER ─────────────────────────────────────────────────────────────────────────────────
|
||||
# DOCKER DAILY RESTART containers restarted daily
|
||||
# DOCKER WEEKLY RESTART containers restarted weekly
|
||||
# DOCKER WATCHDOG memory limits, health URLs, required containers, ignore list
|
||||
# DOCKER NETWORK CONNECT networks and containers for docker_network_connect.sh
|
||||
#
|
||||
# ── FALLBACK ───────────────────────────────────────────────────────────────────────────────
|
||||
# DDNS DDNS containers managed by HOST1
|
||||
# INTERNET LOSS containers stopped when internet is lost
|
||||
# FALLBACK TIERS what HOST1 runs for HOST2 per tier
|
||||
# TIER DELAYS how long HOST1 must be down before each tier activates on HOST2
|
||||
# RSYNC WRITEBACK HOST1 appdata synced back on handback
|
||||
#
|
||||
# ── MEDIA ──────────────────────────────────────────────────────────────────────────────────
|
||||
# MEDIA PERMISSIONS share list for media_shares_permissions.sh
|
||||
# MEDIA CLEANER folder lists for media_cleaner.sh
|
||||
#
|
||||
# ── MONITORS ───────────────────────────────────────────────────────────────────────────────
|
||||
# CERTIFICATE MONITOR domains checked for SSL expiry
|
||||
# SMART HEALTH drives to skip in SMART monitoring
|
||||
# ZFS REPORT pools to exclude from ZFS health report
|
||||
#
|
||||
# ── TRANSCODES ─────────────────────────────────────────────────────────────────────────────
|
||||
# TRANSCODES ramdisk size, thresholds, SSD path, server array
|
||||
#
|
||||
# ── ARR STACK ──────────────────────────────────────────────────────────────────────────────
|
||||
# DOWNLOADERS slskd, SABnzbd, qBittorrent credentials and URLs
|
||||
# LIDARR URL, API key, path map
|
||||
# SONARR URL, API key, path map
|
||||
# RADARR URL, API key, path map
|
||||
# ARR RECOVERY per-arr recovery toggles
|
||||
#
|
||||
# ── SYSTEM WATCHDOG ────────────────────────────────────────────────────────────────────────
|
||||
# SYSTEM WATCHDOG per-host check toggles and NIC configuration
|
||||
#
|
||||
# ── RESOURCE MANAGER ───────────────────────────────────────────────────────────────────────
|
||||
# RESOURCE MANAGER containers paused/stopped under memory pressure
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
# ==============================================================================================
|
||||
# ── IDENTITY & CONNECTIVITY ───────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Identity ━━━
|
||||
# HOST1 hostname lives in master.conf (not a credential — safe for all servers).
|
||||
# SSH key used for all server-to-server operations — rsync, failover container commands.
|
||||
# Must be in /root/.ssh/ and authorised in HOST2's /root/.ssh/authorized_keys.
|
||||
HOST1_SSH_KEY="/root/.ssh/gmer4lfe_rsync_automation"
|
||||
HOST1_OWNER="gmer4lfe"
|
||||
HOST1_OWNER_EMAIL="gmer4lfe@gmail.com"
|
||||
|
||||
# ━━━ Emby ━━━
|
||||
# Referenced by transcode_manager.sh, emby_session_report.sh, emby_database_repair.sh,
|
||||
# weekly_sync_maintenance.sh, and HOST1_TRANSCODE_SERVERS below.
|
||||
# API key: Emby Dashboard → API Keys → + New Key
|
||||
HOST1_EMBY_CONTAINER="Emby"
|
||||
HOST1_EMBY_URL="http://localhost:8096"
|
||||
HOST1_EMBY_API_KEY="0c27448d93a7431f9ac63569f7655829"
|
||||
|
||||
# ━━━ Jellyfin ━━━
|
||||
# API key: Jellyfin Dashboard → Administration → API Keys → + New Key
|
||||
HOST1_JELLYFIN_CONTAINER="Jellyfin"
|
||||
HOST1_JELLYFIN_URL="http://localhost:8095"
|
||||
HOST1_JELLYFIN_API_KEY="4e820e7df74c4933acec212b1996314e"
|
||||
|
||||
# ━━━ Gitea ━━━
|
||||
# Personal access token for gitea_ssh_setup.sh — registers this server's SSH public key
|
||||
# with Gitea so git operations use key auth instead of passwords.
|
||||
# Create in Gitea: Settings → Applications → Generate Token → scope: write:user
|
||||
HOST1_GITEA_API_TOKEN=""
|
||||
|
||||
# ━━━ Notifications ━━━
|
||||
# Discord webhook — leave blank to disable.
|
||||
# Per-host so HOST1 and HOST2 can post to different channels or only one server notifies.
|
||||
HOST1_DISCORD_WEBHOOK=""
|
||||
|
||||
# ━━━ Partnership ━━━
|
||||
# HOST1 is always the owner (source of truth) unless --transfer has been run.
|
||||
# See README-Partnership.md and master.conf PARTNERSHIP section for full lifecycle docs.
|
||||
|
||||
# Auth containers reconfigured on onboard/offboard.
|
||||
# Format: "ContainerName|WebUIPort"
|
||||
# On onboard → WebUI pointed at owner's Tailscale IP (mirror clicks NPM, gets owner's auth)
|
||||
# On offboard → WebUI pointed back at localhost
|
||||
HOST1_PARTNERSHIP_AUTH_WEBUIS=(
|
||||
"NginxProxyManager|81"
|
||||
"Lldap-Gmer4Lfe|17170"
|
||||
"Authelia|9091"
|
||||
"Authelia-Secondary|9092"
|
||||
)
|
||||
|
||||
# XML templates (from this server's templates-user/) pushed to mirror during onboard.
|
||||
# These become the mirror's active auth stack, backed by the rsync-synced appdata.
|
||||
# Update filename if Lldap is renamed to drop the host suffix.
|
||||
HOST1_PARTNERSHIP_AUTH_STACK=(
|
||||
# Dependencies first — Mariadb/Redis must be healthy before Authelia starts
|
||||
"my-Mariadb-Authelia.xml"
|
||||
"my-Mariadb-Authelia-Secondary.xml"
|
||||
"my-Redis-Authelia.xml"
|
||||
"my-Redis-Authelia-Secondary.xml"
|
||||
# Auth apps — deployed after their deps are confirmed healthy
|
||||
"my-Authelia.xml"
|
||||
"my-Authelia-Secondary.xml"
|
||||
"my-NginxProxyManager.xml"
|
||||
"my-Lldap-Gmer4Lfe.xml"
|
||||
# Source of truth — must be available on HOST2 independently of the auth stack
|
||||
"my-Gitea.xml"
|
||||
)
|
||||
|
||||
# XML templates pushed to mirror for the arr stack during onboard.
|
||||
# Deps (e.g. databases) first if any — same ordering rule as auth stack.
|
||||
HOST1_PARTNERSHIP_ARR_STACK=(
|
||||
# "my-Sonarr.xml"
|
||||
# "my-Radarr.xml"
|
||||
# "my-Lidarr.xml"
|
||||
# "my-Prowlarr.xml"
|
||||
# "my-Bazarr.xml"
|
||||
)
|
||||
|
||||
# Paths HOST2 should collect during the grace window after offboard.
|
||||
# Notified on offboard — no auto-deletion, HOST2 must collect manually within PARTNERSHIP_GRACE_HOURS.
|
||||
HOST1_PARTNERSHIP_MIRROR_BACKUPS=(
|
||||
# "/mnt/user/appdata-Fallback/Jayred365-Emby"
|
||||
)
|
||||
|
||||
# Containers parked on this server when partnership is active.
|
||||
# Stopped on onboard (owner deploys its stack instead), restarted on offboard.
|
||||
HOST1_PARTNERSHIP_OWN_CONTAINERS=(
|
||||
# "Emby"
|
||||
# "NginxProxyManager"
|
||||
)
|
||||
|
||||
# Emby admin provisioning — toggle is owner-only, credentials are per-host.
|
||||
# Owner enables/disables the feature. Each host sets the account they want on the shared Emby.
|
||||
# On onboard: owner reads mirror's HOST*_PARTNERSHIP_EMBY_ADMIN_* and creates that account.
|
||||
# On offboard: account is deleted. Username collision → onboard exits with error.
|
||||
HOST1_PARTNERSHIP_PROVISION_EMBY_ADMIN=false # owner controls whether Emby is shared
|
||||
HOST1_PARTNERSHIP_EMBY_PORT=8096
|
||||
HOST1_PARTNERSHIP_EMBY_ADMIN_USER="" # this server's desired Emby username
|
||||
HOST1_PARTNERSHIP_EMBY_ADMIN_PASS="" # this server's desired Emby password
|
||||
|
||||
# ==============================================================================================
|
||||
# ── RSYNC ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Daily Sync Shares ━━━
|
||||
# Shares HOST1 pushes to all other nodes every night (1am via daily_sync_maintenance.sh).
|
||||
# Mesh model: every node pushes every media share — no ownership, no mirrors.
|
||||
# arr_sync ensures all arr libraries converge (union). rsync spreads files (additive, no --delete).
|
||||
# arr_cleanup removes true orphans based on local arr state.
|
||||
# Any node can download content to any share — it propagates to all nodes on the next cycle.
|
||||
# Nextcloud is intentionally one-directional (HOST1→HOST2 offsite backup — not arr-managed).
|
||||
# Uses DEFAULT_RSYNC_OPTS from master.conf — no profile needed.
|
||||
# For shares needing container stops or custom options — add a profile in master.conf.
|
||||
HOST1_DAILY_SYNC_SHARES=(
|
||||
/mnt/user/Books
|
||||
/mnt/user/Intros
|
||||
/mnt/user/Kids_Movies
|
||||
/mnt/user/Kids_Tv_Shows
|
||||
/mnt/user/Movies
|
||||
/mnt/user/Music
|
||||
/mnt/user/Music_Videos
|
||||
/mnt/user/Nextcloud
|
||||
/mnt/user/stand-up_comedy
|
||||
/mnt/user/Sports
|
||||
/mnt/user/Tv_Shows
|
||||
/mnt/user/Anime_Shows-Old
|
||||
/mnt/user/Anime_Movies-Old
|
||||
/mnt/user/Anime_Movies
|
||||
/mnt/user/Anime_Shows
|
||||
)
|
||||
|
||||
# Personal encrypted shares — synced for offsite backup, independent of media shares.
|
||||
# ZFS encrypted at dataset level — remote receives encrypted blocks, cannot read content.
|
||||
# See README-Rsync_Setup.md for ZFS encryption setup before uncommenting.
|
||||
HOST1_PERSONAL_SHARES=(
|
||||
# /mnt/user/HOST1-Personal # uncomment after creating encrypted dataset
|
||||
)
|
||||
|
||||
# ━━━ Weekly Sync Shares ━━━
|
||||
# Appdata shares synced during the weekly maintenance window (Sunday 2:30am).
|
||||
# Containers stopped both sides before sync — full clean state guaranteed.
|
||||
# Profiles drive container stops, excludes, and options — configured in master.conf RSYNC section.
|
||||
# Order matters — Emby first (larger transfer), then Critical-Data (auth stack).
|
||||
HOST1_WEEKLY_SYNC_SHARES=(
|
||||
"/mnt/user/Media_Server/Emby" # emby profile — full clean mirror
|
||||
"/mnt/user/appdata-Fallback/Critical-Data" # critical-data profile — auth stack
|
||||
)
|
||||
|
||||
# ━━━ Intermediate Sync Shares ━━━
|
||||
# Shares synced every 4 hours by intermediate_sync_maintenance.sh.
|
||||
# Uses DEFAULT_RSYNC_OPTS (no --delete) — for sub-daily propagation of metadata or watch state.
|
||||
# Full media share sync stays in the daily window. Leave empty to skip mid-day rsync entirely.
|
||||
HOST1_INTERMEDIATE_SYNC_SHARES=(
|
||||
# Add shares here to enable mid-day rsync
|
||||
# Example: "/mnt/user/Emby_Metadata"
|
||||
)
|
||||
|
||||
# ━━━ Critical Sync Shares ━━━
|
||||
# Appdata shares synced every 30 minutes by critical_sync_maintenance.sh.
|
||||
# Format: "/path/to/share" or "/path/to/share|profile-name"
|
||||
# Order matters — Critical-Data first (auth stack), then Emby dirty sync.
|
||||
HOST1_CRITICAL_SYNC_SHARES=(
|
||||
"/mnt/user/appdata-Fallback/Critical-Data|critical-fallback" # auth dirty sync — stays running
|
||||
"/mnt/user/Media_Server/Emby|emby-fallback" # Emby dirty sync — stays running
|
||||
)
|
||||
|
||||
# ━━━ Backup Verify ━━━
|
||||
# Shares verified by backup_verify.sh — random file checksum comparison against remote.
|
||||
# Leave empty to use HOST1_DAILY_SYNC_SHARES automatically.
|
||||
# Sample size and minimum file size defined in master.conf.
|
||||
HOST1_BACKUP_VERIFY_SHARES=(
|
||||
# leave empty to use HOST1_DAILY_SYNC_SHARES automatically
|
||||
)
|
||||
|
||||
# ━━━ HOST1 Rsync Profile — host1-appdata ━━━
|
||||
# HOST1-specific appdata sync profile — extends the shared PROFILE_* arrays in master.conf.
|
||||
# Use for appdata unique to HOST1 (Organizrv2, VaultWarden, UptimeKuma etc.)
|
||||
# Shared appdata (auth stack, Emby) use dedicated profiles defined in master.conf.
|
||||
# Run manually: bash Rsync/rsync.sh /mnt/user/appdata-Fallback/HOST1-Appdata --profile=host1-appdata
|
||||
PROFILE_RSYNC_OPTS[host1-appdata]="-av --info=progress2 --bwlimit=${PROFILE_BW_LIMIT[host1-appdata]:-8000}"
|
||||
PROFILE_BW_LIMIT[host1-appdata]=8000
|
||||
PROFILE_RETRY_COUNT[host1-appdata]=3
|
||||
PROFILE_SLEEP[host1-appdata]=300
|
||||
PROFILE_CRITICAL_CONTAINER_NAMES[host1-appdata]="Organizrv2-Gmer4Lfe UptimeKuma-Gmer4Lfe VaultWarden-Gmer4Lfe"
|
||||
PROFILE_DELAYED_CONTAINERS[host1-appdata]=""
|
||||
PROFILE_CONTAINER_DELAY[host1-appdata]=5
|
||||
PROFILE_EXCLUDE_DIRS[host1-appdata]="logs *.tmp"
|
||||
|
||||
# ==============================================================================================
|
||||
# ── DOCKER ────────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Docker Daily Restart ━━━
|
||||
# Containers restarted every day via DAILY_MAINTENANCE_SCRIPTS.
|
||||
# Dispatcharr degrades over time without restart — daily is intentional, not just housekeeping.
|
||||
# Order matters — auth stack first, then media services.
|
||||
HOST1_DAILY_RESTART_CONTAINERS=(
|
||||
"NginxProxyManager"
|
||||
"Lldap-Gmer4Lfe"
|
||||
"Authelia"
|
||||
"Authelia-Secondary"
|
||||
"Dispatcharr-Iptv-Users"
|
||||
"Dispatcharr" # Live TV scheduler — degrades without daily restart
|
||||
"Dispatcharr-Basic"
|
||||
"ErsatzTV-Emby"
|
||||
"Slskd" # Soulseek connection drops after extended uptime; restart refreshes share index
|
||||
)
|
||||
|
||||
# ━━━ Docker Weekly Restart ━━━
|
||||
# Less critical services restarted weekly via WEEKLY_MAINTENANCE_SCRIPTS (Sunday 2:30am).
|
||||
# Containers already stopped for weekly sync — restart adds zero extra downtime.
|
||||
HOST1_WEEKLY_RESTART_CONTAINERS=(
|
||||
"NextCloud"
|
||||
"Organizrv2-Gmer4Lfe"
|
||||
"AdGuard-Home"
|
||||
"Immich-Gmer4Lfe"
|
||||
)
|
||||
|
||||
# ━━━ Docker Watchdog ━━━
|
||||
# Per-HOST1 container configuration for docker_watchdog.sh.
|
||||
# Shared thresholds and toggles live in master.conf.
|
||||
|
||||
# Memory hard limits in MB — immediate restart if exceeded.
|
||||
# Set at "container is clearly broken" not "container is busy".
|
||||
# 20GB=20480 18GB=18432 16GB=16384 12GB=12288 8GB=8192 4GB=4096 2GB=2048 1GB=1024
|
||||
declare -A HOST1_WATCHDOG_CONTAINERS=(
|
||||
["Emby"]=20480 # 20GB — large library + active transcodes
|
||||
["LidaTube"]=6144 # 6GB — memory leak over time
|
||||
["Tdarr"]=6144 # 6GB — encoding is memory intensive
|
||||
["Code-Server"]=1024 # 1GB — should never need more
|
||||
)
|
||||
|
||||
# HTTP health check URLs — checked every cycle, strike system before restart.
|
||||
# Only add containers with a meaningful web interface to check.
|
||||
declare -A HOST1_WATCHDOG_CONTAINER_URLS=(
|
||||
["Emby"]="http://localhost:8096"
|
||||
)
|
||||
|
||||
# Required containers — must always be running on HOST1.
|
||||
# Strike system before restart — repeated failures go on skip list, auto-clears on recovery.
|
||||
# Listed in dependency order — dependencies before dependents.
|
||||
HOST1_WATCHDOG_REQUIRED_CONTAINERS=(
|
||||
"NginxProxyManager"
|
||||
"Lldap-Gmer4Lfe"
|
||||
"Mariadb-Authelia"
|
||||
"Mariadb-Authelia-Secondary"
|
||||
"Redis-Authelia"
|
||||
"Redis-Authelia-Secondary"
|
||||
"Authelia"
|
||||
"Authelia-Secondary"
|
||||
)
|
||||
|
||||
# Containers to skip in Tier 2 global scan — legitimately stopped or frequently restarting.
|
||||
# Watchdog leaves these alone entirely — no restart attempts, no crash loop tracking.
|
||||
HOST1_WATCHDOG_SCAN_IGNORE=(
|
||||
"DashGate"
|
||||
"PIA-WG-Config-Generator"
|
||||
"Aperture"
|
||||
"Aperture-Kids"
|
||||
"pgvector-18-Apeture-Kids"
|
||||
"Pgvector18-Aperture"
|
||||
"emby-test" # broken test container (exit 127 — bad image)
|
||||
)
|
||||
|
||||
# Dependency ordering — skip restarting a container if its dependency is also down.
|
||||
# Prevents watchdog from restarting Authelia before Mariadb is back up.
|
||||
# SPACE-SEPARATED STRINGS — converted to array at runtime.
|
||||
declare -A HOST1_WATCHDOG_DEPENDENCIES=(
|
||||
["Authelia"]="Mariadb-Authelia Redis-Authelia"
|
||||
["Authelia-Secondary"]="Mariadb-Authelia Redis-Authelia-Secondary"
|
||||
["NextCloud"]="Postgres-NextCloud"
|
||||
)
|
||||
|
||||
# Per-container appdata growth suppress ceilings in MB.
|
||||
# ONLY needed in specific cases — growth rate detection covers all containers automatically.
|
||||
# Use this when a container legitimately has large stable data and you want to guarantee
|
||||
# it never triggers a false-positive growth alert. Growth warnings are suppressed while the
|
||||
# container's dir stays below this ceiling; above it, warnings resume as normal.
|
||||
# 50GB=51200 25GB=25600 20GB=20480 15GB=15360 10GB=10240 5GB=5120
|
||||
declare -A HOST1_WATCHDOG_APPDATA_SIZES=(
|
||||
["Tdarr"]="25600" # 25GB — transcode cache grows legitimately during active jobs
|
||||
["7dtd"]="20480" # 20GB — game server world data, expected to be large
|
||||
)
|
||||
|
||||
# ━━━ Network Watchdog ━━━
|
||||
# Host-specific connectivity config for Watchdogs/System/network_watchdog.sh.
|
||||
HOST1_NETWORK_WATCHDOG_DDNS_DOMAIN="gmer4lfe.com"
|
||||
HOST1_NETWORK_WATCHDOG_DDNS_CONTAINER="Gmer4Lfe.com"
|
||||
HOST1_NETWORK_WATCHDOG_NPM_URL="https://gmer4lfe.com"
|
||||
|
||||
# ━━━ Docker Network Connect ━━━
|
||||
# Containers connected to custom networks at array start by docker_network_connect.sh.
|
||||
# Networks created if they don't exist — idempotent, safe to re-run.
|
||||
HOST1_NETWORK_CONNECT_CONTAINERS=(
|
||||
"memcached"
|
||||
"Npm-CrowdSec"
|
||||
)
|
||||
|
||||
HOST1_NETWORK_CONNECT_NETWORKS=(
|
||||
"high-availability"
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── FALLBACK ──────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ DDNS ━━━
|
||||
# DDNS containers HOST1 manages — started/stopped by fallback.sh per DDNS absolute rules:
|
||||
# Internet loss → stop immediately
|
||||
# Failover → HOST2 starts HOST1's DDNS as Tier 1 (before any other containers)
|
||||
# Handback → stop HOST1's DDNS on HOST2 → rsync → start containers → start local DDNS last
|
||||
HOST1_DDNS_CONTAINERS=(
|
||||
"Gmer4Lfe.com"
|
||||
)
|
||||
|
||||
# ━━━ Internet Loss ━━━
|
||||
# Containers stopped immediately on HOST1 when internet connection is lost.
|
||||
# Prevents external-facing services from operating without connectivity.
|
||||
FALLBACK_HOST1_STOP_ON_NO_NET=(
|
||||
"Gmer4Lfe.com"
|
||||
)
|
||||
|
||||
# ━━━ Fallback Tiers — HOST1 Runs for HOST2 ━━━
|
||||
# Containers HOST1 starts when HOST2 goes down.
|
||||
# Tier 1 is always immediate — vital services cannot wait.
|
||||
# Higher tiers activate after HOST2_TIER*_DELAY minutes (set in host2.conf).
|
||||
FALLBACK_HOST1_COVERS_HOST2_TIER1=(
|
||||
"Gmer4Lfe.us"
|
||||
"VaultWarden-Jayred365"
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_COVERS_HOST2_TIER2=(
|
||||
# "container-placeholder"
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_COVERS_HOST2_TIER3=(
|
||||
# "container-placeholder"
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_COVERS_HOST2_TIER4=(
|
||||
# "container-placeholder"
|
||||
)
|
||||
|
||||
# ━━━ Tier Delays — HOST1's Containers on HOST2 ━━━
|
||||
# How long HOST1 must be down before each tier activates on HOST2 — in minutes.
|
||||
# Tier 1 is always immediate — no delay var needed.
|
||||
HOST1_TIER2_DELAY=240 # 4 hours — NextCloud, Immich
|
||||
HOST1_TIER3_DELAY=720 # 12 hours — secondary services
|
||||
HOST1_TIER4_DELAY=1440 # 24 hours — arrs + downloaders
|
||||
|
||||
# ━━━ Rsync Writeback — HOST1 Appdata Back on Handback ━━━
|
||||
# Syncs HOST1 appdata BACK to HOST1 when it comes back online after a failover.
|
||||
# Containers stopped before writeback — clean source, no competing writes.
|
||||
#
|
||||
# HOST1_TIER1_WRITEBACK_DELAY: short outages skip Tier 1 writeback — primary state
|
||||
# is more reliable than dirty sync data for brief outages.
|
||||
HOST1_TIER1_WRITEBACK_DELAY=60 # skip Emby writeback if outage under 1hr
|
||||
|
||||
# Tier 4 automatically syncs HOST1_DAILY_SYNC_SHARES — only list paths NOT in that array.
|
||||
FALLBACK_HOST1_WRITEBACK_TIER1=(
|
||||
"/mnt/user/Media_Server/Emby" # watch states built up during outage
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_WRITEBACK_TIER2=(
|
||||
"/mnt/user/appdata-Fallback/Important-Data" # NextCloud + Postgres — files added during outage
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_WRITEBACK_TIER3=(
|
||||
# "location-placeholder"
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_WRITEBACK_TIER4=(
|
||||
"/mnt/user/appdata-Fallback/Arrs_Stack" # arr databases — downloads queued during outage
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── MEDIA ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Media Permissions ━━━
|
||||
# Shares that media_shares_permissions.sh applies PERMISSIONS_MODE and PERMISSIONS_OWNER to.
|
||||
# Runs first in DAILY_MAINTENANCE_SCRIPTS — arr cleanup depends on correct ownership.
|
||||
HOST1_MEDIA_PERMISSION_SHARES=(
|
||||
/mnt/user/Anime_Movies
|
||||
/mnt/user/Anime_Movies-Old
|
||||
/mnt/user/Anime_Shows
|
||||
/mnt/user/Anime_Shows-Old
|
||||
/mnt/user/appcache
|
||||
/mnt/user/Books
|
||||
/mnt/user/Downloads
|
||||
/mnt/user/Games
|
||||
/mnt/user/Intros
|
||||
/mnt/user/Kids_Movies
|
||||
/mnt/user/Kids_Tv_Shows
|
||||
/mnt/user/Movie_Recordings
|
||||
/mnt/user/Movies
|
||||
/mnt/user/Music
|
||||
/mnt/user/Music_Videos
|
||||
/mnt/user/Photo
|
||||
/mnt/user/Sports
|
||||
/mnt/user/stand-up_comedy
|
||||
/mnt/user/Temp_Storage
|
||||
/mnt/user/Tv_Recordings
|
||||
/mnt/user/Tv_Shows
|
||||
/mnt/user/YouTube
|
||||
)
|
||||
|
||||
# ━━━ Media Cleaner ━━━
|
||||
# Folder lists for media_cleaner.sh — two profiles: anime and media.
|
||||
# File patterns shared across all servers — defined in master.conf.
|
||||
# Called via DAILY_MAINTENANCE_SCRIPTS. Run manually: Media/media_cleaner.sh anime|media
|
||||
HOST1_ANIME_CLEAN_FOLDERS=(
|
||||
/mnt/user/Anime_Movies
|
||||
/mnt/user/Anime_Movies-Old
|
||||
/mnt/user/Anime_Shows
|
||||
/mnt/user/Anime_Shows-Old
|
||||
)
|
||||
|
||||
HOST1_MEDIA_CLEAN_FOLDERS=(
|
||||
/mnt/user/Kids_Movies
|
||||
/mnt/user/Kids_Tv_Shows
|
||||
/mnt/user/Movies
|
||||
/mnt/user/Music
|
||||
/mnt/user/Sports
|
||||
/mnt/user/stand-up_comedy
|
||||
/mnt/user/Tv_Shows
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── MONITORS ──────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Certificate Monitor ━━━
|
||||
# Domains checked via direct openssl connection — not relying on NPM's certificate state.
|
||||
# Checks the actual certificate served, not what NPM thinks it has.
|
||||
# Thresholds (CERT_WARN_DAYS, CERT_CRIT_DAYS) defined in master.conf.
|
||||
HOST1_CERT_MONITOR_DOMAINS=(
|
||||
"Gmer4Lfe.com"
|
||||
"Gmer4Lfe.us"
|
||||
)
|
||||
|
||||
# ━━━ SMART Health ━━━
|
||||
# Drives skipped in SMART attribute monitoring — hardware is server-specific.
|
||||
# Thresholds read from dynamix.cfg at runtime — fallbacks in master.conf.
|
||||
HOST1_SMART_IGNORE_DRIVES=(
|
||||
"sda" # boot USB — SMART not meaningful on flash drives
|
||||
)
|
||||
|
||||
# ━━━ ZFS Report ━━━
|
||||
# Pools excluded from the weekly ZFS health report — reduces noise from single-disk array pools.
|
||||
# These are individual array disks formatted as ZFS — converting to XFS over time via unBalance.
|
||||
# Pool health thresholds defined in master.conf.
|
||||
HOST1_ZFS_REPORT_IGNORE_POOLS=(
|
||||
"disk5"
|
||||
"disk6"
|
||||
"disk8"
|
||||
"disk9"
|
||||
"disk10"
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── TRANSCODES ────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# Ramdisk size ceiling — tmpfs only uses RAM actually needed, not the full size upfront.
|
||||
# Real-world: 9 streams peaked at ~5.5GB — 10G gives generous headroom on 128GB RAM.
|
||||
HOST1_RAMDISK_SIZE="10G"
|
||||
|
||||
# Usage thresholds — coupled to HOST1_RAMDISK_SIZE, adjust all three together if size changes.
|
||||
# Hysteresis gap (8.5 - 7 = 1.5GB) prevents flip-flop between ramdisk and SSD.
|
||||
HOST1_RAMDISK_WARN_GB=8.5 # flip to SSD when ramdisk usage reaches this
|
||||
HOST1_RAMDISK_LOW_GB=7 # flip back to ramdisk when usage drops to this
|
||||
|
||||
# SSD fallback path — where transcodes land when ramdisk exceeds HOST1_RAMDISK_WARN_GB.
|
||||
# Must be on cache pool — array disks too slow for active transcode writes.
|
||||
HOST1_TRANSCODE_SSD="/mnt/cache/Temp_Storage/Emby/Transcodes/"
|
||||
|
||||
# Media servers sharing the ramdisk transcode space on HOST1.
|
||||
# Format: "ContainerName|URL|APIKey|Type" — Type: emby | jellyfin | plex
|
||||
# Entries with placeholder API keys are skipped automatically.
|
||||
# ⚠️ Tdarr does NOT belong here — keep Tdarr on SSD, not ramdisk.
|
||||
HOST1_TRANSCODE_SERVERS=(
|
||||
"${HOST1_EMBY_CONTAINER}|${HOST1_EMBY_URL}|${HOST1_EMBY_API_KEY}|emby"
|
||||
"${HOST1_JELLYFIN_CONTAINER}|${HOST1_JELLYFIN_URL}|${HOST1_JELLYFIN_API_KEY}|jellyfin"
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── ARR STACK ─────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Used by arr cleanup scripts and arrs_failed_stalled_recovery.sh.
|
||||
# detect_hosts() selects HOST1 vars when running on HOST1.
|
||||
#
|
||||
# PATH MAPS — container path → host path translation.
|
||||
# Arr stores file paths using container-internal paths — scripts need host paths to scan.
|
||||
# Add one entry per root folder in arr Settings → Media Management → Root Folders.
|
||||
|
||||
# ━━━ Downloaders ━━━
|
||||
# Used by downloaders_reset.sh — runs every 30min via CRITICAL_MAINTENANCE_SCRIPTS.
|
||||
# Clears stuck states, purges old history, prepares each client for a clean cycle.
|
||||
|
||||
# slskd — clears stuck searches, dead transfers, purges expired failed imports.
|
||||
# SLSKD_FAILED_IMPORTS_DIR: where Soularr moves albums Lidarr rejected.
|
||||
HOST1_SLSKD_URL="http://localhost:8980"
|
||||
HOST1_SLSKD_API_KEY="4bF9kL2mNpQrT7vWxYz1A3dEgHjKoRsU"
|
||||
HOST1_SLSKD_FAILED_IMPORTS_DIR="/mnt/user/Temp_Storage/Slskd/completed/failed_imports"
|
||||
|
||||
# SABnzbd
|
||||
HOST1_SABNZBD_URL="http://localhost:8180"
|
||||
HOST1_SABNZBD_API_KEY="8bfefe41d83b4d50883e32859b55ca9a"
|
||||
|
||||
# qBittorrent — deleteFiles=false removes torrent from qBit but leaves files on disk.
|
||||
# Radarr/Sonarr manage actual files independently.
|
||||
HOST1_QBIT_URL="http://localhost:8080"
|
||||
HOST1_QBIT_USERNAME="root"
|
||||
HOST1_QBIT_PASSWORD="Stay0utD!ck"
|
||||
|
||||
# ━━━ Lidarr — HOST1 only ━━━
|
||||
# HOST2 does not run Lidarr — HOST1_LIDARR_RECOVERY flag handles the exit cleanly.
|
||||
HOST1_LIDARR_URL="http://localhost:8686"
|
||||
HOST1_LIDARR_API_KEY="b2977e71ef074bc0a0529d9fcce3b2dc"
|
||||
HOST1_LIDARR_MUSIC_ROOT="/mnt/user/Music-New"
|
||||
HOST1_FANART_API_KEY="Yd7147a43b692df0b364b94dc47efb81"
|
||||
HOST1_LASTFM_API_KEY="be6dc169c33ae263e690c30d18b7491d"
|
||||
|
||||
declare -A HOST1_LIDARR_PATH_MAP=(
|
||||
["/ext-music"]="/mnt/user/Music-New"
|
||||
)
|
||||
|
||||
# ━━━ Sonarr ━━━
|
||||
HOST1_SONARR_URL="http://localhost:8989"
|
||||
HOST1_SONARR_API_KEY="130decd3db5b4c25afad64864cd03f9f"
|
||||
HOST1_SONARR_TV_ROOT="/mnt/user/Tv_Shows"
|
||||
|
||||
# Note: stand-up_comedy in both Sonarr + Radarr — TV specials and movie specials, one folder
|
||||
declare -A HOST1_SONARR_PATH_MAP=(
|
||||
["/tv"]="/mnt/user/Tv_Shows"
|
||||
["/ext-standup-comedy"]="/mnt/user/stand-up_comedy"
|
||||
["/kids tv"]="/mnt/user/Kids_Tv_Shows"
|
||||
["/ext-anime-shows"]="/mnt/user/Anime_Shows-Old"
|
||||
)
|
||||
|
||||
# ━━━ Radarr ━━━
|
||||
HOST1_RADARR_URL="http://localhost:7878"
|
||||
HOST1_RADARR_API_KEY="d43a3ec6cf1549edb4af0cc63f98b2a9"
|
||||
HOST1_TMDB_API_KEY="3dac5e2e49b5540472d2eafec4f01260"
|
||||
HOST1_RADARR_MOVIES_ROOT="/mnt/user/Movies"
|
||||
|
||||
# Note: stand-up_comedy in both Radarr + Sonarr — movie specials and TV specials, one folder
|
||||
declare -A HOST1_RADARR_PATH_MAP=(
|
||||
["/movies"]="/mnt/user/Movies"
|
||||
["/kids movies"]="/mnt/user/Kids_Movies"
|
||||
["/ext-stand-up-comedy"]="/mnt/user/stand-up_comedy"
|
||||
["/anime-movies"]="/mnt/user/Anime_Movies-Old"
|
||||
)
|
||||
|
||||
# ━━━ Arr Recovery Toggles ━━━
|
||||
# false = skip that arr on this host — exits cleanly without error
|
||||
HOST1_SONARR_RECOVERY=true
|
||||
HOST1_RADARR_RECOVERY=true
|
||||
HOST1_LIDARR_RECOVERY=true # HOST1 only — exits cleanly on HOST2
|
||||
|
||||
# ==============================================================================================
|
||||
# ── SYSTEM WATCHDOG ───────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Per-host check toggles and NIC config for system_watchdog.sh.
|
||||
# Aliased by detect_hosts() — script uses unprefixed SYS_WATCHDOG_* names.
|
||||
# HOST1: TR1950X 128GB — full media server, active transcoding, ZFS cache pools.
|
||||
#
|
||||
# Three-tier response — all critical checks enabled by default on HOST1:
|
||||
# Tier 1 (bypass strikes, reboot now): docker daemon, rootfs full, kernel oops, FD, /boot
|
||||
# Tier 2 (bypass strikes with OOM): RAM critical + OOM kills in cycle
|
||||
# Tier 3 (standard strike system): everything else
|
||||
#
|
||||
# RAM tiers, OOM limits, and reboot loop settings in master.conf System Watchdog section.
|
||||
|
||||
# ━━━ Primary NIC ━━━
|
||||
# Network interface for NIC state check — verify with: ip link show | grep "^[0-9]"
|
||||
# Common values: eth0, bond0, br0, eno1
|
||||
HOST1_SYS_WATCHDOG_NIC="eth0"
|
||||
|
||||
# ━━━ Tier 1 — Critical Checks ━━━
|
||||
# These bypass the strike system — a single hit triggers immediate reboot.
|
||||
# Disabling any of these is not recommended — they protect against acute system failure.
|
||||
|
||||
# Docker daemon unresponsive → try restart, reboot if restart fails.
|
||||
# Without a working daemon docker_watchdog.sh is blind and containers cannot be managed.
|
||||
HOST1_SYS_WATCHDOG_CHECK_DOCKER_DAEMON=true
|
||||
|
||||
# rootfs at critical threshold (ROOTFS_CRITICAL_PCT=99) → reboot immediately.
|
||||
# At 99% rootfs writes fail silently — logs stop, Docker errors out, SSH may stop working.
|
||||
# Standard 95% threshold still uses strike system — only 99%+ is critical tier.
|
||||
HOST1_SYS_WATCHDOG_CHECK_ROOTFS=true
|
||||
|
||||
# Kernel BUG/Oops in dmesg delta since last cycle → reboot immediately.
|
||||
# A kernel oops means the kernel ran with a corrupted state — stability is not guaranteed.
|
||||
HOST1_SYS_WATCHDOG_CHECK_KERNEL_OOPS=true
|
||||
|
||||
# File descriptor exhaustion at FD_CRITICAL_PCT (95%) → reboot immediately.
|
||||
# At 95% FD: new connections fail, Docker can't spawn processes, SSH drops.
|
||||
HOST1_SYS_WATCHDOG_CHECK_FD=true
|
||||
|
||||
# /boot read-only detected → reboot immediately.
|
||||
# Unexpected read-only /boot means state files and config writes are silently failing.
|
||||
# Fallback state, watchdog reboot log, and lock files all go stale silently.
|
||||
HOST1_SYS_WATCHDOG_CHECK_BOOT=true
|
||||
|
||||
# ━━━ Tier 2 — Urgent OOM Check ━━━
|
||||
# Bypass strikes when RAM is critically low AND OOM kill rate confirms active crisis.
|
||||
# Both must be enabled for Tier 2 bypass to function — disable either to always use strikes.
|
||||
|
||||
# Track kernel OOM kills each cycle via /proc/vmstat oom_kill delta.
|
||||
# Also provides diagnostic context in reboot messages (which processes were killed).
|
||||
HOST1_SYS_WATCHDOG_CHECK_OOM=true
|
||||
|
||||
# Free RAM check — required for both Tier 2 bypass and RAM tier logic.
|
||||
# Tiers: MEM_WARN_GB(10) → notify | MEM_SHUTDOWN_GB(6) → stop containers | MEM_GB(4) → strikes
|
||||
HOST1_SYS_WATCHDOG_CHECK_RAM=true
|
||||
|
||||
# ━━━ Tier 3 — Standard Checks (strike system) ━━━
|
||||
# Each check must fail SYS_WATCHDOG_STRIKE_LIMIT consecutive cycles before action is taken.
|
||||
# Single spikes are ignored — sustained problems trigger reboot.
|
||||
|
||||
# /var/log filesystem usage above SYS_WATCHDOG_LOG_PCT.
|
||||
# Log spam (Docker log storms, syslog loops) fills rootfs — indicates something broken.
|
||||
HOST1_SYS_WATCHDOG_CHECK_LOG=true
|
||||
|
||||
# ZFS ARC memory pinned above SYS_WATCHDOG_ARC_PINNED_PCT after cache drop.
|
||||
# Enabled on HOST1 — ZFS cache pools actively used. Disable on hosts without ZFS.
|
||||
HOST1_SYS_WATCHDOG_CHECK_ARC=true
|
||||
|
||||
# CPU temperature above SYS_WATCHDOG_CPU_TEMP_MAX (95°C).
|
||||
# Sustained high temp causes kernel throttling or panic. Requires lm-sensors.
|
||||
HOST1_SYS_WATCHDOG_CHECK_CPU_TEMP=true
|
||||
|
||||
# Load average above SYS_WATCHDOG_LOAD_MULTIPLIER × core count.
|
||||
# DISABLED on HOST1 — Tdarr and Emby cause legitimate sustained load spikes during encoding.
|
||||
# Enable on idle servers or adjust SYS_WATCHDOG_LOAD_MULTIPLIER if load is always high.
|
||||
HOST1_SYS_WATCHDOG_CHECK_LOAD=false
|
||||
|
||||
# Zombie process count above SYS_WATCHDOG_ZOMBIE_LIMIT (50).
|
||||
# Large zombie counts indicate serious process management failure — something is stuck.
|
||||
HOST1_SYS_WATCHDOG_CHECK_ZOMBIES=true
|
||||
|
||||
# Check docker_watchdog.sh persistent skip list — required containers on skip list.
|
||||
# Cross-watchdog coordination: if docker_watchdog gave up, system_watchdog escalates.
|
||||
# ENABLED — HOST1 fully built and operational, skip list is meaningful.
|
||||
HOST1_SYS_WATCHDOG_CHECK_CONTAINERS=true
|
||||
|
||||
# /tmp filesystem usage above SYS_WATCHDOG_TMP_PCT with auto-clear attempt.
|
||||
# Script tries to clear aged /tmp files first — only strikes if clear fails.
|
||||
# Lock files, rsync temp files, and Docker ops use /tmp — 100% means lock failures.
|
||||
HOST1_SYS_WATCHDOG_CHECK_TMP=true
|
||||
|
||||
# Array disk error count delta in /proc/mdstat — accumulating errors = disk failing now.
|
||||
# Triggers on SYS_WATCHDOG_MDSTAT_ERROR_LIMIT new errors in one cycle.
|
||||
HOST1_SYS_WATCHDOG_CHECK_MDSTAT=true
|
||||
|
||||
# Primary NIC operstate — detects NIC going down (physical or driver failure).
|
||||
# Uses HOST1_SYS_WATCHDOG_NIC above. Strike system — brief flaps don't trigger reboot.
|
||||
HOST1_SYS_WATCHDOG_CHECK_NETWORK=true
|
||||
|
||||
# sshd running check — attempts restart before escalating.
|
||||
# sshd down = no remote access. Script tries rc.sshd start, notifies, strikes on failure.
|
||||
HOST1_SYS_WATCHDOG_CHECK_SSHD=true
|
||||
|
||||
# Runaway process detection — single process above SYS_WATCHDOG_RUNAWAY_CPU_PCT sustained.
|
||||
# DISABLED — Tdarr encoding and Emby transcoding legitimately peg CPU for extended periods.
|
||||
# Enable only if HOST1 has no CPU-intensive workloads.
|
||||
HOST1_SYS_WATCHDOG_CHECK_RUNAWAY=false
|
||||
|
||||
# ==============================================================================================
|
||||
# ── RESOURCE MANAGER ──────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Containers to manage under pressure — see master.conf RW_CRITICAL_CONTAINERS for exclusions.
|
||||
|
||||
# docker pause at medium pressure (RAM < RW_RAM_MEDIUM_GB or load > medium threshold)
|
||||
# Suspended in-place — instant to pause/unpause, no state lost, no restart delay.
|
||||
HOST1_RW_PAUSE_CONTAINERS=(
|
||||
"Huntarr" # arr search automation — safe to suspend
|
||||
"Cleanuparr" # download cleanup — safe to suspend
|
||||
"Healarr" # arr health checks — safe to suspend
|
||||
"Soularr" # Slskd automation — background only
|
||||
"ChannelTube" # YouTube archiver — background only
|
||||
"Pinchflat" # YouTube archiver — background only
|
||||
)
|
||||
|
||||
# docker stop at hard pressure (RAM < RW_RAM_HARD_GB)
|
||||
# Full stop — these are optional/heavy services that free significant RAM when stopped.
|
||||
# resource_watchdog.sh restarts them when pressure fully clears (RAM >= RW_RAM_RECOVER_GB).
|
||||
HOST1_RW_STOP_CONTAINERS=(
|
||||
"LocalAI" # GPU/CPU heavy — largest RAM consumer when idle
|
||||
"7DaysToDie" # game server — optional
|
||||
"V-Rising" # game server — optional
|
||||
"Code-Server" # IDE — not needed during pressure events
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── AUTH STACK ────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Credentials for the Varaverk Auth Stack page (NPM, lldap, Authelia).
|
||||
|
||||
# ━━━ NginxProxyManager ━━━
|
||||
HOST1_NPM_URL="http://localhost:7818"
|
||||
HOST1_NPM_USER="failedproxy2@gmail.com"
|
||||
HOST1_NPM_PASS="183134\$eanHess"
|
||||
|
||||
# ━━━ lldap ━━━
|
||||
HOST1_LLDAP_URL="http://localhost:17170"
|
||||
HOST1_LLDAP_USER="admin"
|
||||
HOST1_LLDAP_PASS="183134\$eanHess"
|
||||
|
||||
# ━━━ Authelia ━━━
|
||||
HOST1_AUTHELIA_CONFIG="/mnt/user/appdata-Fallback/Critical-Data/Authelia/configuration.yml"
|
||||
HOST1_AUTHELIA_CONTAINER="Authelia"
|
||||
|
||||
# ==============================================================================================
|
||||
# ──────────────────────── End Of HOST1 Variables ──────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
HOST1_UNRAID_API_KEY="1825c3a2e03ea5089974f4da2e171aa2d5907a1dea23cc479c33e492c8ff4dbb"
|
||||
@@ -0,0 +1,790 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ========================== HOST1 CONFIGURATION — unRAID-Gmer4Lfe ============================
|
||||
# ==============================================================================================
|
||||
# HOST1-specific variables — credentials, container names, share paths, failover lists.
|
||||
# Sourced after master.conf — values here extend shared profile arrays and add HOST1-specific
|
||||
# identity, credentials, and container configuration.
|
||||
#
|
||||
# Sparse checkout (git) ensures HOST2 never receives this file.
|
||||
# HOST2 never sees HOST1 credentials — clean separation at the file level.
|
||||
#
|
||||
# DO NOT put shared config here — thresholds, toggles, profiles belong in master.conf.
|
||||
# DO NOT put HOST2 variables here — they belong in host2.conf.
|
||||
#
|
||||
# ── INDEX ─────────────────────────────────────────────────────────────────────────────────────
|
||||
#
|
||||
# ── IDENTITY & CONNECTIVITY ────────────────────────────────────────────────────────────────
|
||||
# IDENTITY hostname, SSH key
|
||||
# EMBY container name, URL, API key
|
||||
# NOTIFICATIONS Discord webhook
|
||||
# PARTNERSHIP auth containers, backup paths
|
||||
#
|
||||
# ── RSYNC ──────────────────────────────────────────────────────────────────────────────────
|
||||
# DAILY SYNC SHARES media shares HOST1 owns and pushes to HOST2
|
||||
# WEEKLY SYNC SHARES appdata shares synced weekly (Sunday 2:30am)
|
||||
# CRITICAL SYNC SHARES appdata shares synced every 30 minutes
|
||||
# BACKUP VERIFY shares for checksum verification against remote
|
||||
# HOST1 RSYNC PROFILE host1-appdata profile for HOST1-specific appdata syncs
|
||||
#
|
||||
# ── DOCKER ─────────────────────────────────────────────────────────────────────────────────
|
||||
# DOCKER DAILY RESTART containers restarted daily
|
||||
# DOCKER WEEKLY RESTART containers restarted weekly
|
||||
# DOCKER WATCHDOG memory limits, health URLs, required containers, ignore list
|
||||
# DOCKER NETWORK CONNECT networks and containers for docker_network_connect.sh
|
||||
#
|
||||
# ── FALLBACK ───────────────────────────────────────────────────────────────────────────────
|
||||
# DDNS DDNS containers managed by HOST1
|
||||
# INTERNET LOSS containers stopped when internet is lost
|
||||
# FALLBACK TIERS what HOST1 runs for HOST2 per tier
|
||||
# TIER DELAYS how long HOST1 must be down before each tier activates on HOST2
|
||||
# RSYNC WRITEBACK HOST1 appdata synced back on handback
|
||||
#
|
||||
# ── MEDIA ──────────────────────────────────────────────────────────────────────────────────
|
||||
# MEDIA PERMISSIONS share list for media_shares_permissions.sh
|
||||
# MEDIA CLEANER folder lists for media_cleaner.sh
|
||||
#
|
||||
# ── MONITORS ───────────────────────────────────────────────────────────────────────────────
|
||||
# CERTIFICATE MONITOR domains checked for SSL expiry
|
||||
# SMART HEALTH drives to skip in SMART monitoring
|
||||
# ZFS REPORT pools to exclude from ZFS health report
|
||||
#
|
||||
# ── TRANSCODES ─────────────────────────────────────────────────────────────────────────────
|
||||
# TRANSCODES ramdisk size, thresholds, SSD path, server array
|
||||
#
|
||||
# ── ARR STACK ──────────────────────────────────────────────────────────────────────────────
|
||||
# DOWNLOADERS slskd, SABnzbd, qBittorrent credentials and URLs
|
||||
# LIDARR URL, API key, path map
|
||||
# SONARR URL, API key, path map
|
||||
# RADARR URL, API key, path map
|
||||
# ARR RECOVERY per-arr recovery toggles
|
||||
#
|
||||
# ── SYSTEM WATCHDOG ────────────────────────────────────────────────────────────────────────
|
||||
# SYSTEM WATCHDOG per-host check toggles and NIC configuration
|
||||
#
|
||||
# ── RESOURCE MANAGER ───────────────────────────────────────────────────────────────────────
|
||||
# RESOURCE MANAGER containers paused/stopped under memory pressure
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
# ==============================================================================================
|
||||
# ── IDENTITY & CONNECTIVITY ───────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Identity ━━━
|
||||
# HOST1 hostname lives in master.conf (not a credential — safe for all servers).
|
||||
# SSH key used for all server-to-server operations — rsync, failover container commands.
|
||||
# Must be in /root/.ssh/ and authorised in HOST2's /root/.ssh/authorized_keys.
|
||||
HOST1_SSH_KEY="/root/.ssh/gmer4lfe_rsync_automation"
|
||||
HOST1_OWNER="gmer4lfe"
|
||||
HOST1_OWNER_EMAIL="gmer4lfe@gmail.com"
|
||||
|
||||
# ━━━ Emby ━━━
|
||||
# Referenced by transcode_manager.sh, emby_session_report.sh, emby_database_repair.sh,
|
||||
# weekly_sync_maintenance.sh, and HOST1_TRANSCODE_SERVERS below.
|
||||
# API key: Emby Dashboard → API Keys → + New Key
|
||||
HOST1_EMBY_CONTAINER="Emby"
|
||||
HOST1_EMBY_URL="http://localhost:8096"
|
||||
HOST1_EMBY_API_KEY="0c27448d93a7431f9ac63569f7655829"
|
||||
|
||||
# ━━━ Jellyfin ━━━
|
||||
# API key: Jellyfin Dashboard → Administration → API Keys → + New Key
|
||||
HOST1_JELLYFIN_CONTAINER="Jellyfin"
|
||||
HOST1_JELLYFIN_URL="http://localhost:8095"
|
||||
HOST1_JELLYFIN_API_KEY="4e820e7df74c4933acec212b1996314e"
|
||||
|
||||
# ━━━ Gitea ━━━
|
||||
# Personal access token for gitea_ssh_setup.sh — registers this server's SSH public key
|
||||
# with Gitea so git operations use key auth instead of passwords.
|
||||
# Create in Gitea: Settings → Applications → Generate Token → scope: write:user
|
||||
HOST1_GITEA_API_TOKEN=""
|
||||
|
||||
# ━━━ Notifications ━━━
|
||||
# Discord webhook — leave blank to disable.
|
||||
# Per-host so HOST1 and HOST2 can post to different channels or only one server notifies.
|
||||
HOST1_DISCORD_WEBHOOK=""
|
||||
|
||||
# ━━━ Partnership ━━━
|
||||
# HOST1 is always the owner (source of truth) unless --transfer has been run.
|
||||
# See README-Partnership.md and master.conf PARTNERSHIP section for full lifecycle docs.
|
||||
|
||||
# Auth containers reconfigured on onboard/offboard.
|
||||
# Format: "ContainerName|WebUIPort"
|
||||
# On onboard → WebUI pointed at owner's Tailscale IP (mirror clicks NPM, gets owner's auth)
|
||||
# On offboard → WebUI pointed back at localhost
|
||||
HOST1_PARTNERSHIP_AUTH_WEBUIS=(
|
||||
"NginxProxyManager|81"
|
||||
"Lldap-Gmer4Lfe|17170"
|
||||
"Authelia|9091"
|
||||
"Authelia-Secondary|9092"
|
||||
)
|
||||
|
||||
# XML templates (from this server's templates-user/) pushed to mirror during onboard.
|
||||
# These become the mirror's active auth stack, backed by the rsync-synced appdata.
|
||||
# Update filename if Lldap is renamed to drop the host suffix.
|
||||
HOST1_PARTNERSHIP_AUTH_STACK=(
|
||||
# Dependencies first — Mariadb/Redis must be healthy before Authelia starts
|
||||
"my-Mariadb-Authelia.xml"
|
||||
"my-Mariadb-Authelia-Secondary.xml"
|
||||
"my-Redis-Authelia.xml"
|
||||
"my-Redis-Authelia-Secondary.xml"
|
||||
# Auth apps — deployed after their deps are confirmed healthy
|
||||
"my-Authelia.xml"
|
||||
"my-Authelia-Secondary.xml"
|
||||
"my-NginxProxyManager.xml"
|
||||
"my-Lldap-Gmer4Lfe.xml"
|
||||
# Source of truth — must be available on HOST2 independently of the auth stack
|
||||
"my-Gitea.xml"
|
||||
)
|
||||
|
||||
# XML templates pushed to mirror for the arr stack during onboard.
|
||||
# Deps (e.g. databases) first if any — same ordering rule as auth stack.
|
||||
HOST1_PARTNERSHIP_ARR_STACK=(
|
||||
# "my-Sonarr.xml"
|
||||
# "my-Radarr.xml"
|
||||
# "my-Lidarr.xml"
|
||||
# "my-Prowlarr.xml"
|
||||
# "my-Bazarr.xml"
|
||||
)
|
||||
|
||||
# Paths HOST2 should collect during the grace window after offboard.
|
||||
# Notified on offboard — no auto-deletion, HOST2 must collect manually within PARTNERSHIP_GRACE_HOURS.
|
||||
HOST1_PARTNERSHIP_MIRROR_BACKUPS=(
|
||||
# "/mnt/user/appdata-Fallback/Jayred365-Emby"
|
||||
)
|
||||
|
||||
# Containers parked on this server when partnership is active.
|
||||
# Stopped on onboard (owner deploys its stack instead), restarted on offboard.
|
||||
HOST1_PARTNERSHIP_OWN_CONTAINERS=(
|
||||
# "Emby"
|
||||
# "NginxProxyManager"
|
||||
)
|
||||
|
||||
# Emby admin provisioning — toggle is owner-only, credentials are per-host.
|
||||
# Owner enables/disables the feature. Each host sets the account they want on the shared Emby.
|
||||
# On onboard: owner reads mirror's HOST*_PARTNERSHIP_EMBY_ADMIN_* and creates that account.
|
||||
# On offboard: account is deleted. Username collision → onboard exits with error.
|
||||
HOST1_PARTNERSHIP_PROVISION_EMBY_ADMIN=false # owner controls whether Emby is shared
|
||||
HOST1_PARTNERSHIP_EMBY_PORT=8096
|
||||
HOST1_PARTNERSHIP_EMBY_ADMIN_USER="" # this server's desired Emby username
|
||||
HOST1_PARTNERSHIP_EMBY_ADMIN_PASS="" # this server's desired Emby password
|
||||
|
||||
# ==============================================================================================
|
||||
# ── RSYNC ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Daily Sync Shares ━━━
|
||||
# Shares HOST1 pushes to all other nodes every night (1am via daily_sync_maintenance.sh).
|
||||
# Mesh model: every node pushes every media share — no ownership, no mirrors.
|
||||
# arr_sync ensures all arr libraries converge (union). rsync spreads files (additive, no --delete).
|
||||
# arr_cleanup removes true orphans based on local arr state.
|
||||
# Any node can download content to any share — it propagates to all nodes on the next cycle.
|
||||
# Nextcloud is intentionally one-directional (HOST1→HOST2 offsite backup — not arr-managed).
|
||||
# Uses DEFAULT_RSYNC_OPTS from master.conf — no profile needed.
|
||||
# For shares needing container stops or custom options — add a profile in master.conf.
|
||||
HOST1_DAILY_SYNC_SHARES=(
|
||||
/mnt/user/Books
|
||||
/mnt/user/Intros
|
||||
/mnt/user/Kids_Movies
|
||||
/mnt/user/Kids_Tv_Shows
|
||||
/mnt/user/Movies
|
||||
/mnt/user/Music
|
||||
/mnt/user/Music_Videos
|
||||
/mnt/user/Nextcloud
|
||||
/mnt/user/stand-up_comedy
|
||||
/mnt/user/Sports
|
||||
/mnt/user/Tv_Shows
|
||||
/mnt/user/Anime_Shows-Old
|
||||
/mnt/user/Anime_Movies-Old
|
||||
/mnt/user/Anime_Movies
|
||||
/mnt/user/Anime_Shows
|
||||
)
|
||||
|
||||
# Personal encrypted shares — synced for offsite backup, independent of media shares.
|
||||
# ZFS encrypted at dataset level — remote receives encrypted blocks, cannot read content.
|
||||
# See README-Rsync_Setup.md for ZFS encryption setup before uncommenting.
|
||||
HOST1_PERSONAL_SHARES=(
|
||||
# /mnt/user/HOST1-Personal # uncomment after creating encrypted dataset
|
||||
)
|
||||
|
||||
# ━━━ Weekly Sync Shares ━━━
|
||||
# Appdata shares synced during the weekly maintenance window (Sunday 2:30am).
|
||||
# Containers stopped both sides before sync — full clean state guaranteed.
|
||||
# Profiles drive container stops, excludes, and options — configured in master.conf RSYNC section.
|
||||
# Order matters — Emby first (larger transfer), then Critical-Data (auth stack).
|
||||
HOST1_WEEKLY_SYNC_SHARES=(
|
||||
"/mnt/user/Media_Server/Emby" # emby profile — full clean mirror
|
||||
"/mnt/user/appdata-Fallback/Critical-Data" # critical-data profile — auth stack
|
||||
)
|
||||
|
||||
# ━━━ Intermediate Sync Shares ━━━
|
||||
# Shares synced every 4 hours by intermediate_sync_maintenance.sh.
|
||||
# Uses DEFAULT_RSYNC_OPTS (no --delete) — for sub-daily propagation of metadata or watch state.
|
||||
# Full media share sync stays in the daily window. Leave empty to skip mid-day rsync entirely.
|
||||
HOST1_INTERMEDIATE_SYNC_SHARES=(
|
||||
# Add shares here to enable mid-day rsync
|
||||
# Example: "/mnt/user/Emby_Metadata"
|
||||
)
|
||||
|
||||
# ━━━ Critical Sync Shares ━━━
|
||||
# Appdata shares synced every 30 minutes by critical_sync_maintenance.sh.
|
||||
# Format: "/path/to/share" or "/path/to/share|profile-name"
|
||||
# Order matters — Critical-Data first (auth stack), then Emby dirty sync.
|
||||
HOST1_CRITICAL_SYNC_SHARES=(
|
||||
"/mnt/user/appdata-Fallback/Critical-Data|critical-fallback" # auth dirty sync — stays running
|
||||
"/mnt/user/Media_Server/Emby|emby-fallback" # Emby dirty sync — stays running
|
||||
)
|
||||
|
||||
# ━━━ Backup Verify ━━━
|
||||
# Shares verified by backup_verify.sh — random file checksum comparison against remote.
|
||||
# Leave empty to use HOST1_DAILY_SYNC_SHARES automatically.
|
||||
# Sample size and minimum file size defined in master.conf.
|
||||
HOST1_BACKUP_VERIFY_SHARES=(
|
||||
# leave empty to use HOST1_DAILY_SYNC_SHARES automatically
|
||||
)
|
||||
|
||||
# ━━━ HOST1 Rsync Profile — host1-appdata ━━━
|
||||
# HOST1-specific appdata sync profile — extends the shared PROFILE_* arrays in master.conf.
|
||||
# Use for appdata unique to HOST1 (Organizrv2, VaultWarden, UptimeKuma etc.)
|
||||
# Shared appdata (auth stack, Emby) use dedicated profiles defined in master.conf.
|
||||
# Run manually: bash Rsync/rsync.sh /mnt/user/appdata-Fallback/HOST1-Appdata --profile=host1-appdata
|
||||
PROFILE_RSYNC_OPTS[host1-appdata]="-av --info=progress2 --bwlimit=${PROFILE_BW_LIMIT[host1-appdata]:-8000}"
|
||||
PROFILE_BW_LIMIT[host1-appdata]=8000
|
||||
PROFILE_RETRY_COUNT[host1-appdata]=3
|
||||
PROFILE_SLEEP[host1-appdata]=300
|
||||
PROFILE_CRITICAL_CONTAINER_NAMES[host1-appdata]="Organizrv2-Gmer4Lfe UptimeKuma-Gmer4Lfe VaultWarden-Gmer4Lfe"
|
||||
PROFILE_DELAYED_CONTAINERS[host1-appdata]=""
|
||||
PROFILE_CONTAINER_DELAY[host1-appdata]=5
|
||||
PROFILE_EXCLUDE_DIRS[host1-appdata]="logs *.tmp"
|
||||
|
||||
# ==============================================================================================
|
||||
# ── DOCKER ────────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Docker Daily Restart ━━━
|
||||
# Containers restarted every day via DAILY_MAINTENANCE_SCRIPTS.
|
||||
# Dispatcharr degrades over time without restart — daily is intentional, not just housekeeping.
|
||||
# Order matters — auth stack first, then media services.
|
||||
HOST1_DAILY_RESTART_CONTAINERS=(
|
||||
"NginxProxyManager"
|
||||
"Lldap-Gmer4Lfe"
|
||||
"Authelia"
|
||||
"Authelia-Secondary"
|
||||
"Dispatcharr-Iptv-Users"
|
||||
"Dispatcharr" # Live TV scheduler — degrades without daily restart
|
||||
"Dispatcharr-Basic"
|
||||
"ErsatzTV-Emby"
|
||||
"Slskd" # Soulseek connection drops after extended uptime; restart refreshes share index
|
||||
)
|
||||
|
||||
# ━━━ Docker Weekly Restart ━━━
|
||||
# Less critical services restarted weekly via WEEKLY_MAINTENANCE_SCRIPTS (Sunday 2:30am).
|
||||
# Containers already stopped for weekly sync — restart adds zero extra downtime.
|
||||
HOST1_WEEKLY_RESTART_CONTAINERS=(
|
||||
"NextCloud"
|
||||
"Organizrv2-Gmer4Lfe"
|
||||
"AdGuard-Home"
|
||||
"Immich-Gmer4Lfe"
|
||||
)
|
||||
|
||||
# ━━━ Docker Watchdog ━━━
|
||||
# Per-HOST1 container configuration for docker_watchdog.sh.
|
||||
# Shared thresholds and toggles live in master.conf.
|
||||
|
||||
# Memory hard limits in MB — immediate restart if exceeded.
|
||||
# Set at "container is clearly broken" not "container is busy".
|
||||
# 20GB=20480 18GB=18432 16GB=16384 12GB=12288 8GB=8192 4GB=4096 2GB=2048 1GB=1024
|
||||
declare -A HOST1_WATCHDOG_CONTAINERS=(
|
||||
["Emby"]=20480 # 20GB — large library + active transcodes
|
||||
["LidaTube"]=6144 # 6GB — memory leak over time
|
||||
["Tdarr"]=6144 # 6GB — encoding is memory intensive
|
||||
["Code-Server"]=1024 # 1GB — should never need more
|
||||
)
|
||||
|
||||
# HTTP health check URLs — checked every cycle, strike system before restart.
|
||||
# Only add containers with a meaningful web interface to check.
|
||||
declare -A HOST1_WATCHDOG_CONTAINER_URLS=(
|
||||
["Emby"]="http://localhost:8096"
|
||||
)
|
||||
|
||||
# Required containers — must always be running on HOST1.
|
||||
# Strike system before restart — repeated failures go on skip list, auto-clears on recovery.
|
||||
# Listed in dependency order — dependencies before dependents.
|
||||
HOST1_WATCHDOG_REQUIRED_CONTAINERS=(
|
||||
"NginxProxyManager"
|
||||
"Lldap-Gmer4Lfe"
|
||||
"Mariadb-Authelia"
|
||||
"Mariadb-Authelia-Secondary"
|
||||
"Redis-Authelia"
|
||||
"Redis-Authelia-Secondary"
|
||||
"Authelia"
|
||||
"Authelia-Secondary"
|
||||
)
|
||||
|
||||
# Containers to skip in Tier 2 global scan — legitimately stopped or frequently restarting.
|
||||
# Watchdog leaves these alone entirely — no restart attempts, no crash loop tracking.
|
||||
HOST1_WATCHDOG_SCAN_IGNORE=(
|
||||
"DashGate"
|
||||
"PIA-WG-Config-Generator"
|
||||
"Aperture"
|
||||
"Aperture-Kids"
|
||||
"pgvector-18-Apeture-Kids"
|
||||
"Pgvector18-Aperture"
|
||||
"emby-test" # broken test container (exit 127 — bad image)
|
||||
)
|
||||
|
||||
# Dependency ordering — skip restarting a container if its dependency is also down.
|
||||
# Prevents watchdog from restarting Authelia before Mariadb is back up.
|
||||
# SPACE-SEPARATED STRINGS — converted to array at runtime.
|
||||
declare -A HOST1_WATCHDOG_DEPENDENCIES=(
|
||||
["Authelia"]="Mariadb-Authelia Redis-Authelia"
|
||||
["Authelia-Secondary"]="Mariadb-Authelia Redis-Authelia-Secondary"
|
||||
["NextCloud"]="Postgres-NextCloud"
|
||||
)
|
||||
|
||||
# Per-container appdata growth suppress ceilings in MB.
|
||||
# ONLY needed in specific cases — growth rate detection covers all containers automatically.
|
||||
# Use this when a container legitimately has large stable data and you want to guarantee
|
||||
# it never triggers a false-positive growth alert. Growth warnings are suppressed while the
|
||||
# container's dir stays below this ceiling; above it, warnings resume as normal.
|
||||
# 50GB=51200 25GB=25600 20GB=20480 15GB=15360 10GB=10240 5GB=5120
|
||||
declare -A HOST1_WATCHDOG_APPDATA_SIZES=(
|
||||
["Tdarr"]="25600" # 25GB — transcode cache grows legitimately during active jobs
|
||||
["7dtd"]="20480" # 20GB — game server world data, expected to be large
|
||||
)
|
||||
|
||||
# ━━━ Network Watchdog ━━━
|
||||
# Host-specific connectivity config for Watchdogs/System/network_watchdog.sh.
|
||||
HOST1_NETWORK_WATCHDOG_DDNS_DOMAIN="gmer4lfe.com"
|
||||
HOST1_NETWORK_WATCHDOG_DDNS_CONTAINER="Gmer4Lfe.com"
|
||||
HOST1_NETWORK_WATCHDOG_NPM_URL="https://gmer4lfe.com"
|
||||
|
||||
# ━━━ Docker Network Connect ━━━
|
||||
# Containers connected to custom networks at array start by docker_network_connect.sh.
|
||||
# Networks created if they don't exist — idempotent, safe to re-run.
|
||||
HOST1_NETWORK_CONNECT_CONTAINERS=(
|
||||
"memcached"
|
||||
"Npm-CrowdSec"
|
||||
)
|
||||
|
||||
HOST1_NETWORK_CONNECT_NETWORKS=(
|
||||
"high-availability"
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── FALLBACK ──────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ DDNS ━━━
|
||||
# DDNS containers HOST1 manages — started/stopped by fallback.sh per DDNS absolute rules:
|
||||
# Internet loss → stop immediately
|
||||
# Failover → HOST2 starts HOST1's DDNS as Tier 1 (before any other containers)
|
||||
# Handback → stop HOST1's DDNS on HOST2 → rsync → start containers → start local DDNS last
|
||||
HOST1_DDNS_CONTAINERS=(
|
||||
"Gmer4Lfe.com"
|
||||
)
|
||||
|
||||
# ━━━ Internet Loss ━━━
|
||||
# Containers stopped immediately on HOST1 when internet connection is lost.
|
||||
# Prevents external-facing services from operating without connectivity.
|
||||
FALLBACK_HOST1_STOP_ON_NO_NET=(
|
||||
"Gmer4Lfe.com"
|
||||
)
|
||||
|
||||
# ━━━ Fallback Tiers — HOST1 Runs for HOST2 ━━━
|
||||
# Containers HOST1 starts when HOST2 goes down.
|
||||
# Tier 1 is always immediate — vital services cannot wait.
|
||||
# Higher tiers activate after HOST2_TIER*_DELAY minutes (set in host2.conf).
|
||||
FALLBACK_HOST1_COVERS_HOST2_TIER1=(
|
||||
"Gmer4Lfe.us"
|
||||
"VaultWarden-Jayred365"
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_COVERS_HOST2_TIER2=(
|
||||
# "container-placeholder"
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_COVERS_HOST2_TIER3=(
|
||||
# "container-placeholder"
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_COVERS_HOST2_TIER4=(
|
||||
# "container-placeholder"
|
||||
)
|
||||
|
||||
# ━━━ Tier Delays — HOST1's Containers on HOST2 ━━━
|
||||
# How long HOST1 must be down before each tier activates on HOST2 — in minutes.
|
||||
# Tier 1 is always immediate — no delay var needed.
|
||||
HOST1_TIER2_DELAY=240 # 4 hours — NextCloud, Immich
|
||||
HOST1_TIER3_DELAY=720 # 12 hours — secondary services
|
||||
HOST1_TIER4_DELAY=1440 # 24 hours — arrs + downloaders
|
||||
|
||||
# ━━━ Rsync Writeback — HOST1 Appdata Back on Handback ━━━
|
||||
# Syncs HOST1 appdata BACK to HOST1 when it comes back online after a failover.
|
||||
# Containers stopped before writeback — clean source, no competing writes.
|
||||
#
|
||||
# HOST1_TIER1_WRITEBACK_DELAY: short outages skip Tier 1 writeback — primary state
|
||||
# is more reliable than dirty sync data for brief outages.
|
||||
HOST1_TIER1_WRITEBACK_DELAY=60 # skip Emby writeback if outage under 1hr
|
||||
|
||||
# Tier 4 automatically syncs HOST1_DAILY_SYNC_SHARES — only list paths NOT in that array.
|
||||
FALLBACK_HOST1_WRITEBACK_TIER1=(
|
||||
"/mnt/user/Media_Server/Emby" # watch states built up during outage
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_WRITEBACK_TIER2=(
|
||||
"/mnt/user/appdata-Fallback/Important-Data" # NextCloud + Postgres — files added during outage
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_WRITEBACK_TIER3=(
|
||||
# "location-placeholder"
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_WRITEBACK_TIER4=(
|
||||
"/mnt/user/appdata-Fallback/Arrs_Stack" # arr databases — downloads queued during outage
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── MEDIA ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Media Permissions ━━━
|
||||
# Shares that media_shares_permissions.sh applies PERMISSIONS_MODE and PERMISSIONS_OWNER to.
|
||||
# Runs first in DAILY_MAINTENANCE_SCRIPTS — arr cleanup depends on correct ownership.
|
||||
HOST1_MEDIA_PERMISSION_SHARES=(
|
||||
/mnt/user/Anime_Movies
|
||||
/mnt/user/Anime_Movies-Old
|
||||
/mnt/user/Anime_Shows
|
||||
/mnt/user/Anime_Shows-Old
|
||||
/mnt/user/appcache
|
||||
/mnt/user/Books
|
||||
/mnt/user/Downloads
|
||||
/mnt/user/Games
|
||||
/mnt/user/Intros
|
||||
/mnt/user/Kids_Movies
|
||||
/mnt/user/Kids_Tv_Shows
|
||||
/mnt/user/Movie_Recordings
|
||||
/mnt/user/Movies
|
||||
/mnt/user/Music
|
||||
/mnt/user/Music_Videos
|
||||
/mnt/user/Photo
|
||||
/mnt/user/Sports
|
||||
/mnt/user/stand-up_comedy
|
||||
/mnt/user/Temp_Storage
|
||||
/mnt/user/Tv_Recordings
|
||||
/mnt/user/Tv_Shows
|
||||
/mnt/user/YouTube
|
||||
)
|
||||
|
||||
# ━━━ Media Cleaner ━━━
|
||||
# Folder lists for media_cleaner.sh — two profiles: anime and media.
|
||||
# File patterns shared across all servers — defined in master.conf.
|
||||
# Called via DAILY_MAINTENANCE_SCRIPTS. Run manually: Media/media_cleaner.sh anime|media
|
||||
HOST1_ANIME_CLEAN_FOLDERS=(
|
||||
/mnt/user/Anime_Movies
|
||||
/mnt/user/Anime_Movies-Old
|
||||
/mnt/user/Anime_Shows
|
||||
/mnt/user/Anime_Shows-Old
|
||||
)
|
||||
|
||||
HOST1_MEDIA_CLEAN_FOLDERS=(
|
||||
/mnt/user/Kids_Movies
|
||||
/mnt/user/Kids_Tv_Shows
|
||||
/mnt/user/Movies
|
||||
/mnt/user/Music
|
||||
/mnt/user/Sports
|
||||
/mnt/user/stand-up_comedy
|
||||
/mnt/user/Tv_Shows
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── MONITORS ──────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Certificate Monitor ━━━
|
||||
# Domains checked via direct openssl connection — not relying on NPM's certificate state.
|
||||
# Checks the actual certificate served, not what NPM thinks it has.
|
||||
# Thresholds (CERT_WARN_DAYS, CERT_CRIT_DAYS) defined in master.conf.
|
||||
HOST1_CERT_MONITOR_DOMAINS=(
|
||||
"Gmer4Lfe.com"
|
||||
"Gmer4Lfe.us"
|
||||
)
|
||||
|
||||
# ━━━ SMART Health ━━━
|
||||
# Drives skipped in SMART attribute monitoring — hardware is server-specific.
|
||||
# Thresholds read from dynamix.cfg at runtime — fallbacks in master.conf.
|
||||
HOST1_SMART_IGNORE_DRIVES=(
|
||||
"sda" # boot USB — SMART not meaningful on flash drives
|
||||
)
|
||||
|
||||
# ━━━ ZFS Report ━━━
|
||||
# Pools excluded from the weekly ZFS health report — reduces noise from single-disk array pools.
|
||||
# These are individual array disks formatted as ZFS — converting to XFS over time via unBalance.
|
||||
# Pool health thresholds defined in master.conf.
|
||||
HOST1_ZFS_REPORT_IGNORE_POOLS=(
|
||||
"disk5"
|
||||
"disk6"
|
||||
"disk8"
|
||||
"disk9"
|
||||
"disk10"
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── TRANSCODES ────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# Ramdisk size ceiling — tmpfs only uses RAM actually needed, not the full size upfront.
|
||||
# Real-world: 9 streams peaked at ~5.5GB — 10G gives generous headroom on 128GB RAM.
|
||||
HOST1_RAMDISK_SIZE="10G"
|
||||
|
||||
# Usage thresholds — coupled to HOST1_RAMDISK_SIZE, adjust all three together if size changes.
|
||||
# Hysteresis gap (8.5 - 7 = 1.5GB) prevents flip-flop between ramdisk and SSD.
|
||||
HOST1_RAMDISK_WARN_GB=8.5 # flip to SSD when ramdisk usage reaches this
|
||||
HOST1_RAMDISK_LOW_GB=7 # flip back to ramdisk when usage drops to this
|
||||
|
||||
# SSD fallback path — where transcodes land when ramdisk exceeds HOST1_RAMDISK_WARN_GB.
|
||||
# Must be on cache pool — array disks too slow for active transcode writes.
|
||||
HOST1_TRANSCODE_SSD="/mnt/cache/Temp_Storage/Emby/Transcodes/"
|
||||
|
||||
# Media servers sharing the ramdisk transcode space on HOST1.
|
||||
# Format: "ContainerName|URL|APIKey|Type" — Type: emby | jellyfin | plex
|
||||
# Entries with placeholder API keys are skipped automatically.
|
||||
# ⚠️ Tdarr does NOT belong here — keep Tdarr on SSD, not ramdisk.
|
||||
HOST1_TRANSCODE_SERVERS=(
|
||||
"${HOST1_EMBY_CONTAINER}|${HOST1_EMBY_URL}|${HOST1_EMBY_API_KEY}|emby"
|
||||
"${HOST1_JELLYFIN_CONTAINER}|${HOST1_JELLYFIN_URL}|${HOST1_JELLYFIN_API_KEY}|jellyfin"
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── ARR STACK ─────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Used by arr cleanup scripts and arrs_failed_stalled_recovery.sh.
|
||||
# detect_hosts() selects HOST1 vars when running on HOST1.
|
||||
#
|
||||
# PATH MAPS — container path → host path translation.
|
||||
# Arr stores file paths using container-internal paths — scripts need host paths to scan.
|
||||
# Add one entry per root folder in arr Settings → Media Management → Root Folders.
|
||||
|
||||
# ━━━ Downloaders ━━━
|
||||
# Used by downloaders_reset.sh — runs every 30min via CRITICAL_MAINTENANCE_SCRIPTS.
|
||||
# Clears stuck states, purges old history, prepares each client for a clean cycle.
|
||||
|
||||
# slskd — clears stuck searches, dead transfers, purges expired failed imports.
|
||||
# SLSKD_FAILED_IMPORTS_DIR: where Soularr moves albums Lidarr rejected.
|
||||
HOST1_SLSKD_URL="http://localhost:8980"
|
||||
HOST1_SLSKD_API_KEY="4bF9kL2mNpQrT7vWxYz1A3dEgHjKoRsU"
|
||||
HOST1_SLSKD_FAILED_IMPORTS_DIR="/mnt/user/Temp_Storage/Slskd/completed/failed_imports"
|
||||
|
||||
# SABnzbd
|
||||
HOST1_SABNZBD_URL="http://localhost:8180"
|
||||
HOST1_SABNZBD_API_KEY="8bfefe41d83b4d50883e32859b55ca9a"
|
||||
|
||||
# qBittorrent — deleteFiles=false removes torrent from qBit but leaves files on disk.
|
||||
# Radarr/Sonarr manage actual files independently.
|
||||
HOST1_QBIT_URL="http://localhost:8080"
|
||||
HOST1_QBIT_USERNAME="root"
|
||||
HOST1_QBIT_PASSWORD="Stay0utD!ck"
|
||||
|
||||
# ━━━ Lidarr — HOST1 only ━━━
|
||||
# HOST2 does not run Lidarr — HOST1_LIDARR_RECOVERY flag handles the exit cleanly.
|
||||
HOST1_LIDARR_URL="http://localhost:8686"
|
||||
HOST1_LIDARR_API_KEY="b2977e71ef074bc0a0529d9fcce3b2dc"
|
||||
HOST1_LIDARR_MUSIC_ROOT="/mnt/user/Music-New"
|
||||
HOST1_FANART_API_KEY="Yd7147a43b692df0b364b94dc47efb81"
|
||||
HOST1_LASTFM_API_KEY="be6dc169c33ae263e690c30d18b7491d"
|
||||
|
||||
declare -A HOST1_LIDARR_PATH_MAP=(
|
||||
["/ext-music"]="/mnt/user/Music-New"
|
||||
)
|
||||
|
||||
# ━━━ Sonarr ━━━
|
||||
HOST1_SONARR_URL="http://localhost:8989"
|
||||
HOST1_SONARR_API_KEY="130decd3db5b4c25afad64864cd03f9f"
|
||||
HOST1_SONARR_TV_ROOT="/mnt/user/Tv_Shows"
|
||||
|
||||
# Note: stand-up_comedy in both Sonarr + Radarr — TV specials and movie specials, one folder
|
||||
declare -A HOST1_SONARR_PATH_MAP=(
|
||||
["/tv"]="/mnt/user/Tv_Shows"
|
||||
["/ext-standup-comedy"]="/mnt/user/stand-up_comedy"
|
||||
["/kids tv"]="/mnt/user/Kids_Tv_Shows"
|
||||
["/ext-anime-shows"]="/mnt/user/Anime_Shows-Old"
|
||||
)
|
||||
|
||||
# ━━━ Radarr ━━━
|
||||
HOST1_RADARR_URL="http://localhost:7878"
|
||||
HOST1_RADARR_API_KEY="d43a3ec6cf1549edb4af0cc63f98b2a9"
|
||||
HOST1_TMDB_API_KEY="3dac5e2e49b5540472d2eafec4f01260"
|
||||
HOST1_RADARR_MOVIES_ROOT="/mnt/user/Movies"
|
||||
|
||||
# Note: stand-up_comedy in both Radarr + Sonarr — movie specials and TV specials, one folder
|
||||
declare -A HOST1_RADARR_PATH_MAP=(
|
||||
["/movies"]="/mnt/user/Movies"
|
||||
["/kids movies"]="/mnt/user/Kids_Movies"
|
||||
["/ext-stand-up-comedy"]="/mnt/user/stand-up_comedy"
|
||||
["/anime-movies"]="/mnt/user/Anime_Movies-Old"
|
||||
)
|
||||
|
||||
# ━━━ Arr Recovery Toggles ━━━
|
||||
# false = skip that arr on this host — exits cleanly without error
|
||||
HOST1_SONARR_RECOVERY=true
|
||||
HOST1_RADARR_RECOVERY=true
|
||||
HOST1_LIDARR_RECOVERY=true # HOST1 only — exits cleanly on HOST2
|
||||
|
||||
# ==============================================================================================
|
||||
# ── SYSTEM WATCHDOG ───────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Per-host check toggles and NIC config for system_watchdog.sh.
|
||||
# Aliased by detect_hosts() — script uses unprefixed SYS_WATCHDOG_* names.
|
||||
# HOST1: TR1950X 128GB — full media server, active transcoding, ZFS cache pools.
|
||||
#
|
||||
# Three-tier response — all critical checks enabled by default on HOST1:
|
||||
# Tier 1 (bypass strikes, reboot now): docker daemon, rootfs full, kernel oops, FD, /boot
|
||||
# Tier 2 (bypass strikes with OOM): RAM critical + OOM kills in cycle
|
||||
# Tier 3 (standard strike system): everything else
|
||||
#
|
||||
# RAM tiers, OOM limits, and reboot loop settings in master.conf System Watchdog section.
|
||||
|
||||
# ━━━ Primary NIC ━━━
|
||||
# Network interface for NIC state check — verify with: ip link show | grep "^[0-9]"
|
||||
# Common values: eth0, bond0, br0, eno1
|
||||
HOST1_SYS_WATCHDOG_NIC="eth0"
|
||||
|
||||
# ━━━ Tier 1 — Critical Checks ━━━
|
||||
# These bypass the strike system — a single hit triggers immediate reboot.
|
||||
# Disabling any of these is not recommended — they protect against acute system failure.
|
||||
|
||||
# Docker daemon unresponsive → try restart, reboot if restart fails.
|
||||
# Without a working daemon docker_watchdog.sh is blind and containers cannot be managed.
|
||||
HOST1_SYS_WATCHDOG_CHECK_DOCKER_DAEMON=true
|
||||
|
||||
# rootfs at critical threshold (ROOTFS_CRITICAL_PCT=99) → reboot immediately.
|
||||
# At 99% rootfs writes fail silently — logs stop, Docker errors out, SSH may stop working.
|
||||
# Standard 95% threshold still uses strike system — only 99%+ is critical tier.
|
||||
HOST1_SYS_WATCHDOG_CHECK_ROOTFS=true
|
||||
|
||||
# Kernel BUG/Oops in dmesg delta since last cycle → reboot immediately.
|
||||
# A kernel oops means the kernel ran with a corrupted state — stability is not guaranteed.
|
||||
HOST1_SYS_WATCHDOG_CHECK_KERNEL_OOPS=true
|
||||
|
||||
# File descriptor exhaustion at FD_CRITICAL_PCT (95%) → reboot immediately.
|
||||
# At 95% FD: new connections fail, Docker can't spawn processes, SSH drops.
|
||||
HOST1_SYS_WATCHDOG_CHECK_FD=true
|
||||
|
||||
# /boot read-only detected → reboot immediately.
|
||||
# Unexpected read-only /boot means state files and config writes are silently failing.
|
||||
# Fallback state, watchdog reboot log, and lock files all go stale silently.
|
||||
HOST1_SYS_WATCHDOG_CHECK_BOOT=true
|
||||
|
||||
# ━━━ Tier 2 — Urgent OOM Check ━━━
|
||||
# Bypass strikes when RAM is critically low AND OOM kill rate confirms active crisis.
|
||||
# Both must be enabled for Tier 2 bypass to function — disable either to always use strikes.
|
||||
|
||||
# Track kernel OOM kills each cycle via /proc/vmstat oom_kill delta.
|
||||
# Also provides diagnostic context in reboot messages (which processes were killed).
|
||||
HOST1_SYS_WATCHDOG_CHECK_OOM=true
|
||||
|
||||
# Free RAM check — required for both Tier 2 bypass and RAM tier logic.
|
||||
# Tiers: MEM_WARN_GB(10) → notify | MEM_SHUTDOWN_GB(6) → stop containers | MEM_GB(4) → strikes
|
||||
HOST1_SYS_WATCHDOG_CHECK_RAM=true
|
||||
|
||||
# ━━━ Tier 3 — Standard Checks (strike system) ━━━
|
||||
# Each check must fail SYS_WATCHDOG_STRIKE_LIMIT consecutive cycles before action is taken.
|
||||
# Single spikes are ignored — sustained problems trigger reboot.
|
||||
|
||||
# /var/log filesystem usage above SYS_WATCHDOG_LOG_PCT.
|
||||
# Log spam (Docker log storms, syslog loops) fills rootfs — indicates something broken.
|
||||
HOST1_SYS_WATCHDOG_CHECK_LOG=true
|
||||
|
||||
# ZFS ARC memory pinned above SYS_WATCHDOG_ARC_PINNED_PCT after cache drop.
|
||||
# Enabled on HOST1 — ZFS cache pools actively used. Disable on hosts without ZFS.
|
||||
HOST1_SYS_WATCHDOG_CHECK_ARC=true
|
||||
|
||||
# CPU temperature above SYS_WATCHDOG_CPU_TEMP_MAX (95°C).
|
||||
# Sustained high temp causes kernel throttling or panic. Requires lm-sensors.
|
||||
HOST1_SYS_WATCHDOG_CHECK_CPU_TEMP=true
|
||||
|
||||
# Load average above SYS_WATCHDOG_LOAD_MULTIPLIER × core count.
|
||||
# DISABLED on HOST1 — Tdarr and Emby cause legitimate sustained load spikes during encoding.
|
||||
# Enable on idle servers or adjust SYS_WATCHDOG_LOAD_MULTIPLIER if load is always high.
|
||||
HOST1_SYS_WATCHDOG_CHECK_LOAD=false
|
||||
|
||||
# Zombie process count above SYS_WATCHDOG_ZOMBIE_LIMIT (50).
|
||||
# Large zombie counts indicate serious process management failure — something is stuck.
|
||||
HOST1_SYS_WATCHDOG_CHECK_ZOMBIES=true
|
||||
|
||||
# Check docker_watchdog.sh persistent skip list — required containers on skip list.
|
||||
# Cross-watchdog coordination: if docker_watchdog gave up, system_watchdog escalates.
|
||||
# ENABLED — HOST1 fully built and operational, skip list is meaningful.
|
||||
HOST1_SYS_WATCHDOG_CHECK_CONTAINERS=true
|
||||
|
||||
# /tmp filesystem usage above SYS_WATCHDOG_TMP_PCT with auto-clear attempt.
|
||||
# Script tries to clear aged /tmp files first — only strikes if clear fails.
|
||||
# Lock files, rsync temp files, and Docker ops use /tmp — 100% means lock failures.
|
||||
HOST1_SYS_WATCHDOG_CHECK_TMP=true
|
||||
|
||||
# Array disk error count delta in /proc/mdstat — accumulating errors = disk failing now.
|
||||
# Triggers on SYS_WATCHDOG_MDSTAT_ERROR_LIMIT new errors in one cycle.
|
||||
HOST1_SYS_WATCHDOG_CHECK_MDSTAT=true
|
||||
|
||||
# Primary NIC operstate — detects NIC going down (physical or driver failure).
|
||||
# Uses HOST1_SYS_WATCHDOG_NIC above. Strike system — brief flaps don't trigger reboot.
|
||||
HOST1_SYS_WATCHDOG_CHECK_NETWORK=true
|
||||
|
||||
# sshd running check — attempts restart before escalating.
|
||||
# sshd down = no remote access. Script tries rc.sshd start, notifies, strikes on failure.
|
||||
HOST1_SYS_WATCHDOG_CHECK_SSHD=true
|
||||
|
||||
# Runaway process detection — single process above SYS_WATCHDOG_RUNAWAY_CPU_PCT sustained.
|
||||
# DISABLED — Tdarr encoding and Emby transcoding legitimately peg CPU for extended periods.
|
||||
# Enable only if HOST1 has no CPU-intensive workloads.
|
||||
HOST1_SYS_WATCHDOG_CHECK_RUNAWAY=false
|
||||
|
||||
# ==============================================================================================
|
||||
# ── RESOURCE MANAGER ──────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Containers to manage under pressure — see master.conf RW_CRITICAL_CONTAINERS for exclusions.
|
||||
|
||||
# docker pause at medium pressure (RAM < RW_RAM_MEDIUM_GB or load > medium threshold)
|
||||
# Suspended in-place — instant to pause/unpause, no state lost, no restart delay.
|
||||
HOST1_RW_PAUSE_CONTAINERS=(
|
||||
"Huntarr" # arr search automation — safe to suspend
|
||||
"Cleanuparr" # download cleanup — safe to suspend
|
||||
"Healarr" # arr health checks — safe to suspend
|
||||
"Soularr" # Slskd automation — background only
|
||||
"ChannelTube" # YouTube archiver — background only
|
||||
"Pinchflat" # YouTube archiver — background only
|
||||
)
|
||||
|
||||
# docker stop at hard pressure (RAM < RW_RAM_HARD_GB)
|
||||
# Full stop — these are optional/heavy services that free significant RAM when stopped.
|
||||
# resource_watchdog.sh restarts them when pressure fully clears (RAM >= RW_RAM_RECOVER_GB).
|
||||
HOST1_RW_STOP_CONTAINERS=(
|
||||
"LocalAI" # GPU/CPU heavy — largest RAM consumer when idle
|
||||
"7DaysToDie" # game server — optional
|
||||
"V-Rising" # game server — optional
|
||||
"Code-Server" # IDE — not needed during pressure events
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── AUTH STACK ────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Credentials for the Varaverk Auth Stack page (NPM, lldap, Authelia).
|
||||
|
||||
# ━━━ NginxProxyManager ━━━
|
||||
HOST1_NPM_URL="http://localhost:7818"
|
||||
HOST1_NPM_USER="failedproxy@gmail.com"
|
||||
HOST1_NPM_PASS="183134\$eanHess"
|
||||
|
||||
# ━━━ lldap ━━━
|
||||
HOST1_LLDAP_URL="http://localhost:17170"
|
||||
HOST1_LLDAP_USER="admin"
|
||||
HOST1_LLDAP_PASS="183134\$eanHess"
|
||||
|
||||
# ━━━ Authelia ━━━
|
||||
HOST1_AUTHELIA_CONFIG="/mnt/user/appdata-Fallback/Critical-Data/Authelia/configuration.yml"
|
||||
HOST1_AUTHELIA_CONTAINER="Authelia"
|
||||
|
||||
# ==============================================================================================
|
||||
# ──────────────────────── End Of HOST1 Variables ──────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
HOST1_UNRAID_API_KEY="1825c3a2e03ea5089974f4da2e171aa2d5907a1dea23cc479c33e492c8ff4dbb"
|
||||
@@ -0,0 +1,789 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ========================== HOST1 CONFIGURATION — unRAID-Gmer4Lfe ============================
|
||||
# ==============================================================================================
|
||||
# HOST1-specific variables — credentials, container names, share paths, failover lists.
|
||||
# Sourced after master.conf — values here extend shared profile arrays and add HOST1-specific
|
||||
# identity, credentials, and container configuration.
|
||||
#
|
||||
# Sparse checkout (git) ensures HOST2 never receives this file.
|
||||
# HOST2 never sees HOST1 credentials — clean separation at the file level.
|
||||
#
|
||||
# DO NOT put shared config here — thresholds, toggles, profiles belong in master.conf.
|
||||
# DO NOT put HOST2 variables here — they belong in host2.conf.
|
||||
#
|
||||
# ── INDEX ─────────────────────────────────────────────────────────────────────────────────────
|
||||
#
|
||||
# ── IDENTITY & CONNECTIVITY ────────────────────────────────────────────────────────────────
|
||||
# IDENTITY hostname, SSH key
|
||||
# EMBY container name, URL, API key
|
||||
# NOTIFICATIONS Discord webhook
|
||||
# PARTNERSHIP auth containers, backup paths
|
||||
#
|
||||
# ── RSYNC ──────────────────────────────────────────────────────────────────────────────────
|
||||
# DAILY SYNC SHARES media shares HOST1 owns and pushes to HOST2
|
||||
# WEEKLY SYNC SHARES appdata shares synced weekly (Sunday 2:30am)
|
||||
# CRITICAL SYNC SHARES appdata shares synced every 30 minutes
|
||||
# BACKUP VERIFY shares for checksum verification against remote
|
||||
# HOST1 RSYNC PROFILE host1-appdata profile for HOST1-specific appdata syncs
|
||||
#
|
||||
# ── DOCKER ─────────────────────────────────────────────────────────────────────────────────
|
||||
# DOCKER DAILY RESTART containers restarted daily
|
||||
# DOCKER WEEKLY RESTART containers restarted weekly
|
||||
# DOCKER WATCHDOG memory limits, health URLs, required containers, ignore list
|
||||
# DOCKER NETWORK CONNECT networks and containers for docker_network_connect.sh
|
||||
#
|
||||
# ── FALLBACK ───────────────────────────────────────────────────────────────────────────────
|
||||
# DDNS DDNS containers managed by HOST1
|
||||
# INTERNET LOSS containers stopped when internet is lost
|
||||
# FALLBACK TIERS what HOST1 runs for HOST2 per tier
|
||||
# TIER DELAYS how long HOST1 must be down before each tier activates on HOST2
|
||||
# RSYNC WRITEBACK HOST1 appdata synced back on handback
|
||||
#
|
||||
# ── MEDIA ──────────────────────────────────────────────────────────────────────────────────
|
||||
# MEDIA PERMISSIONS share list for media_shares_permissions.sh
|
||||
# MEDIA CLEANER folder lists for media_cleaner.sh
|
||||
#
|
||||
# ── MONITORS ───────────────────────────────────────────────────────────────────────────────
|
||||
# CERTIFICATE MONITOR domains checked for SSL expiry
|
||||
# SMART HEALTH drives to skip in SMART monitoring
|
||||
# ZFS REPORT pools to exclude from ZFS health report
|
||||
#
|
||||
# ── TRANSCODES ─────────────────────────────────────────────────────────────────────────────
|
||||
# TRANSCODES ramdisk size, thresholds, SSD path, server array
|
||||
#
|
||||
# ── ARR STACK ──────────────────────────────────────────────────────────────────────────────
|
||||
# DOWNLOADERS slskd, SABnzbd, qBittorrent credentials and URLs
|
||||
# LIDARR URL, API key, path map
|
||||
# SONARR URL, API key, path map
|
||||
# RADARR URL, API key, path map
|
||||
# ARR RECOVERY per-arr recovery toggles
|
||||
#
|
||||
# ── SYSTEM WATCHDOG ────────────────────────────────────────────────────────────────────────
|
||||
# SYSTEM WATCHDOG per-host check toggles and NIC configuration
|
||||
#
|
||||
# ── RESOURCE MANAGER ───────────────────────────────────────────────────────────────────────
|
||||
# RESOURCE MANAGER containers paused/stopped under memory pressure
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
# ==============================================================================================
|
||||
# ── IDENTITY & CONNECTIVITY ───────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Identity ━━━
|
||||
# HOST1 hostname lives in master.conf (not a credential — safe for all servers).
|
||||
# SSH key used for all server-to-server operations — rsync, failover container commands.
|
||||
# Must be in /root/.ssh/ and authorised in HOST2's /root/.ssh/authorized_keys.
|
||||
HOST1_SSH_KEY="/root/.ssh/gmer4lfe_rsync_automation"
|
||||
HOST1_OWNER="gmer4lfe"
|
||||
HOST1_OWNER_EMAIL="gmer4lfe@gmail.com"
|
||||
|
||||
# ━━━ Emby ━━━
|
||||
# Referenced by transcode_manager.sh, emby_session_report.sh, emby_database_repair.sh,
|
||||
# weekly_sync_maintenance.sh, and HOST1_TRANSCODE_SERVERS below.
|
||||
# API key: Emby Dashboard → API Keys → + New Key
|
||||
HOST1_EMBY_CONTAINER="Emby"
|
||||
HOST1_EMBY_URL="http://localhost:8096"
|
||||
HOST1_EMBY_API_KEY="0c27448d93a7431f9ac63569f7655829"
|
||||
|
||||
# ━━━ Jellyfin ━━━
|
||||
# API key: Jellyfin Dashboard → Administration → API Keys → + New Key
|
||||
HOST1_JELLYFIN_CONTAINER="Jellyfin"
|
||||
HOST1_JELLYFIN_URL="http://localhost:8095"
|
||||
HOST1_JELLYFIN_API_KEY="4e820e7df74c4933acec212b1996314e"
|
||||
|
||||
# ━━━ Gitea ━━━
|
||||
# Personal access token for gitea_ssh_setup.sh — registers this server's SSH public key
|
||||
# with Gitea so git operations use key auth instead of passwords.
|
||||
# Create in Gitea: Settings → Applications → Generate Token → scope: write:user
|
||||
HOST1_GITEA_API_TOKEN=""
|
||||
|
||||
# ━━━ Notifications ━━━
|
||||
# Discord webhook — leave blank to disable.
|
||||
# Per-host so HOST1 and HOST2 can post to different channels or only one server notifies.
|
||||
HOST1_DISCORD_WEBHOOK=""
|
||||
|
||||
# ━━━ Partnership ━━━
|
||||
# HOST1 is always the owner (source of truth) unless --transfer has been run.
|
||||
# See README-Partnership.md and master.conf PARTNERSHIP section for full lifecycle docs.
|
||||
|
||||
# Auth containers reconfigured on onboard/offboard.
|
||||
# Format: "ContainerName|WebUIPort"
|
||||
# On onboard → WebUI pointed at owner's Tailscale IP (mirror clicks NPM, gets owner's auth)
|
||||
# On offboard → WebUI pointed back at localhost
|
||||
HOST1_PARTNERSHIP_AUTH_WEBUIS=(
|
||||
"NginxProxyManager|81"
|
||||
"Lldap-Gmer4Lfe|17170"
|
||||
"Authelia|9091"
|
||||
"Authelia-Secondary|9092"
|
||||
)
|
||||
|
||||
# XML templates (from this server's templates-user/) pushed to mirror during onboard.
|
||||
# These become the mirror's active auth stack, backed by the rsync-synced appdata.
|
||||
# Update filename if Lldap is renamed to drop the host suffix.
|
||||
HOST1_PARTNERSHIP_AUTH_STACK=(
|
||||
# Dependencies first — Mariadb/Redis must be healthy before Authelia starts
|
||||
"my-Mariadb-Authelia.xml"
|
||||
"my-Mariadb-Authelia-Secondary.xml"
|
||||
"my-Redis-Authelia.xml"
|
||||
"my-Redis-Authelia-Secondary.xml"
|
||||
# Auth apps — deployed after their deps are confirmed healthy
|
||||
"my-Authelia.xml"
|
||||
"my-Authelia-Secondary.xml"
|
||||
"my-NginxProxyManager.xml"
|
||||
"my-Lldap-Gmer4Lfe.xml"
|
||||
# Source of truth — must be available on HOST2 independently of the auth stack
|
||||
"my-Gitea.xml"
|
||||
)
|
||||
|
||||
# XML templates pushed to mirror for the arr stack during onboard.
|
||||
# Deps (e.g. databases) first if any — same ordering rule as auth stack.
|
||||
HOST1_PARTNERSHIP_ARR_STACK=(
|
||||
# "my-Sonarr.xml"
|
||||
# "my-Radarr.xml"
|
||||
# "my-Lidarr.xml"
|
||||
# "my-Prowlarr.xml"
|
||||
# "my-Bazarr.xml"
|
||||
)
|
||||
|
||||
# Paths HOST2 should collect during the grace window after offboard.
|
||||
# Notified on offboard — no auto-deletion, HOST2 must collect manually within PARTNERSHIP_GRACE_HOURS.
|
||||
HOST1_PARTNERSHIP_MIRROR_BACKUPS=(
|
||||
# "/mnt/user/appdata-Fallback/Jayred365-Emby"
|
||||
)
|
||||
|
||||
# Containers parked on this server when partnership is active.
|
||||
# Stopped on onboard (owner deploys its stack instead), restarted on offboard.
|
||||
HOST1_PARTNERSHIP_OWN_CONTAINERS=(
|
||||
# "Emby"
|
||||
# "NginxProxyManager"
|
||||
)
|
||||
|
||||
# Emby admin provisioning — toggle is owner-only, credentials are per-host.
|
||||
# Owner enables/disables the feature. Each host sets the account they want on the shared Emby.
|
||||
# On onboard: owner reads mirror's HOST*_PARTNERSHIP_EMBY_ADMIN_* and creates that account.
|
||||
# On offboard: account is deleted. Username collision → onboard exits with error.
|
||||
HOST1_PARTNERSHIP_PROVISION_EMBY_ADMIN=false # owner controls whether Emby is shared
|
||||
HOST1_PARTNERSHIP_EMBY_PORT=8096
|
||||
HOST1_PARTNERSHIP_EMBY_ADMIN_USER="" # this server's desired Emby username
|
||||
HOST1_PARTNERSHIP_EMBY_ADMIN_PASS="" # this server's desired Emby password
|
||||
|
||||
# ==============================================================================================
|
||||
# ── RSYNC ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Daily Sync Shares ━━━
|
||||
# Shares HOST1 pushes to all other nodes every night (1am via daily_sync_maintenance.sh).
|
||||
# Mesh model: every node pushes every media share — no ownership, no mirrors.
|
||||
# arr_sync ensures all arr libraries converge (union). rsync spreads files (additive, no --delete).
|
||||
# arr_cleanup removes true orphans based on local arr state.
|
||||
# Any node can download content to any share — it propagates to all nodes on the next cycle.
|
||||
# Nextcloud is intentionally one-directional (HOST1→HOST2 offsite backup — not arr-managed).
|
||||
# Uses DEFAULT_RSYNC_OPTS from master.conf — no profile needed.
|
||||
# For shares needing container stops or custom options — add a profile in master.conf.
|
||||
HOST1_DAILY_SYNC_SHARES=(
|
||||
/mnt/user/Books
|
||||
/mnt/user/Intros
|
||||
/mnt/user/Kids_Movies
|
||||
/mnt/user/Kids_Tv_Shows
|
||||
/mnt/user/Movies
|
||||
/mnt/user/Music
|
||||
/mnt/user/Music_Videos
|
||||
/mnt/user/Nextcloud
|
||||
/mnt/user/stand-up_comedy
|
||||
/mnt/user/Sports
|
||||
/mnt/user/Tv_Shows
|
||||
/mnt/user/Anime_Shows-Old
|
||||
/mnt/user/Anime_Movies-Old
|
||||
/mnt/user/Anime_Movies
|
||||
/mnt/user/Anime_Shows
|
||||
)
|
||||
|
||||
# Personal encrypted shares — synced for offsite backup, independent of media shares.
|
||||
# ZFS encrypted at dataset level — remote receives encrypted blocks, cannot read content.
|
||||
# See README-Rsync_Setup.md for ZFS encryption setup before uncommenting.
|
||||
HOST1_PERSONAL_SHARES=(
|
||||
# /mnt/user/HOST1-Personal # uncomment after creating encrypted dataset
|
||||
)
|
||||
|
||||
# ━━━ Weekly Sync Shares ━━━
|
||||
# Appdata shares synced during the weekly maintenance window (Sunday 2:30am).
|
||||
# Containers stopped both sides before sync — full clean state guaranteed.
|
||||
# Profiles drive container stops, excludes, and options — configured in master.conf RSYNC section.
|
||||
# Order matters — Emby first (larger transfer), then Critical-Data (auth stack).
|
||||
HOST1_WEEKLY_SYNC_SHARES=(
|
||||
"/mnt/user/Media_Server/Emby" # emby profile — full clean mirror
|
||||
"/mnt/user/appdata-Fallback/Critical-Data" # critical-data profile — auth stack
|
||||
)
|
||||
|
||||
# ━━━ Intermediate Sync Shares ━━━
|
||||
# Shares synced every 4 hours by intermediate_sync_maintenance.sh.
|
||||
# Uses DEFAULT_RSYNC_OPTS (no --delete) — for sub-daily propagation of metadata or watch state.
|
||||
# Full media share sync stays in the daily window. Leave empty to skip mid-day rsync entirely.
|
||||
HOST1_INTERMEDIATE_SYNC_SHARES=(
|
||||
# Add shares here to enable mid-day rsync
|
||||
# Example: "/mnt/user/Emby_Metadata"
|
||||
)
|
||||
|
||||
# ━━━ Critical Sync Shares ━━━
|
||||
# Appdata shares synced every 30 minutes by critical_sync_maintenance.sh.
|
||||
# Format: "/path/to/share" or "/path/to/share|profile-name"
|
||||
# Order matters — Critical-Data first (auth stack), then Emby dirty sync.
|
||||
HOST1_CRITICAL_SYNC_SHARES=(
|
||||
"/mnt/user/appdata-Fallback/Critical-Data|critical-fallback" # auth dirty sync — stays running
|
||||
"/mnt/user/Media_Server/Emby|emby-fallback" # Emby dirty sync — stays running
|
||||
)
|
||||
|
||||
# ━━━ Backup Verify ━━━
|
||||
# Shares verified by backup_verify.sh — random file checksum comparison against remote.
|
||||
# Leave empty to use HOST1_DAILY_SYNC_SHARES automatically.
|
||||
# Sample size and minimum file size defined in master.conf.
|
||||
HOST1_BACKUP_VERIFY_SHARES=(
|
||||
# leave empty to use HOST1_DAILY_SYNC_SHARES automatically
|
||||
)
|
||||
|
||||
# ━━━ HOST1 Rsync Profile — host1-appdata ━━━
|
||||
# HOST1-specific appdata sync profile — extends the shared PROFILE_* arrays in master.conf.
|
||||
# Use for appdata unique to HOST1 (Organizrv2, VaultWarden, UptimeKuma etc.)
|
||||
# Shared appdata (auth stack, Emby) use dedicated profiles defined in master.conf.
|
||||
# Run manually: bash Rsync/rsync.sh /mnt/user/appdata-Fallback/HOST1-Appdata --profile=host1-appdata
|
||||
PROFILE_RSYNC_OPTS[host1-appdata]="-av --info=progress2 --bwlimit=${PROFILE_BW_LIMIT[host1-appdata]:-8000}"
|
||||
PROFILE_BW_LIMIT[host1-appdata]=8000
|
||||
PROFILE_RETRY_COUNT[host1-appdata]=3
|
||||
PROFILE_SLEEP[host1-appdata]=300
|
||||
PROFILE_CRITICAL_CONTAINER_NAMES[host1-appdata]="Organizrv2-Gmer4Lfe UptimeKuma-Gmer4Lfe VaultWarden-Gmer4Lfe"
|
||||
PROFILE_DELAYED_CONTAINERS[host1-appdata]=""
|
||||
PROFILE_CONTAINER_DELAY[host1-appdata]=5
|
||||
PROFILE_EXCLUDE_DIRS[host1-appdata]="logs *.tmp"
|
||||
|
||||
# ==============================================================================================
|
||||
# ── DOCKER ────────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Docker Daily Restart ━━━
|
||||
# Containers restarted every day via DAILY_MAINTENANCE_SCRIPTS.
|
||||
# Dispatcharr degrades over time without restart — daily is intentional, not just housekeeping.
|
||||
# Order matters — auth stack first, then media services.
|
||||
HOST1_DAILY_RESTART_CONTAINERS=(
|
||||
"NginxProxyManager"
|
||||
"Lldap-Gmer4Lfe"
|
||||
"Authelia"
|
||||
"Authelia-Secondary"
|
||||
"Dispatcharr-Iptv-Users"
|
||||
"Dispatcharr" # Live TV scheduler — degrades without daily restart
|
||||
"Dispatcharr-Basic"
|
||||
"ErsatzTV-Emby"
|
||||
"Slskd" # Soulseek connection drops after extended uptime; restart refreshes share index
|
||||
)
|
||||
|
||||
# ━━━ Docker Weekly Restart ━━━
|
||||
# Less critical services restarted weekly via WEEKLY_MAINTENANCE_SCRIPTS (Sunday 2:30am).
|
||||
# Containers already stopped for weekly sync — restart adds zero extra downtime.
|
||||
HOST1_WEEKLY_RESTART_CONTAINERS=(
|
||||
"NextCloud"
|
||||
"Organizrv2-Gmer4Lfe"
|
||||
"AdGuard-Home"
|
||||
"Immich-Gmer4Lfe"
|
||||
)
|
||||
|
||||
# ━━━ Docker Watchdog ━━━
|
||||
# Per-HOST1 container configuration for docker_watchdog.sh.
|
||||
# Shared thresholds and toggles live in master.conf.
|
||||
|
||||
# Memory hard limits in MB — immediate restart if exceeded.
|
||||
# Set at "container is clearly broken" not "container is busy".
|
||||
# 20GB=20480 18GB=18432 16GB=16384 12GB=12288 8GB=8192 4GB=4096 2GB=2048 1GB=1024
|
||||
declare -A HOST1_WATCHDOG_CONTAINERS=(
|
||||
["Emby"]=20480 # 20GB — large library + active transcodes
|
||||
["LidaTube"]=6144 # 6GB — memory leak over time
|
||||
["Tdarr"]=6144 # 6GB — encoding is memory intensive
|
||||
["Code-Server"]=1024 # 1GB — should never need more
|
||||
)
|
||||
|
||||
# HTTP health check URLs — checked every cycle, strike system before restart.
|
||||
# Only add containers with a meaningful web interface to check.
|
||||
declare -A HOST1_WATCHDOG_CONTAINER_URLS=(
|
||||
["Emby"]="http://localhost:8096"
|
||||
)
|
||||
|
||||
# Required containers — must always be running on HOST1.
|
||||
# Strike system before restart — repeated failures go on skip list, auto-clears on recovery.
|
||||
# Listed in dependency order — dependencies before dependents.
|
||||
HOST1_WATCHDOG_REQUIRED_CONTAINERS=(
|
||||
"NginxProxyManager"
|
||||
"Lldap-Gmer4Lfe"
|
||||
"Mariadb-Authelia"
|
||||
"Mariadb-Authelia-Secondary"
|
||||
"Redis-Authelia"
|
||||
"Redis-Authelia-Secondary"
|
||||
"Authelia"
|
||||
"Authelia-Secondary"
|
||||
)
|
||||
|
||||
# Containers to skip in Tier 2 global scan — legitimately stopped or frequently restarting.
|
||||
# Watchdog leaves these alone entirely — no restart attempts, no crash loop tracking.
|
||||
HOST1_WATCHDOG_SCAN_IGNORE=(
|
||||
"DashGate"
|
||||
"PIA-WG-Config-Generator"
|
||||
"Aperture"
|
||||
"Aperture-Kids"
|
||||
"pgvector-18-Apeture-Kids"
|
||||
"Pgvector18-Aperture"
|
||||
"emby-test" # broken test container (exit 127 — bad image)
|
||||
)
|
||||
|
||||
# Dependency ordering — skip restarting a container if its dependency is also down.
|
||||
# Prevents watchdog from restarting Authelia before Mariadb is back up.
|
||||
# SPACE-SEPARATED STRINGS — converted to array at runtime.
|
||||
declare -A HOST1_WATCHDOG_DEPENDENCIES=(
|
||||
["Authelia"]="Mariadb-Authelia Redis-Authelia"
|
||||
["Authelia-Secondary"]="Mariadb-Authelia Redis-Authelia-Secondary"
|
||||
["NextCloud"]="Postgres-NextCloud"
|
||||
)
|
||||
|
||||
# Per-container appdata growth suppress ceilings in MB.
|
||||
# ONLY needed in specific cases — growth rate detection covers all containers automatically.
|
||||
# Use this when a container legitimately has large stable data and you want to guarantee
|
||||
# it never triggers a false-positive growth alert. Growth warnings are suppressed while the
|
||||
# container's dir stays below this ceiling; above it, warnings resume as normal.
|
||||
# 50GB=51200 25GB=25600 20GB=20480 15GB=15360 10GB=10240 5GB=5120
|
||||
declare -A HOST1_WATCHDOG_APPDATA_SIZES=(
|
||||
["Tdarr"]="25600" # 25GB — transcode cache grows legitimately during active jobs
|
||||
["7dtd"]="20480" # 20GB — game server world data, expected to be large
|
||||
)
|
||||
|
||||
# ━━━ Network Watchdog ━━━
|
||||
# Host-specific connectivity config for Watchdogs/System/network_watchdog.sh.
|
||||
HOST1_NETWORK_WATCHDOG_DDNS_DOMAIN="gmer4lfe.com"
|
||||
HOST1_NETWORK_WATCHDOG_DDNS_CONTAINER="Gmer4Lfe.com"
|
||||
HOST1_NETWORK_WATCHDOG_NPM_URL="https://gmer4lfe.com"
|
||||
|
||||
# ━━━ Docker Network Connect ━━━
|
||||
# Containers connected to custom networks at array start by docker_network_connect.sh.
|
||||
# Networks created if they don't exist — idempotent, safe to re-run.
|
||||
HOST1_NETWORK_CONNECT_CONTAINERS=(
|
||||
"memcached"
|
||||
"Npm-CrowdSec"
|
||||
)
|
||||
|
||||
HOST1_NETWORK_CONNECT_NETWORKS=(
|
||||
"high-availability"
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── FALLBACK ──────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ DDNS ━━━
|
||||
# DDNS containers HOST1 manages — started/stopped by fallback.sh per DDNS absolute rules:
|
||||
# Internet loss → stop immediately
|
||||
# Failover → HOST2 starts HOST1's DDNS as Tier 1 (before any other containers)
|
||||
# Handback → stop HOST1's DDNS on HOST2 → rsync → start containers → start local DDNS last
|
||||
HOST1_DDNS_CONTAINERS=(
|
||||
"Gmer4Lfe.com"
|
||||
)
|
||||
|
||||
# ━━━ Internet Loss ━━━
|
||||
# Containers stopped immediately on HOST1 when internet connection is lost.
|
||||
# Prevents external-facing services from operating without connectivity.
|
||||
FALLBACK_HOST1_STOP_ON_NO_NET=(
|
||||
"Gmer4Lfe.com"
|
||||
)
|
||||
|
||||
# ━━━ Fallback Tiers — HOST1 Runs for HOST2 ━━━
|
||||
# Containers HOST1 starts when HOST2 goes down.
|
||||
# Tier 1 is always immediate — vital services cannot wait.
|
||||
# Higher tiers activate after HOST2_TIER*_DELAY minutes (set in host2.conf).
|
||||
FALLBACK_HOST1_COVERS_HOST2_TIER1=(
|
||||
"Gmer4Lfe.us"
|
||||
"VaultWarden-Jayred365"
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_COVERS_HOST2_TIER2=(
|
||||
# "container-placeholder"
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_COVERS_HOST2_TIER3=(
|
||||
# "container-placeholder"
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_COVERS_HOST2_TIER4=(
|
||||
# "container-placeholder"
|
||||
)
|
||||
|
||||
# ━━━ Tier Delays — HOST1's Containers on HOST2 ━━━
|
||||
# How long HOST1 must be down before each tier activates on HOST2 — in minutes.
|
||||
# Tier 1 is always immediate — no delay var needed.
|
||||
HOST1_TIER2_DELAY=240 # 4 hours — NextCloud, Immich
|
||||
HOST1_TIER3_DELAY=720 # 12 hours — secondary services
|
||||
HOST1_TIER4_DELAY=1440 # 24 hours — arrs + downloaders
|
||||
|
||||
# ━━━ Rsync Writeback — HOST1 Appdata Back on Handback ━━━
|
||||
# Syncs HOST1 appdata BACK to HOST1 when it comes back online after a failover.
|
||||
# Containers stopped before writeback — clean source, no competing writes.
|
||||
#
|
||||
# HOST1_TIER1_WRITEBACK_DELAY: short outages skip Tier 1 writeback — primary state
|
||||
# is more reliable than dirty sync data for brief outages.
|
||||
HOST1_TIER1_WRITEBACK_DELAY=60 # skip Emby writeback if outage under 1hr
|
||||
|
||||
# Tier 4 automatically syncs HOST1_DAILY_SYNC_SHARES — only list paths NOT in that array.
|
||||
FALLBACK_HOST1_WRITEBACK_TIER1=(
|
||||
"/mnt/user/Media_Server/Emby" # watch states built up during outage
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_WRITEBACK_TIER2=(
|
||||
"/mnt/user/appdata-Fallback/Important-Data" # NextCloud + Postgres — files added during outage
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_WRITEBACK_TIER3=(
|
||||
# "location-placeholder"
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_WRITEBACK_TIER4=(
|
||||
"/mnt/user/appdata-Fallback/Arrs_Stack" # arr databases — downloads queued during outage
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── MEDIA ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Media Permissions ━━━
|
||||
# Shares that media_shares_permissions.sh applies PERMISSIONS_MODE and PERMISSIONS_OWNER to.
|
||||
# Runs first in DAILY_MAINTENANCE_SCRIPTS — arr cleanup depends on correct ownership.
|
||||
HOST1_MEDIA_PERMISSION_SHARES=(
|
||||
/mnt/user/Anime_Movies
|
||||
/mnt/user/Anime_Movies-Old
|
||||
/mnt/user/Anime_Shows
|
||||
/mnt/user/Anime_Shows-Old
|
||||
/mnt/user/appcache
|
||||
/mnt/user/Books
|
||||
/mnt/user/Downloads
|
||||
/mnt/user/Games
|
||||
/mnt/user/Intros
|
||||
/mnt/user/Kids_Movies
|
||||
/mnt/user/Kids_Tv_Shows
|
||||
/mnt/user/Movie_Recordings
|
||||
/mnt/user/Movies
|
||||
/mnt/user/Music
|
||||
/mnt/user/Music_Videos
|
||||
/mnt/user/Photo
|
||||
/mnt/user/Sports
|
||||
/mnt/user/stand-up_comedy
|
||||
/mnt/user/Tv_Recordings
|
||||
/mnt/user/Tv_Shows
|
||||
/mnt/user/YouTube
|
||||
)
|
||||
|
||||
# ━━━ Media Cleaner ━━━
|
||||
# Folder lists for media_cleaner.sh — two profiles: anime and media.
|
||||
# File patterns shared across all servers — defined in master.conf.
|
||||
# Called via DAILY_MAINTENANCE_SCRIPTS. Run manually: Media/media_cleaner.sh anime|media
|
||||
HOST1_ANIME_CLEAN_FOLDERS=(
|
||||
/mnt/user/Anime_Movies
|
||||
/mnt/user/Anime_Movies-Old
|
||||
/mnt/user/Anime_Shows
|
||||
/mnt/user/Anime_Shows-Old
|
||||
)
|
||||
|
||||
HOST1_MEDIA_CLEAN_FOLDERS=(
|
||||
/mnt/user/Kids_Movies
|
||||
/mnt/user/Kids_Tv_Shows
|
||||
/mnt/user/Movies
|
||||
/mnt/user/Music
|
||||
/mnt/user/Sports
|
||||
/mnt/user/stand-up_comedy
|
||||
/mnt/user/Tv_Shows
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── MONITORS ──────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Certificate Monitor ━━━
|
||||
# Domains checked via direct openssl connection — not relying on NPM's certificate state.
|
||||
# Checks the actual certificate served, not what NPM thinks it has.
|
||||
# Thresholds (CERT_WARN_DAYS, CERT_CRIT_DAYS) defined in master.conf.
|
||||
HOST1_CERT_MONITOR_DOMAINS=(
|
||||
"Gmer4Lfe.com"
|
||||
"Gmer4Lfe.us"
|
||||
)
|
||||
|
||||
# ━━━ SMART Health ━━━
|
||||
# Drives skipped in SMART attribute monitoring — hardware is server-specific.
|
||||
# Thresholds read from dynamix.cfg at runtime — fallbacks in master.conf.
|
||||
HOST1_SMART_IGNORE_DRIVES=(
|
||||
"sda" # boot USB — SMART not meaningful on flash drives
|
||||
)
|
||||
|
||||
# ━━━ ZFS Report ━━━
|
||||
# Pools excluded from the weekly ZFS health report — reduces noise from single-disk array pools.
|
||||
# These are individual array disks formatted as ZFS — converting to XFS over time via unBalance.
|
||||
# Pool health thresholds defined in master.conf.
|
||||
HOST1_ZFS_REPORT_IGNORE_POOLS=(
|
||||
"disk5"
|
||||
"disk6"
|
||||
"disk8"
|
||||
"disk9"
|
||||
"disk10"
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── TRANSCODES ────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# Ramdisk size ceiling — tmpfs only uses RAM actually needed, not the full size upfront.
|
||||
# Real-world: 9 streams peaked at ~5.5GB — 10G gives generous headroom on 128GB RAM.
|
||||
HOST1_RAMDISK_SIZE="10G"
|
||||
|
||||
# Usage thresholds — coupled to HOST1_RAMDISK_SIZE, adjust all three together if size changes.
|
||||
# Hysteresis gap (8.5 - 7 = 1.5GB) prevents flip-flop between ramdisk and SSD.
|
||||
HOST1_RAMDISK_WARN_GB=8.5 # flip to SSD when ramdisk usage reaches this
|
||||
HOST1_RAMDISK_LOW_GB=7 # flip back to ramdisk when usage drops to this
|
||||
|
||||
# SSD fallback path — where transcodes land when ramdisk exceeds HOST1_RAMDISK_WARN_GB.
|
||||
# Must be on cache pool — array disks too slow for active transcode writes.
|
||||
HOST1_TRANSCODE_SSD="/mnt/cache/Temp_Storage/Emby/Transcodes/"
|
||||
|
||||
# Media servers sharing the ramdisk transcode space on HOST1.
|
||||
# Format: "ContainerName|URL|APIKey|Type" — Type: emby | jellyfin | plex
|
||||
# Entries with placeholder API keys are skipped automatically.
|
||||
# ⚠️ Tdarr does NOT belong here — keep Tdarr on SSD, not ramdisk.
|
||||
HOST1_TRANSCODE_SERVERS=(
|
||||
"${HOST1_EMBY_CONTAINER}|${HOST1_EMBY_URL}|${HOST1_EMBY_API_KEY}|emby"
|
||||
"${HOST1_JELLYFIN_CONTAINER}|${HOST1_JELLYFIN_URL}|${HOST1_JELLYFIN_API_KEY}|jellyfin"
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── ARR STACK ─────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Used by arr cleanup scripts and arrs_failed_stalled_recovery.sh.
|
||||
# detect_hosts() selects HOST1 vars when running on HOST1.
|
||||
#
|
||||
# PATH MAPS — container path → host path translation.
|
||||
# Arr stores file paths using container-internal paths — scripts need host paths to scan.
|
||||
# Add one entry per root folder in arr Settings → Media Management → Root Folders.
|
||||
|
||||
# ━━━ Downloaders ━━━
|
||||
# Used by downloaders_reset.sh — runs every 30min via CRITICAL_MAINTENANCE_SCRIPTS.
|
||||
# Clears stuck states, purges old history, prepares each client for a clean cycle.
|
||||
|
||||
# slskd — clears stuck searches, dead transfers, purges expired failed imports.
|
||||
# SLSKD_FAILED_IMPORTS_DIR: where Soularr moves albums Lidarr rejected.
|
||||
HOST1_SLSKD_URL="http://localhost:8980"
|
||||
HOST1_SLSKD_API_KEY="4bF9kL2mNpQrT7vWxYz1A3dEgHjKoRsU"
|
||||
HOST1_SLSKD_FAILED_IMPORTS_DIR="/mnt/user/Temp_Storage/Slskd/completed/failed_imports"
|
||||
|
||||
# SABnzbd
|
||||
HOST1_SABNZBD_URL="http://localhost:8180"
|
||||
HOST1_SABNZBD_API_KEY="8bfefe41d83b4d50883e32859b55ca9a"
|
||||
|
||||
# qBittorrent — deleteFiles=false removes torrent from qBit but leaves files on disk.
|
||||
# Radarr/Sonarr manage actual files independently.
|
||||
HOST1_QBIT_URL="http://localhost:8080"
|
||||
HOST1_QBIT_USERNAME="root"
|
||||
HOST1_QBIT_PASSWORD="Stay0utD!ck"
|
||||
|
||||
# ━━━ Lidarr — HOST1 only ━━━
|
||||
# HOST2 does not run Lidarr — HOST1_LIDARR_RECOVERY flag handles the exit cleanly.
|
||||
HOST1_LIDARR_URL="http://localhost:8686"
|
||||
HOST1_LIDARR_API_KEY="b2977e71ef074bc0a0529d9fcce3b2dc"
|
||||
HOST1_LIDARR_MUSIC_ROOT="/mnt/user/Music-New"
|
||||
HOST1_FANART_API_KEY="Yd7147a43b692df0b364b94dc47efb81"
|
||||
HOST1_LASTFM_API_KEY="be6dc169c33ae263e690c30d18b7491d"
|
||||
|
||||
declare -A HOST1_LIDARR_PATH_MAP=(
|
||||
["/ext-music"]="/mnt/user/Music-New"
|
||||
)
|
||||
|
||||
# ━━━ Sonarr ━━━
|
||||
HOST1_SONARR_URL="http://localhost:8989"
|
||||
HOST1_SONARR_API_KEY="130decd3db5b4c25afad64864cd03f9f"
|
||||
HOST1_SONARR_TV_ROOT="/mnt/user/Tv_Shows"
|
||||
|
||||
# Note: stand-up_comedy in both Sonarr + Radarr — TV specials and movie specials, one folder
|
||||
declare -A HOST1_SONARR_PATH_MAP=(
|
||||
["/tv"]="/mnt/user/Tv_Shows"
|
||||
["/ext-standup-comedy"]="/mnt/user/stand-up_comedy"
|
||||
["/kids tv"]="/mnt/user/Kids_Tv_Shows"
|
||||
["/ext-anime-shows"]="/mnt/user/Anime_Shows-Old"
|
||||
)
|
||||
|
||||
# ━━━ Radarr ━━━
|
||||
HOST1_RADARR_URL="http://localhost:7878"
|
||||
HOST1_RADARR_API_KEY="d43a3ec6cf1549edb4af0cc63f98b2a9"
|
||||
HOST1_TMDB_API_KEY="3dac5e2e49b5540472d2eafec4f01260"
|
||||
HOST1_RADARR_MOVIES_ROOT="/mnt/user/Movies"
|
||||
|
||||
# Note: stand-up_comedy in both Radarr + Sonarr — movie specials and TV specials, one folder
|
||||
declare -A HOST1_RADARR_PATH_MAP=(
|
||||
["/movies"]="/mnt/user/Movies"
|
||||
["/kids movies"]="/mnt/user/Kids_Movies"
|
||||
["/ext-stand-up-comedy"]="/mnt/user/stand-up_comedy"
|
||||
["/anime-movies"]="/mnt/user/Anime_Movies-Old"
|
||||
)
|
||||
|
||||
# ━━━ Arr Recovery Toggles ━━━
|
||||
# false = skip that arr on this host — exits cleanly without error
|
||||
HOST1_SONARR_RECOVERY=true
|
||||
HOST1_RADARR_RECOVERY=true
|
||||
HOST1_LIDARR_RECOVERY=true # HOST1 only — exits cleanly on HOST2
|
||||
|
||||
# ==============================================================================================
|
||||
# ── SYSTEM WATCHDOG ───────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Per-host check toggles and NIC config for system_watchdog.sh.
|
||||
# Aliased by detect_hosts() — script uses unprefixed SYS_WATCHDOG_* names.
|
||||
# HOST1: TR1950X 128GB — full media server, active transcoding, ZFS cache pools.
|
||||
#
|
||||
# Three-tier response — all critical checks enabled by default on HOST1:
|
||||
# Tier 1 (bypass strikes, reboot now): docker daemon, rootfs full, kernel oops, FD, /boot
|
||||
# Tier 2 (bypass strikes with OOM): RAM critical + OOM kills in cycle
|
||||
# Tier 3 (standard strike system): everything else
|
||||
#
|
||||
# RAM tiers, OOM limits, and reboot loop settings in master.conf System Watchdog section.
|
||||
|
||||
# ━━━ Primary NIC ━━━
|
||||
# Network interface for NIC state check — verify with: ip link show | grep "^[0-9]"
|
||||
# Common values: eth0, bond0, br0, eno1
|
||||
HOST1_SYS_WATCHDOG_NIC="eth0"
|
||||
|
||||
# ━━━ Tier 1 — Critical Checks ━━━
|
||||
# These bypass the strike system — a single hit triggers immediate reboot.
|
||||
# Disabling any of these is not recommended — they protect against acute system failure.
|
||||
|
||||
# Docker daemon unresponsive → try restart, reboot if restart fails.
|
||||
# Without a working daemon docker_watchdog.sh is blind and containers cannot be managed.
|
||||
HOST1_SYS_WATCHDOG_CHECK_DOCKER_DAEMON=true
|
||||
|
||||
# rootfs at critical threshold (ROOTFS_CRITICAL_PCT=99) → reboot immediately.
|
||||
# At 99% rootfs writes fail silently — logs stop, Docker errors out, SSH may stop working.
|
||||
# Standard 95% threshold still uses strike system — only 99%+ is critical tier.
|
||||
HOST1_SYS_WATCHDOG_CHECK_ROOTFS=true
|
||||
|
||||
# Kernel BUG/Oops in dmesg delta since last cycle → reboot immediately.
|
||||
# A kernel oops means the kernel ran with a corrupted state — stability is not guaranteed.
|
||||
HOST1_SYS_WATCHDOG_CHECK_KERNEL_OOPS=true
|
||||
|
||||
# File descriptor exhaustion at FD_CRITICAL_PCT (95%) → reboot immediately.
|
||||
# At 95% FD: new connections fail, Docker can't spawn processes, SSH drops.
|
||||
HOST1_SYS_WATCHDOG_CHECK_FD=true
|
||||
|
||||
# /boot read-only detected → reboot immediately.
|
||||
# Unexpected read-only /boot means state files and config writes are silently failing.
|
||||
# Fallback state, watchdog reboot log, and lock files all go stale silently.
|
||||
HOST1_SYS_WATCHDOG_CHECK_BOOT=true
|
||||
|
||||
# ━━━ Tier 2 — Urgent OOM Check ━━━
|
||||
# Bypass strikes when RAM is critically low AND OOM kill rate confirms active crisis.
|
||||
# Both must be enabled for Tier 2 bypass to function — disable either to always use strikes.
|
||||
|
||||
# Track kernel OOM kills each cycle via /proc/vmstat oom_kill delta.
|
||||
# Also provides diagnostic context in reboot messages (which processes were killed).
|
||||
HOST1_SYS_WATCHDOG_CHECK_OOM=true
|
||||
|
||||
# Free RAM check — required for both Tier 2 bypass and RAM tier logic.
|
||||
# Tiers: MEM_WARN_GB(10) → notify | MEM_SHUTDOWN_GB(6) → stop containers | MEM_GB(4) → strikes
|
||||
HOST1_SYS_WATCHDOG_CHECK_RAM=true
|
||||
|
||||
# ━━━ Tier 3 — Standard Checks (strike system) ━━━
|
||||
# Each check must fail SYS_WATCHDOG_STRIKE_LIMIT consecutive cycles before action is taken.
|
||||
# Single spikes are ignored — sustained problems trigger reboot.
|
||||
|
||||
# /var/log filesystem usage above SYS_WATCHDOG_LOG_PCT.
|
||||
# Log spam (Docker log storms, syslog loops) fills rootfs — indicates something broken.
|
||||
HOST1_SYS_WATCHDOG_CHECK_LOG=true
|
||||
|
||||
# ZFS ARC memory pinned above SYS_WATCHDOG_ARC_PINNED_PCT after cache drop.
|
||||
# Enabled on HOST1 — ZFS cache pools actively used. Disable on hosts without ZFS.
|
||||
HOST1_SYS_WATCHDOG_CHECK_ARC=true
|
||||
|
||||
# CPU temperature above SYS_WATCHDOG_CPU_TEMP_MAX (95°C).
|
||||
# Sustained high temp causes kernel throttling or panic. Requires lm-sensors.
|
||||
HOST1_SYS_WATCHDOG_CHECK_CPU_TEMP=true
|
||||
|
||||
# Load average above SYS_WATCHDOG_LOAD_MULTIPLIER × core count.
|
||||
# DISABLED on HOST1 — Tdarr and Emby cause legitimate sustained load spikes during encoding.
|
||||
# Enable on idle servers or adjust SYS_WATCHDOG_LOAD_MULTIPLIER if load is always high.
|
||||
HOST1_SYS_WATCHDOG_CHECK_LOAD=false
|
||||
|
||||
# Zombie process count above SYS_WATCHDOG_ZOMBIE_LIMIT (50).
|
||||
# Large zombie counts indicate serious process management failure — something is stuck.
|
||||
HOST1_SYS_WATCHDOG_CHECK_ZOMBIES=true
|
||||
|
||||
# Check docker_watchdog.sh persistent skip list — required containers on skip list.
|
||||
# Cross-watchdog coordination: if docker_watchdog gave up, system_watchdog escalates.
|
||||
# ENABLED — HOST1 fully built and operational, skip list is meaningful.
|
||||
HOST1_SYS_WATCHDOG_CHECK_CONTAINERS=true
|
||||
|
||||
# /tmp filesystem usage above SYS_WATCHDOG_TMP_PCT with auto-clear attempt.
|
||||
# Script tries to clear aged /tmp files first — only strikes if clear fails.
|
||||
# Lock files, rsync temp files, and Docker ops use /tmp — 100% means lock failures.
|
||||
HOST1_SYS_WATCHDOG_CHECK_TMP=true
|
||||
|
||||
# Array disk error count delta in /proc/mdstat — accumulating errors = disk failing now.
|
||||
# Triggers on SYS_WATCHDOG_MDSTAT_ERROR_LIMIT new errors in one cycle.
|
||||
HOST1_SYS_WATCHDOG_CHECK_MDSTAT=true
|
||||
|
||||
# Primary NIC operstate — detects NIC going down (physical or driver failure).
|
||||
# Uses HOST1_SYS_WATCHDOG_NIC above. Strike system — brief flaps don't trigger reboot.
|
||||
HOST1_SYS_WATCHDOG_CHECK_NETWORK=true
|
||||
|
||||
# sshd running check — attempts restart before escalating.
|
||||
# sshd down = no remote access. Script tries rc.sshd start, notifies, strikes on failure.
|
||||
HOST1_SYS_WATCHDOG_CHECK_SSHD=true
|
||||
|
||||
# Runaway process detection — single process above SYS_WATCHDOG_RUNAWAY_CPU_PCT sustained.
|
||||
# DISABLED — Tdarr encoding and Emby transcoding legitimately peg CPU for extended periods.
|
||||
# Enable only if HOST1 has no CPU-intensive workloads.
|
||||
HOST1_SYS_WATCHDOG_CHECK_RUNAWAY=false
|
||||
|
||||
# ==============================================================================================
|
||||
# ── RESOURCE MANAGER ──────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Containers to manage under pressure — see master.conf RW_CRITICAL_CONTAINERS for exclusions.
|
||||
|
||||
# docker pause at medium pressure (RAM < RW_RAM_MEDIUM_GB or load > medium threshold)
|
||||
# Suspended in-place — instant to pause/unpause, no state lost, no restart delay.
|
||||
HOST1_RW_PAUSE_CONTAINERS=(
|
||||
"Huntarr" # arr search automation — safe to suspend
|
||||
"Cleanuparr" # download cleanup — safe to suspend
|
||||
"Healarr" # arr health checks — safe to suspend
|
||||
"Soularr" # Slskd automation — background only
|
||||
"ChannelTube" # YouTube archiver — background only
|
||||
"Pinchflat" # YouTube archiver — background only
|
||||
)
|
||||
|
||||
# docker stop at hard pressure (RAM < RW_RAM_HARD_GB)
|
||||
# Full stop — these are optional/heavy services that free significant RAM when stopped.
|
||||
# resource_watchdog.sh restarts them when pressure fully clears (RAM >= RW_RAM_RECOVER_GB).
|
||||
HOST1_RW_STOP_CONTAINERS=(
|
||||
"LocalAI" # GPU/CPU heavy — largest RAM consumer when idle
|
||||
"7DaysToDie" # game server — optional
|
||||
"V-Rising" # game server — optional
|
||||
"Code-Server" # IDE — not needed during pressure events
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── AUTH STACK ────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Credentials for the Varaverk Auth Stack page (NPM, lldap, Authelia).
|
||||
|
||||
# ━━━ NginxProxyManager ━━━
|
||||
HOST1_NPM_URL="http://localhost:7818"
|
||||
HOST1_NPM_USER="failedproxy@gmail.com"
|
||||
HOST1_NPM_PASS="183134\$eanHess"
|
||||
|
||||
# ━━━ lldap ━━━
|
||||
HOST1_LLDAP_URL="http://localhost:17170"
|
||||
HOST1_LLDAP_USER="admin"
|
||||
HOST1_LLDAP_PASS="183134\$eanHess"
|
||||
|
||||
# ━━━ Authelia ━━━
|
||||
HOST1_AUTHELIA_CONFIG="/mnt/user/appdata-Fallback/Critical-Data/Authelia/configuration.yml"
|
||||
HOST1_AUTHELIA_CONTAINER="Authelia"
|
||||
|
||||
# ==============================================================================================
|
||||
# ──────────────────────── End Of HOST1 Variables ──────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
HOST1_UNRAID_API_KEY="1825c3a2e03ea5089974f4da2e171aa2d5907a1dea23cc479c33e492c8ff4dbb"
|
||||
@@ -0,0 +1,789 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ========================== HOST1 CONFIGURATION — unRAID-Gmer4Lfe ============================
|
||||
# ==============================================================================================
|
||||
# HOST1-specific variables — credentials, container names, share paths, failover lists.
|
||||
# Sourced after master.conf — values here extend shared profile arrays and add HOST1-specific
|
||||
# identity, credentials, and container configuration.
|
||||
#
|
||||
# Sparse checkout (git) ensures HOST2 never receives this file.
|
||||
# HOST2 never sees HOST1 credentials — clean separation at the file level.
|
||||
#
|
||||
# DO NOT put shared config here — thresholds, toggles, profiles belong in master.conf.
|
||||
# DO NOT put HOST2 variables here — they belong in host2.conf.
|
||||
#
|
||||
# ── INDEX ─────────────────────────────────────────────────────────────────────────────────────
|
||||
#
|
||||
# ── IDENTITY & CONNECTIVITY ────────────────────────────────────────────────────────────────
|
||||
# IDENTITY hostname, SSH key
|
||||
# EMBY container name, URL, API key
|
||||
# NOTIFICATIONS Discord webhook
|
||||
# PARTNERSHIP auth containers, backup paths
|
||||
#
|
||||
# ── RSYNC ──────────────────────────────────────────────────────────────────────────────────
|
||||
# DAILY SYNC SHARES media shares HOST1 owns and pushes to HOST2
|
||||
# WEEKLY SYNC SHARES appdata shares synced weekly (Sunday 2:30am)
|
||||
# CRITICAL SYNC SHARES appdata shares synced every 30 minutes
|
||||
# BACKUP VERIFY shares for checksum verification against remote
|
||||
# HOST1 RSYNC PROFILE host1-appdata profile for HOST1-specific appdata syncs
|
||||
#
|
||||
# ── DOCKER ─────────────────────────────────────────────────────────────────────────────────
|
||||
# DOCKER DAILY RESTART containers restarted daily
|
||||
# DOCKER WEEKLY RESTART containers restarted weekly
|
||||
# DOCKER WATCHDOG memory limits, health URLs, required containers, ignore list
|
||||
# DOCKER NETWORK CONNECT networks and containers for docker_network_connect.sh
|
||||
#
|
||||
# ── FALLBACK ───────────────────────────────────────────────────────────────────────────────
|
||||
# DDNS DDNS containers managed by HOST1
|
||||
# INTERNET LOSS containers stopped when internet is lost
|
||||
# FALLBACK TIERS what HOST1 runs for HOST2 per tier
|
||||
# TIER DELAYS how long HOST1 must be down before each tier activates on HOST2
|
||||
# RSYNC WRITEBACK HOST1 appdata synced back on handback
|
||||
#
|
||||
# ── MEDIA ──────────────────────────────────────────────────────────────────────────────────
|
||||
# MEDIA PERMISSIONS share list for media_shares_permissions.sh
|
||||
# MEDIA CLEANER folder lists for media_cleaner.sh
|
||||
#
|
||||
# ── MONITORS ───────────────────────────────────────────────────────────────────────────────
|
||||
# CERTIFICATE MONITOR domains checked for SSL expiry
|
||||
# SMART HEALTH drives to skip in SMART monitoring
|
||||
# ZFS REPORT pools to exclude from ZFS health report
|
||||
#
|
||||
# ── TRANSCODES ─────────────────────────────────────────────────────────────────────────────
|
||||
# TRANSCODES ramdisk size, thresholds, SSD path, server array
|
||||
#
|
||||
# ── ARR STACK ──────────────────────────────────────────────────────────────────────────────
|
||||
# DOWNLOADERS slskd, SABnzbd, qBittorrent credentials and URLs
|
||||
# LIDARR URL, API key, path map
|
||||
# SONARR URL, API key, path map
|
||||
# RADARR URL, API key, path map
|
||||
# ARR RECOVERY per-arr recovery toggles
|
||||
#
|
||||
# ── SYSTEM WATCHDOG ────────────────────────────────────────────────────────────────────────
|
||||
# SYSTEM WATCHDOG per-host check toggles and NIC configuration
|
||||
#
|
||||
# ── RESOURCE MANAGER ───────────────────────────────────────────────────────────────────────
|
||||
# RESOURCE MANAGER containers paused/stopped under memory pressure
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
# ==============================================================================================
|
||||
# ── IDENTITY & CONNECTIVITY ───────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Identity ━━━
|
||||
# HOST1 hostname lives in master.conf (not a credential — safe for all servers).
|
||||
# SSH key used for all server-to-server operations — rsync, failover container commands.
|
||||
# Must be in /root/.ssh/ and authorised in HOST2's /root/.ssh/authorized_keys.
|
||||
HOST1_SSH_KEY="/root/.ssh/gmer4lfe_rsync_automation"
|
||||
HOST1_OWNER="gmer4lfe"
|
||||
HOST1_OWNER_EMAIL="gmer4lfe@gmail.com"
|
||||
|
||||
# ━━━ Emby ━━━
|
||||
# Referenced by transcode_manager.sh, emby_session_report.sh, emby_database_repair.sh,
|
||||
# weekly_sync_maintenance.sh, and HOST1_TRANSCODE_SERVERS below.
|
||||
# API key: Emby Dashboard → API Keys → + New Key
|
||||
HOST1_EMBY_CONTAINER="Emby"
|
||||
HOST1_EMBY_URL="http://localhost:8096"
|
||||
HOST1_EMBY_API_KEY="0c27448d93a7431f9ac63569f7655829"
|
||||
|
||||
# ━━━ Jellyfin ━━━
|
||||
# API key: Jellyfin Dashboard → Administration → API Keys → + New Key
|
||||
HOST1_JELLYFIN_CONTAINER="Jellyfin"
|
||||
HOST1_JELLYFIN_URL="http://localhost:8095"
|
||||
HOST1_JELLYFIN_API_KEY="4e820e7df74c4933acec212b1996314e"
|
||||
|
||||
# ━━━ Gitea ━━━
|
||||
# Personal access token for gitea_ssh_setup.sh — registers this server's SSH public key
|
||||
# with Gitea so git operations use key auth instead of passwords.
|
||||
# Create in Gitea: Settings → Applications → Generate Token → scope: write:user
|
||||
HOST1_GITEA_API_TOKEN=""
|
||||
|
||||
# ━━━ Notifications ━━━
|
||||
# Discord webhook — leave blank to disable.
|
||||
# Per-host so HOST1 and HOST2 can post to different channels or only one server notifies.
|
||||
HOST1_DISCORD_WEBHOOK=""
|
||||
|
||||
# ━━━ Partnership ━━━
|
||||
# HOST1 is always the owner (source of truth) unless --transfer has been run.
|
||||
# See README-Partnership.md and master.conf PARTNERSHIP section for full lifecycle docs.
|
||||
|
||||
# Auth containers reconfigured on onboard/offboard.
|
||||
# Format: "ContainerName|WebUIPort"
|
||||
# On onboard → WebUI pointed at owner's Tailscale IP (mirror clicks NPM, gets owner's auth)
|
||||
# On offboard → WebUI pointed back at localhost
|
||||
HOST1_PARTNERSHIP_AUTH_WEBUIS=(
|
||||
"NginxProxyManager|81"
|
||||
"Lldap-Gmer4Lfe|17170"
|
||||
"Authelia|9091"
|
||||
"Authelia-Secondary|9092"
|
||||
)
|
||||
|
||||
# XML templates (from this server's templates-user/) pushed to mirror during onboard.
|
||||
# These become the mirror's active auth stack, backed by the rsync-synced appdata.
|
||||
# Update filename if Lldap is renamed to drop the host suffix.
|
||||
HOST1_PARTNERSHIP_AUTH_STACK=(
|
||||
# Dependencies first — Mariadb/Redis must be healthy before Authelia starts
|
||||
"my-Mariadb-Authelia.xml"
|
||||
"my-Mariadb-Authelia-Secondary.xml"
|
||||
"my-Redis-Authelia.xml"
|
||||
"my-Redis-Authelia-Secondary.xml"
|
||||
# Auth apps — deployed after their deps are confirmed healthy
|
||||
"my-Authelia.xml"
|
||||
"my-Authelia-Secondary.xml"
|
||||
"my-NginxProxyManager.xml"
|
||||
"my-Lldap-Gmer4Lfe.xml"
|
||||
# Source of truth — must be available on HOST2 independently of the auth stack
|
||||
"my-Gitea.xml"
|
||||
)
|
||||
|
||||
# XML templates pushed to mirror for the arr stack during onboard.
|
||||
# Deps (e.g. databases) first if any — same ordering rule as auth stack.
|
||||
HOST1_PARTNERSHIP_ARR_STACK=(
|
||||
# "my-Sonarr.xml"
|
||||
# "my-Radarr.xml"
|
||||
# "my-Lidarr.xml"
|
||||
# "my-Prowlarr.xml"
|
||||
# "my-Bazarr.xml"
|
||||
)
|
||||
|
||||
# Paths HOST2 should collect during the grace window after offboard.
|
||||
# Notified on offboard — no auto-deletion, HOST2 must collect manually within PARTNERSHIP_GRACE_HOURS.
|
||||
HOST1_PARTNERSHIP_MIRROR_BACKUPS=(
|
||||
# "/mnt/user/appdata-Fallback/Jayred365-Emby"
|
||||
)
|
||||
|
||||
# Containers parked on this server when partnership is active.
|
||||
# Stopped on onboard (owner deploys its stack instead), restarted on offboard.
|
||||
HOST1_PARTNERSHIP_OWN_CONTAINERS=(
|
||||
# "Emby"
|
||||
# "NginxProxyManager"
|
||||
)
|
||||
|
||||
# Emby admin provisioning — toggle is owner-only, credentials are per-host.
|
||||
# Owner enables/disables the feature. Each host sets the account they want on the shared Emby.
|
||||
# On onboard: owner reads mirror's HOST*_PARTNERSHIP_EMBY_ADMIN_* and creates that account.
|
||||
# On offboard: account is deleted. Username collision → onboard exits with error.
|
||||
HOST1_PARTNERSHIP_PROVISION_EMBY_ADMIN=false # owner controls whether Emby is shared
|
||||
HOST1_PARTNERSHIP_EMBY_PORT=8096
|
||||
HOST1_PARTNERSHIP_EMBY_ADMIN_USER="" # this server's desired Emby username
|
||||
HOST1_PARTNERSHIP_EMBY_ADMIN_PASS="" # this server's desired Emby password
|
||||
|
||||
# ==============================================================================================
|
||||
# ── RSYNC ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Daily Sync Shares ━━━
|
||||
# Shares HOST1 pushes to all other nodes every night (1am via daily_sync_maintenance.sh).
|
||||
# Mesh model: every node pushes every media share — no ownership, no mirrors.
|
||||
# arr_sync ensures all arr libraries converge (union). rsync spreads files (additive, no --delete).
|
||||
# arr_cleanup removes true orphans based on local arr state.
|
||||
# Any node can download content to any share — it propagates to all nodes on the next cycle.
|
||||
# Nextcloud is intentionally one-directional (HOST1→HOST2 offsite backup — not arr-managed).
|
||||
# Uses DEFAULT_RSYNC_OPTS from master.conf — no profile needed.
|
||||
# For shares needing container stops or custom options — add a profile in master.conf.
|
||||
HOST1_DAILY_SYNC_SHARES=(
|
||||
/mnt/user/Books
|
||||
/mnt/user/Intros
|
||||
/mnt/user/Kids_Movies
|
||||
/mnt/user/Kids_Tv_Shows
|
||||
/mnt/user/Movies
|
||||
/mnt/user/Music
|
||||
/mnt/user/Music_Videos
|
||||
/mnt/user/Nextcloud
|
||||
/mnt/user/stand-up_comedy
|
||||
/mnt/user/Sports
|
||||
# /mnt/user/Tv_Shows
|
||||
/mnt/user/Anime_Shows-Old
|
||||
/mnt/user/Anime_Movies-Old
|
||||
/mnt/user/Anime_Movies
|
||||
/mnt/user/Anime_Shows
|
||||
)
|
||||
|
||||
# Personal encrypted shares — synced for offsite backup, independent of media shares.
|
||||
# ZFS encrypted at dataset level — remote receives encrypted blocks, cannot read content.
|
||||
# See README-Rsync_Setup.md for ZFS encryption setup before uncommenting.
|
||||
HOST1_PERSONAL_SHARES=(
|
||||
# /mnt/user/HOST1-Personal # uncomment after creating encrypted dataset
|
||||
)
|
||||
|
||||
# ━━━ Weekly Sync Shares ━━━
|
||||
# Appdata shares synced during the weekly maintenance window (Sunday 2:30am).
|
||||
# Containers stopped both sides before sync — full clean state guaranteed.
|
||||
# Profiles drive container stops, excludes, and options — configured in master.conf RSYNC section.
|
||||
# Order matters — Emby first (larger transfer), then Critical-Data (auth stack).
|
||||
HOST1_WEEKLY_SYNC_SHARES=(
|
||||
"/mnt/user/Media_Server/Emby" # emby profile — full clean mirror
|
||||
"/mnt/user/appdata-Fallback/Critical-Data" # critical-data profile — auth stack
|
||||
)
|
||||
|
||||
# ━━━ Intermediate Sync Shares ━━━
|
||||
# Shares synced every 4 hours by intermediate_sync_maintenance.sh.
|
||||
# Uses DEFAULT_RSYNC_OPTS (no --delete) — for sub-daily propagation of metadata or watch state.
|
||||
# Full media share sync stays in the daily window. Leave empty to skip mid-day rsync entirely.
|
||||
HOST1_INTERMEDIATE_SYNC_SHARES=(
|
||||
# Add shares here to enable mid-day rsync
|
||||
# Example: "/mnt/user/Emby_Metadata"
|
||||
)
|
||||
|
||||
# ━━━ Critical Sync Shares ━━━
|
||||
# Appdata shares synced every 30 minutes by critical_sync_maintenance.sh.
|
||||
# Format: "/path/to/share" or "/path/to/share|profile-name"
|
||||
# Order matters — Critical-Data first (auth stack), then Emby dirty sync.
|
||||
HOST1_CRITICAL_SYNC_SHARES=(
|
||||
"/mnt/user/appdata-Fallback/Critical-Data|critical-fallback" # auth dirty sync — stays running
|
||||
"/mnt/user/Media_Server/Emby|emby-fallback" # Emby dirty sync — stays running
|
||||
)
|
||||
|
||||
# ━━━ Backup Verify ━━━
|
||||
# Shares verified by backup_verify.sh — random file checksum comparison against remote.
|
||||
# Leave empty to use HOST1_DAILY_SYNC_SHARES automatically.
|
||||
# Sample size and minimum file size defined in master.conf.
|
||||
HOST1_BACKUP_VERIFY_SHARES=(
|
||||
# leave empty to use HOST1_DAILY_SYNC_SHARES automatically
|
||||
)
|
||||
|
||||
# ━━━ HOST1 Rsync Profile — host1-appdata ━━━
|
||||
# HOST1-specific appdata sync profile — extends the shared PROFILE_* arrays in master.conf.
|
||||
# Use for appdata unique to HOST1 (Organizrv2, VaultWarden, UptimeKuma etc.)
|
||||
# Shared appdata (auth stack, Emby) use dedicated profiles defined in master.conf.
|
||||
# Run manually: bash Rsync/rsync.sh /mnt/user/appdata-Fallback/HOST1-Appdata --profile=host1-appdata
|
||||
PROFILE_RSYNC_OPTS[host1-appdata]="-av --info=progress2 --bwlimit=${PROFILE_BW_LIMIT[host1-appdata]:-8000}"
|
||||
PROFILE_BW_LIMIT[host1-appdata]=8000
|
||||
PROFILE_RETRY_COUNT[host1-appdata]=3
|
||||
PROFILE_SLEEP[host1-appdata]=300
|
||||
PROFILE_CRITICAL_CONTAINER_NAMES[host1-appdata]="Organizrv2-Gmer4Lfe UptimeKuma-Gmer4Lfe VaultWarden-Gmer4Lfe"
|
||||
PROFILE_DELAYED_CONTAINERS[host1-appdata]=""
|
||||
PROFILE_CONTAINER_DELAY[host1-appdata]=5
|
||||
PROFILE_EXCLUDE_DIRS[host1-appdata]="logs *.tmp"
|
||||
|
||||
# ==============================================================================================
|
||||
# ── DOCKER ────────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Docker Daily Restart ━━━
|
||||
# Containers restarted every day via DAILY_MAINTENANCE_SCRIPTS.
|
||||
# Dispatcharr degrades over time without restart — daily is intentional, not just housekeeping.
|
||||
# Order matters — auth stack first, then media services.
|
||||
HOST1_DAILY_RESTART_CONTAINERS=(
|
||||
"NginxProxyManager"
|
||||
"Lldap-Gmer4Lfe"
|
||||
"Authelia"
|
||||
"Authelia-Secondary"
|
||||
"Dispatcharr-Iptv-Users"
|
||||
"Dispatcharr" # Live TV scheduler — degrades without daily restart
|
||||
"Dispatcharr-Basic"
|
||||
"ErsatzTV-Emby"
|
||||
"Slskd" # Soulseek connection drops after extended uptime; restart refreshes share index
|
||||
)
|
||||
|
||||
# ━━━ Docker Weekly Restart ━━━
|
||||
# Less critical services restarted weekly via WEEKLY_MAINTENANCE_SCRIPTS (Sunday 2:30am).
|
||||
# Containers already stopped for weekly sync — restart adds zero extra downtime.
|
||||
HOST1_WEEKLY_RESTART_CONTAINERS=(
|
||||
"NextCloud"
|
||||
"Organizrv2-Gmer4Lfe"
|
||||
"AdGuard-Home"
|
||||
"Immich-Gmer4Lfe"
|
||||
)
|
||||
|
||||
# ━━━ Docker Watchdog ━━━
|
||||
# Per-HOST1 container configuration for docker_watchdog.sh.
|
||||
# Shared thresholds and toggles live in master.conf.
|
||||
|
||||
# Memory hard limits in MB — immediate restart if exceeded.
|
||||
# Set at "container is clearly broken" not "container is busy".
|
||||
# 20GB=20480 18GB=18432 16GB=16384 12GB=12288 8GB=8192 4GB=4096 2GB=2048 1GB=1024
|
||||
declare -A HOST1_WATCHDOG_CONTAINERS=(
|
||||
["Emby"]=20480 # 20GB — large library + active transcodes
|
||||
["LidaTube"]=6144 # 6GB — memory leak over time
|
||||
["Tdarr"]=6144 # 6GB — encoding is memory intensive
|
||||
["Code-Server"]=1024 # 1GB — should never need more
|
||||
)
|
||||
|
||||
# HTTP health check URLs — checked every cycle, strike system before restart.
|
||||
# Only add containers with a meaningful web interface to check.
|
||||
declare -A HOST1_WATCHDOG_CONTAINER_URLS=(
|
||||
["Emby"]="http://localhost:8096"
|
||||
)
|
||||
|
||||
# Required containers — must always be running on HOST1.
|
||||
# Strike system before restart — repeated failures go on skip list, auto-clears on recovery.
|
||||
# Listed in dependency order — dependencies before dependents.
|
||||
HOST1_WATCHDOG_REQUIRED_CONTAINERS=(
|
||||
"NginxProxyManager"
|
||||
"Lldap-Gmer4Lfe"
|
||||
"Mariadb-Authelia"
|
||||
"Mariadb-Authelia-Secondary"
|
||||
"Redis-Authelia"
|
||||
"Redis-Authelia-Secondary"
|
||||
"Authelia"
|
||||
"Authelia-Secondary"
|
||||
)
|
||||
|
||||
# Containers to skip in Tier 2 global scan — legitimately stopped or frequently restarting.
|
||||
# Watchdog leaves these alone entirely — no restart attempts, no crash loop tracking.
|
||||
HOST1_WATCHDOG_SCAN_IGNORE=(
|
||||
"DashGate"
|
||||
"PIA-WG-Config-Generator"
|
||||
"Aperture"
|
||||
"Aperture-Kids"
|
||||
"pgvector-18-Apeture-Kids"
|
||||
"Pgvector18-Aperture"
|
||||
"emby-test" # broken test container (exit 127 — bad image)
|
||||
)
|
||||
|
||||
# Dependency ordering — skip restarting a container if its dependency is also down.
|
||||
# Prevents watchdog from restarting Authelia before Mariadb is back up.
|
||||
# SPACE-SEPARATED STRINGS — converted to array at runtime.
|
||||
declare -A HOST1_WATCHDOG_DEPENDENCIES=(
|
||||
["Authelia"]="Mariadb-Authelia Redis-Authelia"
|
||||
["Authelia-Secondary"]="Mariadb-Authelia Redis-Authelia-Secondary"
|
||||
["NextCloud"]="Postgres-NextCloud"
|
||||
)
|
||||
|
||||
# Per-container appdata growth suppress ceilings in MB.
|
||||
# ONLY needed in specific cases — growth rate detection covers all containers automatically.
|
||||
# Use this when a container legitimately has large stable data and you want to guarantee
|
||||
# it never triggers a false-positive growth alert. Growth warnings are suppressed while the
|
||||
# container's dir stays below this ceiling; above it, warnings resume as normal.
|
||||
# 50GB=51200 25GB=25600 20GB=20480 15GB=15360 10GB=10240 5GB=5120
|
||||
declare -A HOST1_WATCHDOG_APPDATA_SIZES=(
|
||||
["Tdarr"]="25600" # 25GB — transcode cache grows legitimately during active jobs
|
||||
["7dtd"]="20480" # 20GB — game server world data, expected to be large
|
||||
)
|
||||
|
||||
# ━━━ Network Watchdog ━━━
|
||||
# Host-specific connectivity config for Watchdogs/System/network_watchdog.sh.
|
||||
HOST1_NETWORK_WATCHDOG_DDNS_DOMAIN="gmer4lfe.com"
|
||||
HOST1_NETWORK_WATCHDOG_DDNS_CONTAINER="Gmer4Lfe.com"
|
||||
HOST1_NETWORK_WATCHDOG_NPM_URL="https://gmer4lfe.com"
|
||||
|
||||
# ━━━ Docker Network Connect ━━━
|
||||
# Containers connected to custom networks at array start by docker_network_connect.sh.
|
||||
# Networks created if they don't exist — idempotent, safe to re-run.
|
||||
HOST1_NETWORK_CONNECT_CONTAINERS=(
|
||||
"memcached"
|
||||
"Npm-CrowdSec"
|
||||
)
|
||||
|
||||
HOST1_NETWORK_CONNECT_NETWORKS=(
|
||||
"high-availability"
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── FALLBACK ──────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ DDNS ━━━
|
||||
# DDNS containers HOST1 manages — started/stopped by fallback.sh per DDNS absolute rules:
|
||||
# Internet loss → stop immediately
|
||||
# Failover → HOST2 starts HOST1's DDNS as Tier 1 (before any other containers)
|
||||
# Handback → stop HOST1's DDNS on HOST2 → rsync → start containers → start local DDNS last
|
||||
HOST1_DDNS_CONTAINERS=(
|
||||
"Gmer4Lfe.com"
|
||||
)
|
||||
|
||||
# ━━━ Internet Loss ━━━
|
||||
# Containers stopped immediately on HOST1 when internet connection is lost.
|
||||
# Prevents external-facing services from operating without connectivity.
|
||||
FALLBACK_HOST1_STOP_ON_NO_NET=(
|
||||
"Gmer4Lfe.com"
|
||||
)
|
||||
|
||||
# ━━━ Fallback Tiers — HOST1 Runs for HOST2 ━━━
|
||||
# Containers HOST1 starts when HOST2 goes down.
|
||||
# Tier 1 is always immediate — vital services cannot wait.
|
||||
# Higher tiers activate after HOST2_TIER*_DELAY minutes (set in host2.conf).
|
||||
FALLBACK_HOST1_COVERS_HOST2_TIER1=(
|
||||
"Gmer4Lfe.us"
|
||||
"VaultWarden-Jayred365"
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_COVERS_HOST2_TIER2=(
|
||||
# "container-placeholder"
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_COVERS_HOST2_TIER3=(
|
||||
# "container-placeholder"
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_COVERS_HOST2_TIER4=(
|
||||
# "container-placeholder"
|
||||
)
|
||||
|
||||
# ━━━ Tier Delays — HOST1's Containers on HOST2 ━━━
|
||||
# How long HOST1 must be down before each tier activates on HOST2 — in minutes.
|
||||
# Tier 1 is always immediate — no delay var needed.
|
||||
HOST1_TIER2_DELAY=240 # 4 hours — NextCloud, Immich
|
||||
HOST1_TIER3_DELAY=720 # 12 hours — secondary services
|
||||
HOST1_TIER4_DELAY=1440 # 24 hours — arrs + downloaders
|
||||
|
||||
# ━━━ Rsync Writeback — HOST1 Appdata Back on Handback ━━━
|
||||
# Syncs HOST1 appdata BACK to HOST1 when it comes back online after a failover.
|
||||
# Containers stopped before writeback — clean source, no competing writes.
|
||||
#
|
||||
# HOST1_TIER1_WRITEBACK_DELAY: short outages skip Tier 1 writeback — primary state
|
||||
# is more reliable than dirty sync data for brief outages.
|
||||
HOST1_TIER1_WRITEBACK_DELAY=60 # skip Emby writeback if outage under 1hr
|
||||
|
||||
# Tier 4 automatically syncs HOST1_DAILY_SYNC_SHARES — only list paths NOT in that array.
|
||||
FALLBACK_HOST1_WRITEBACK_TIER1=(
|
||||
"/mnt/user/Media_Server/Emby" # watch states built up during outage
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_WRITEBACK_TIER2=(
|
||||
"/mnt/user/appdata-Fallback/Important-Data" # NextCloud + Postgres — files added during outage
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_WRITEBACK_TIER3=(
|
||||
# "location-placeholder"
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_WRITEBACK_TIER4=(
|
||||
"/mnt/user/appdata-Fallback/Arrs_Stack" # arr databases — downloads queued during outage
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── MEDIA ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Media Permissions ━━━
|
||||
# Shares that media_shares_permissions.sh applies PERMISSIONS_MODE and PERMISSIONS_OWNER to.
|
||||
# Runs first in DAILY_MAINTENANCE_SCRIPTS — arr cleanup depends on correct ownership.
|
||||
HOST1_MEDIA_PERMISSION_SHARES=(
|
||||
/mnt/user/Anime_Movies
|
||||
/mnt/user/Anime_Movies-Old
|
||||
/mnt/user/Anime_Shows
|
||||
/mnt/user/Anime_Shows-Old
|
||||
/mnt/user/appcache
|
||||
/mnt/user/Books
|
||||
/mnt/user/Downloads
|
||||
/mnt/user/Games
|
||||
/mnt/user/Intros
|
||||
/mnt/user/Kids_Movies
|
||||
/mnt/user/Kids_Tv_Shows
|
||||
/mnt/user/Movie_Recordings
|
||||
/mnt/user/Movies
|
||||
/mnt/user/Music
|
||||
/mnt/user/Music_Videos
|
||||
/mnt/user/Photo
|
||||
/mnt/user/Sports
|
||||
/mnt/user/stand-up_comedy
|
||||
/mnt/user/Tv_Recordings
|
||||
/mnt/user/Tv_Shows
|
||||
/mnt/user/YouTube
|
||||
)
|
||||
|
||||
# ━━━ Media Cleaner ━━━
|
||||
# Folder lists for media_cleaner.sh — two profiles: anime and media.
|
||||
# File patterns shared across all servers — defined in master.conf.
|
||||
# Called via DAILY_MAINTENANCE_SCRIPTS. Run manually: Media/media_cleaner.sh anime|media
|
||||
HOST1_ANIME_CLEAN_FOLDERS=(
|
||||
/mnt/user/Anime_Movies
|
||||
/mnt/user/Anime_Movies-Old
|
||||
/mnt/user/Anime_Shows
|
||||
/mnt/user/Anime_Shows-Old
|
||||
)
|
||||
|
||||
HOST1_MEDIA_CLEAN_FOLDERS=(
|
||||
/mnt/user/Kids_Movies
|
||||
/mnt/user/Kids_Tv_Shows
|
||||
/mnt/user/Movies
|
||||
/mnt/user/Music
|
||||
/mnt/user/Sports
|
||||
/mnt/user/stand-up_comedy
|
||||
/mnt/user/Tv_Shows
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── MONITORS ──────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Certificate Monitor ━━━
|
||||
# Domains checked via direct openssl connection — not relying on NPM's certificate state.
|
||||
# Checks the actual certificate served, not what NPM thinks it has.
|
||||
# Thresholds (CERT_WARN_DAYS, CERT_CRIT_DAYS) defined in master.conf.
|
||||
HOST1_CERT_MONITOR_DOMAINS=(
|
||||
"Gmer4Lfe.com"
|
||||
"Gmer4Lfe.us"
|
||||
)
|
||||
|
||||
# ━━━ SMART Health ━━━
|
||||
# Drives skipped in SMART attribute monitoring — hardware is server-specific.
|
||||
# Thresholds read from dynamix.cfg at runtime — fallbacks in master.conf.
|
||||
HOST1_SMART_IGNORE_DRIVES=(
|
||||
"sda" # boot USB — SMART not meaningful on flash drives
|
||||
)
|
||||
|
||||
# ━━━ ZFS Report ━━━
|
||||
# Pools excluded from the weekly ZFS health report — reduces noise from single-disk array pools.
|
||||
# These are individual array disks formatted as ZFS — converting to XFS over time via unBalance.
|
||||
# Pool health thresholds defined in master.conf.
|
||||
HOST1_ZFS_REPORT_IGNORE_POOLS=(
|
||||
"disk5"
|
||||
"disk6"
|
||||
"disk8"
|
||||
"disk9"
|
||||
"disk10"
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── TRANSCODES ────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# Ramdisk size ceiling — tmpfs only uses RAM actually needed, not the full size upfront.
|
||||
# Real-world: 9 streams peaked at ~5.5GB — 10G gives generous headroom on 128GB RAM.
|
||||
HOST1_RAMDISK_SIZE="10G"
|
||||
|
||||
# Usage thresholds — coupled to HOST1_RAMDISK_SIZE, adjust all three together if size changes.
|
||||
# Hysteresis gap (8.5 - 7 = 1.5GB) prevents flip-flop between ramdisk and SSD.
|
||||
HOST1_RAMDISK_WARN_GB=8.5 # flip to SSD when ramdisk usage reaches this
|
||||
HOST1_RAMDISK_LOW_GB=7 # flip back to ramdisk when usage drops to this
|
||||
|
||||
# SSD fallback path — where transcodes land when ramdisk exceeds HOST1_RAMDISK_WARN_GB.
|
||||
# Must be on cache pool — array disks too slow for active transcode writes.
|
||||
HOST1_TRANSCODE_SSD="/mnt/cache/Temp_Storage/Emby/Transcodes/"
|
||||
|
||||
# Media servers sharing the ramdisk transcode space on HOST1.
|
||||
# Format: "ContainerName|URL|APIKey|Type" — Type: emby | jellyfin | plex
|
||||
# Entries with placeholder API keys are skipped automatically.
|
||||
# ⚠️ Tdarr does NOT belong here — keep Tdarr on SSD, not ramdisk.
|
||||
HOST1_TRANSCODE_SERVERS=(
|
||||
"${HOST1_EMBY_CONTAINER}|${HOST1_EMBY_URL}|${HOST1_EMBY_API_KEY}|emby"
|
||||
"${HOST1_JELLYFIN_CONTAINER}|${HOST1_JELLYFIN_URL}|${HOST1_JELLYFIN_API_KEY}|jellyfin"
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── ARR STACK ─────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Used by arr cleanup scripts and arrs_failed_stalled_recovery.sh.
|
||||
# detect_hosts() selects HOST1 vars when running on HOST1.
|
||||
#
|
||||
# PATH MAPS — container path → host path translation.
|
||||
# Arr stores file paths using container-internal paths — scripts need host paths to scan.
|
||||
# Add one entry per root folder in arr Settings → Media Management → Root Folders.
|
||||
|
||||
# ━━━ Downloaders ━━━
|
||||
# Used by downloaders_reset.sh — runs every 30min via CRITICAL_MAINTENANCE_SCRIPTS.
|
||||
# Clears stuck states, purges old history, prepares each client for a clean cycle.
|
||||
|
||||
# slskd — clears stuck searches, dead transfers, purges expired failed imports.
|
||||
# SLSKD_FAILED_IMPORTS_DIR: where Soularr moves albums Lidarr rejected.
|
||||
HOST1_SLSKD_URL="http://localhost:8980"
|
||||
HOST1_SLSKD_API_KEY="4bF9kL2mNpQrT7vWxYz1A3dEgHjKoRsU"
|
||||
HOST1_SLSKD_FAILED_IMPORTS_DIR="/mnt/user/Temp_Storage/Slskd/completed/failed_imports"
|
||||
|
||||
# SABnzbd
|
||||
HOST1_SABNZBD_URL="http://localhost:8180"
|
||||
HOST1_SABNZBD_API_KEY="8bfefe41d83b4d50883e32859b55ca9a"
|
||||
|
||||
# qBittorrent — deleteFiles=false removes torrent from qBit but leaves files on disk.
|
||||
# Radarr/Sonarr manage actual files independently.
|
||||
HOST1_QBIT_URL="http://localhost:8080"
|
||||
HOST1_QBIT_USERNAME="root"
|
||||
HOST1_QBIT_PASSWORD="Stay0utD!ck"
|
||||
|
||||
# ━━━ Lidarr — HOST1 only ━━━
|
||||
# HOST2 does not run Lidarr — HOST1_LIDARR_RECOVERY flag handles the exit cleanly.
|
||||
HOST1_LIDARR_URL="http://localhost:8686"
|
||||
HOST1_LIDARR_API_KEY="b2977e71ef074bc0a0529d9fcce3b2dc"
|
||||
HOST1_LIDARR_MUSIC_ROOT="/mnt/user/Music-New"
|
||||
HOST1_FANART_API_KEY="Yd7147a43b692df0b364b94dc47efb81"
|
||||
HOST1_LASTFM_API_KEY="be6dc169c33ae263e690c30d18b7491d"
|
||||
|
||||
declare -A HOST1_LIDARR_PATH_MAP=(
|
||||
["/ext-music"]="/mnt/user/Music-New"
|
||||
)
|
||||
|
||||
# ━━━ Sonarr ━━━
|
||||
HOST1_SONARR_URL="http://localhost:8989"
|
||||
HOST1_SONARR_API_KEY="130decd3db5b4c25afad64864cd03f9f"
|
||||
HOST1_SONARR_TV_ROOT="/mnt/user/Tv_Shows"
|
||||
|
||||
# Note: stand-up_comedy in both Sonarr + Radarr — TV specials and movie specials, one folder
|
||||
declare -A HOST1_SONARR_PATH_MAP=(
|
||||
["/tv"]="/mnt/user/Tv_Shows"
|
||||
["/ext-standup-comedy"]="/mnt/user/stand-up_comedy"
|
||||
["/kids tv"]="/mnt/user/Kids_Tv_Shows"
|
||||
["/ext-anime-shows"]="/mnt/user/Anime_Shows-Old"
|
||||
)
|
||||
|
||||
# ━━━ Radarr ━━━
|
||||
HOST1_RADARR_URL="http://localhost:7878"
|
||||
HOST1_RADARR_API_KEY="d43a3ec6cf1549edb4af0cc63f98b2a9"
|
||||
HOST1_TMDB_API_KEY="3dac5e2e49b5540472d2eafec4f01260"
|
||||
HOST1_RADARR_MOVIES_ROOT="/mnt/user/Movies"
|
||||
|
||||
# Note: stand-up_comedy in both Radarr + Sonarr — movie specials and TV specials, one folder
|
||||
declare -A HOST1_RADARR_PATH_MAP=(
|
||||
["/movies"]="/mnt/user/Movies"
|
||||
["/kids movies"]="/mnt/user/Kids_Movies"
|
||||
["/ext-stand-up-comedy"]="/mnt/user/stand-up_comedy"
|
||||
["/anime-movies"]="/mnt/user/Anime_Movies-Old"
|
||||
)
|
||||
|
||||
# ━━━ Arr Recovery Toggles ━━━
|
||||
# false = skip that arr on this host — exits cleanly without error
|
||||
HOST1_SONARR_RECOVERY=true
|
||||
HOST1_RADARR_RECOVERY=true
|
||||
HOST1_LIDARR_RECOVERY=true # HOST1 only — exits cleanly on HOST2
|
||||
|
||||
# ==============================================================================================
|
||||
# ── SYSTEM WATCHDOG ───────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Per-host check toggles and NIC config for system_watchdog.sh.
|
||||
# Aliased by detect_hosts() — script uses unprefixed SYS_WATCHDOG_* names.
|
||||
# HOST1: TR1950X 128GB — full media server, active transcoding, ZFS cache pools.
|
||||
#
|
||||
# Three-tier response — all critical checks enabled by default on HOST1:
|
||||
# Tier 1 (bypass strikes, reboot now): docker daemon, rootfs full, kernel oops, FD, /boot
|
||||
# Tier 2 (bypass strikes with OOM): RAM critical + OOM kills in cycle
|
||||
# Tier 3 (standard strike system): everything else
|
||||
#
|
||||
# RAM tiers, OOM limits, and reboot loop settings in master.conf System Watchdog section.
|
||||
|
||||
# ━━━ Primary NIC ━━━
|
||||
# Network interface for NIC state check — verify with: ip link show | grep "^[0-9]"
|
||||
# Common values: eth0, bond0, br0, eno1
|
||||
HOST1_SYS_WATCHDOG_NIC="eth0"
|
||||
|
||||
# ━━━ Tier 1 — Critical Checks ━━━
|
||||
# These bypass the strike system — a single hit triggers immediate reboot.
|
||||
# Disabling any of these is not recommended — they protect against acute system failure.
|
||||
|
||||
# Docker daemon unresponsive → try restart, reboot if restart fails.
|
||||
# Without a working daemon docker_watchdog.sh is blind and containers cannot be managed.
|
||||
HOST1_SYS_WATCHDOG_CHECK_DOCKER_DAEMON=true
|
||||
|
||||
# rootfs at critical threshold (ROOTFS_CRITICAL_PCT=99) → reboot immediately.
|
||||
# At 99% rootfs writes fail silently — logs stop, Docker errors out, SSH may stop working.
|
||||
# Standard 95% threshold still uses strike system — only 99%+ is critical tier.
|
||||
HOST1_SYS_WATCHDOG_CHECK_ROOTFS=true
|
||||
|
||||
# Kernel BUG/Oops in dmesg delta since last cycle → reboot immediately.
|
||||
# A kernel oops means the kernel ran with a corrupted state — stability is not guaranteed.
|
||||
HOST1_SYS_WATCHDOG_CHECK_KERNEL_OOPS=true
|
||||
|
||||
# File descriptor exhaustion at FD_CRITICAL_PCT (95%) → reboot immediately.
|
||||
# At 95% FD: new connections fail, Docker can't spawn processes, SSH drops.
|
||||
HOST1_SYS_WATCHDOG_CHECK_FD=true
|
||||
|
||||
# /boot read-only detected → reboot immediately.
|
||||
# Unexpected read-only /boot means state files and config writes are silently failing.
|
||||
# Fallback state, watchdog reboot log, and lock files all go stale silently.
|
||||
HOST1_SYS_WATCHDOG_CHECK_BOOT=true
|
||||
|
||||
# ━━━ Tier 2 — Urgent OOM Check ━━━
|
||||
# Bypass strikes when RAM is critically low AND OOM kill rate confirms active crisis.
|
||||
# Both must be enabled for Tier 2 bypass to function — disable either to always use strikes.
|
||||
|
||||
# Track kernel OOM kills each cycle via /proc/vmstat oom_kill delta.
|
||||
# Also provides diagnostic context in reboot messages (which processes were killed).
|
||||
HOST1_SYS_WATCHDOG_CHECK_OOM=true
|
||||
|
||||
# Free RAM check — required for both Tier 2 bypass and RAM tier logic.
|
||||
# Tiers: MEM_WARN_GB(10) → notify | MEM_SHUTDOWN_GB(6) → stop containers | MEM_GB(4) → strikes
|
||||
HOST1_SYS_WATCHDOG_CHECK_RAM=true
|
||||
|
||||
# ━━━ Tier 3 — Standard Checks (strike system) ━━━
|
||||
# Each check must fail SYS_WATCHDOG_STRIKE_LIMIT consecutive cycles before action is taken.
|
||||
# Single spikes are ignored — sustained problems trigger reboot.
|
||||
|
||||
# /var/log filesystem usage above SYS_WATCHDOG_LOG_PCT.
|
||||
# Log spam (Docker log storms, syslog loops) fills rootfs — indicates something broken.
|
||||
HOST1_SYS_WATCHDOG_CHECK_LOG=true
|
||||
|
||||
# ZFS ARC memory pinned above SYS_WATCHDOG_ARC_PINNED_PCT after cache drop.
|
||||
# Enabled on HOST1 — ZFS cache pools actively used. Disable on hosts without ZFS.
|
||||
HOST1_SYS_WATCHDOG_CHECK_ARC=true
|
||||
|
||||
# CPU temperature above SYS_WATCHDOG_CPU_TEMP_MAX (95°C).
|
||||
# Sustained high temp causes kernel throttling or panic. Requires lm-sensors.
|
||||
HOST1_SYS_WATCHDOG_CHECK_CPU_TEMP=true
|
||||
|
||||
# Load average above SYS_WATCHDOG_LOAD_MULTIPLIER × core count.
|
||||
# DISABLED on HOST1 — Tdarr and Emby cause legitimate sustained load spikes during encoding.
|
||||
# Enable on idle servers or adjust SYS_WATCHDOG_LOAD_MULTIPLIER if load is always high.
|
||||
HOST1_SYS_WATCHDOG_CHECK_LOAD=false
|
||||
|
||||
# Zombie process count above SYS_WATCHDOG_ZOMBIE_LIMIT (50).
|
||||
# Large zombie counts indicate serious process management failure — something is stuck.
|
||||
HOST1_SYS_WATCHDOG_CHECK_ZOMBIES=true
|
||||
|
||||
# Check docker_watchdog.sh persistent skip list — required containers on skip list.
|
||||
# Cross-watchdog coordination: if docker_watchdog gave up, system_watchdog escalates.
|
||||
# ENABLED — HOST1 fully built and operational, skip list is meaningful.
|
||||
HOST1_SYS_WATCHDOG_CHECK_CONTAINERS=true
|
||||
|
||||
# /tmp filesystem usage above SYS_WATCHDOG_TMP_PCT with auto-clear attempt.
|
||||
# Script tries to clear aged /tmp files first — only strikes if clear fails.
|
||||
# Lock files, rsync temp files, and Docker ops use /tmp — 100% means lock failures.
|
||||
HOST1_SYS_WATCHDOG_CHECK_TMP=true
|
||||
|
||||
# Array disk error count delta in /proc/mdstat — accumulating errors = disk failing now.
|
||||
# Triggers on SYS_WATCHDOG_MDSTAT_ERROR_LIMIT new errors in one cycle.
|
||||
HOST1_SYS_WATCHDOG_CHECK_MDSTAT=true
|
||||
|
||||
# Primary NIC operstate — detects NIC going down (physical or driver failure).
|
||||
# Uses HOST1_SYS_WATCHDOG_NIC above. Strike system — brief flaps don't trigger reboot.
|
||||
HOST1_SYS_WATCHDOG_CHECK_NETWORK=true
|
||||
|
||||
# sshd running check — attempts restart before escalating.
|
||||
# sshd down = no remote access. Script tries rc.sshd start, notifies, strikes on failure.
|
||||
HOST1_SYS_WATCHDOG_CHECK_SSHD=true
|
||||
|
||||
# Runaway process detection — single process above SYS_WATCHDOG_RUNAWAY_CPU_PCT sustained.
|
||||
# DISABLED — Tdarr encoding and Emby transcoding legitimately peg CPU for extended periods.
|
||||
# Enable only if HOST1 has no CPU-intensive workloads.
|
||||
HOST1_SYS_WATCHDOG_CHECK_RUNAWAY=false
|
||||
|
||||
# ==============================================================================================
|
||||
# ── RESOURCE MANAGER ──────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Containers to manage under pressure — see master.conf RW_CRITICAL_CONTAINERS for exclusions.
|
||||
|
||||
# docker pause at medium pressure (RAM < RW_RAM_MEDIUM_GB or load > medium threshold)
|
||||
# Suspended in-place — instant to pause/unpause, no state lost, no restart delay.
|
||||
HOST1_RW_PAUSE_CONTAINERS=(
|
||||
"Huntarr" # arr search automation — safe to suspend
|
||||
"Cleanuparr" # download cleanup — safe to suspend
|
||||
"Healarr" # arr health checks — safe to suspend
|
||||
"Soularr" # Slskd automation — background only
|
||||
"ChannelTube" # YouTube archiver — background only
|
||||
"Pinchflat" # YouTube archiver — background only
|
||||
)
|
||||
|
||||
# docker stop at hard pressure (RAM < RW_RAM_HARD_GB)
|
||||
# Full stop — these are optional/heavy services that free significant RAM when stopped.
|
||||
# resource_watchdog.sh restarts them when pressure fully clears (RAM >= RW_RAM_RECOVER_GB).
|
||||
HOST1_RW_STOP_CONTAINERS=(
|
||||
"LocalAI" # GPU/CPU heavy — largest RAM consumer when idle
|
||||
"7DaysToDie" # game server — optional
|
||||
"V-Rising" # game server — optional
|
||||
"Code-Server" # IDE — not needed during pressure events
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── AUTH STACK ────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Credentials for the Varaverk Auth Stack page (NPM, lldap, Authelia).
|
||||
|
||||
# ━━━ NginxProxyManager ━━━
|
||||
HOST1_NPM_URL="http://localhost:7818"
|
||||
HOST1_NPM_USER="failedproxy@gmail.com"
|
||||
HOST1_NPM_PASS="183134\$eanHess"
|
||||
|
||||
# ━━━ lldap ━━━
|
||||
HOST1_LLDAP_URL="http://localhost:17170"
|
||||
HOST1_LLDAP_USER="admin"
|
||||
HOST1_LLDAP_PASS="183134\$eanHess"
|
||||
|
||||
# ━━━ Authelia ━━━
|
||||
HOST1_AUTHELIA_CONFIG="/mnt/user/appdata-Fallback/Critical-Data/Authelia/configuration.yml"
|
||||
HOST1_AUTHELIA_CONTAINER="Authelia"
|
||||
|
||||
# ==============================================================================================
|
||||
# ──────────────────────── End Of HOST1 Variables ──────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
HOST1_UNRAID_API_KEY="1825c3a2e03ea5089974f4da2e171aa2d5907a1dea23cc479c33e492c8ff4dbb"
|
||||
@@ -0,0 +1,789 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ========================== HOST1 CONFIGURATION — unRAID-Gmer4Lfe ============================
|
||||
# ==============================================================================================
|
||||
# HOST1-specific variables — credentials, container names, share paths, failover lists.
|
||||
# Sourced after master.conf — values here extend shared profile arrays and add HOST1-specific
|
||||
# identity, credentials, and container configuration.
|
||||
#
|
||||
# Sparse checkout (git) ensures HOST2 never receives this file.
|
||||
# HOST2 never sees HOST1 credentials — clean separation at the file level.
|
||||
#
|
||||
# DO NOT put shared config here — thresholds, toggles, profiles belong in master.conf.
|
||||
# DO NOT put HOST2 variables here — they belong in host2.conf.
|
||||
#
|
||||
# ── INDEX ─────────────────────────────────────────────────────────────────────────────────────
|
||||
#
|
||||
# ── IDENTITY & CONNECTIVITY ────────────────────────────────────────────────────────────────
|
||||
# IDENTITY hostname, SSH key
|
||||
# EMBY container name, URL, API key
|
||||
# NOTIFICATIONS Discord webhook
|
||||
# PARTNERSHIP auth containers, backup paths
|
||||
#
|
||||
# ── RSYNC ──────────────────────────────────────────────────────────────────────────────────
|
||||
# DAILY SYNC SHARES media shares HOST1 owns and pushes to HOST2
|
||||
# WEEKLY SYNC SHARES appdata shares synced weekly (Sunday 2:30am)
|
||||
# CRITICAL SYNC SHARES appdata shares synced every 30 minutes
|
||||
# BACKUP VERIFY shares for checksum verification against remote
|
||||
# HOST1 RSYNC PROFILE host1-appdata profile for HOST1-specific appdata syncs
|
||||
#
|
||||
# ── DOCKER ─────────────────────────────────────────────────────────────────────────────────
|
||||
# DOCKER DAILY RESTART containers restarted daily
|
||||
# DOCKER WEEKLY RESTART containers restarted weekly
|
||||
# DOCKER WATCHDOG memory limits, health URLs, required containers, ignore list
|
||||
# DOCKER NETWORK CONNECT networks and containers for docker_network_connect.sh
|
||||
#
|
||||
# ── FALLBACK ───────────────────────────────────────────────────────────────────────────────
|
||||
# DDNS DDNS containers managed by HOST1
|
||||
# INTERNET LOSS containers stopped when internet is lost
|
||||
# FALLBACK TIERS what HOST1 runs for HOST2 per tier
|
||||
# TIER DELAYS how long HOST1 must be down before each tier activates on HOST2
|
||||
# RSYNC WRITEBACK HOST1 appdata synced back on handback
|
||||
#
|
||||
# ── MEDIA ──────────────────────────────────────────────────────────────────────────────────
|
||||
# MEDIA PERMISSIONS share list for media_shares_permissions.sh
|
||||
# MEDIA CLEANER folder lists for media_cleaner.sh
|
||||
#
|
||||
# ── MONITORS ───────────────────────────────────────────────────────────────────────────────
|
||||
# CERTIFICATE MONITOR domains checked for SSL expiry
|
||||
# SMART HEALTH drives to skip in SMART monitoring
|
||||
# ZFS REPORT pools to exclude from ZFS health report
|
||||
#
|
||||
# ── TRANSCODES ─────────────────────────────────────────────────────────────────────────────
|
||||
# TRANSCODES ramdisk size, thresholds, SSD path, server array
|
||||
#
|
||||
# ── ARR STACK ──────────────────────────────────────────────────────────────────────────────
|
||||
# DOWNLOADERS slskd, SABnzbd, qBittorrent credentials and URLs
|
||||
# LIDARR URL, API key, path map
|
||||
# SONARR URL, API key, path map
|
||||
# RADARR URL, API key, path map
|
||||
# ARR RECOVERY per-arr recovery toggles
|
||||
#
|
||||
# ── SYSTEM WATCHDOG ────────────────────────────────────────────────────────────────────────
|
||||
# SYSTEM WATCHDOG per-host check toggles and NIC configuration
|
||||
#
|
||||
# ── RESOURCE MANAGER ───────────────────────────────────────────────────────────────────────
|
||||
# RESOURCE MANAGER containers paused/stopped under memory pressure
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
# ==============================================================================================
|
||||
# ── IDENTITY & CONNECTIVITY ───────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Identity ━━━
|
||||
# HOST1 hostname lives in master.conf (not a credential — safe for all servers).
|
||||
# SSH key used for all server-to-server operations — rsync, failover container commands.
|
||||
# Must be in /root/.ssh/ and authorised in HOST2's /root/.ssh/authorized_keys.
|
||||
HOST1_SSH_KEY="/root/.ssh/gmer4lfe_rsync_automation"
|
||||
HOST1_OWNER="gmer4lfe"
|
||||
HOST1_OWNER_EMAIL="gmer4lfe@gmail.com"
|
||||
|
||||
# ━━━ Emby ━━━
|
||||
# Referenced by transcode_manager.sh, emby_session_report.sh, emby_database_repair.sh,
|
||||
# weekly_sync_maintenance.sh, and HOST1_TRANSCODE_SERVERS below.
|
||||
# API key: Emby Dashboard → API Keys → + New Key
|
||||
HOST1_EMBY_CONTAINER="Emby"
|
||||
HOST1_EMBY_URL="http://localhost:8096"
|
||||
HOST1_EMBY_API_KEY="0c27448d93a7431f9ac63569f7655829"
|
||||
|
||||
# ━━━ Jellyfin ━━━
|
||||
# API key: Jellyfin Dashboard → Administration → API Keys → + New Key
|
||||
HOST1_JELLYFIN_CONTAINER="Jellyfin"
|
||||
HOST1_JELLYFIN_URL="http://localhost:8095"
|
||||
HOST1_JELLYFIN_API_KEY="4e820e7df74c4933acec212b1996314e"
|
||||
|
||||
# ━━━ Gitea ━━━
|
||||
# Personal access token for gitea_ssh_setup.sh — registers this server's SSH public key
|
||||
# with Gitea so git operations use key auth instead of passwords.
|
||||
# Create in Gitea: Settings → Applications → Generate Token → scope: write:user
|
||||
HOST1_GITEA_API_TOKEN=""
|
||||
|
||||
# ━━━ Notifications ━━━
|
||||
# Discord webhook — leave blank to disable.
|
||||
# Per-host so HOST1 and HOST2 can post to different channels or only one server notifies.
|
||||
HOST1_DISCORD_WEBHOOK=""
|
||||
|
||||
# ━━━ Partnership ━━━
|
||||
# HOST1 is always the owner (source of truth) unless --transfer has been run.
|
||||
# See README-Partnership.md and master.conf PARTNERSHIP section for full lifecycle docs.
|
||||
|
||||
# Auth containers reconfigured on onboard/offboard.
|
||||
# Format: "ContainerName|WebUIPort"
|
||||
# On onboard → WebUI pointed at owner's Tailscale IP (mirror clicks NPM, gets owner's auth)
|
||||
# On offboard → WebUI pointed back at localhost
|
||||
HOST1_PARTNERSHIP_AUTH_WEBUIS=(
|
||||
"NginxProxyManager|81"
|
||||
"Lldap-Gmer4Lfe|17170"
|
||||
"Authelia|9091"
|
||||
"Authelia-Secondary|9092"
|
||||
)
|
||||
|
||||
# XML templates (from this server's templates-user/) pushed to mirror during onboard.
|
||||
# These become the mirror's active auth stack, backed by the rsync-synced appdata.
|
||||
# Update filename if Lldap is renamed to drop the host suffix.
|
||||
HOST1_PARTNERSHIP_AUTH_STACK=(
|
||||
# Dependencies first — Mariadb/Redis must be healthy before Authelia starts
|
||||
"my-Mariadb-Authelia.xml"
|
||||
"my-Mariadb-Authelia-Secondary.xml"
|
||||
"my-Redis-Authelia.xml"
|
||||
"my-Redis-Authelia-Secondary.xml"
|
||||
# Auth apps — deployed after their deps are confirmed healthy
|
||||
"my-Authelia.xml"
|
||||
"my-Authelia-Secondary.xml"
|
||||
"my-NginxProxyManager.xml"
|
||||
"my-Lldap-Gmer4Lfe.xml"
|
||||
# Source of truth — must be available on HOST2 independently of the auth stack
|
||||
"my-Gitea.xml"
|
||||
)
|
||||
|
||||
# XML templates pushed to mirror for the arr stack during onboard.
|
||||
# Deps (e.g. databases) first if any — same ordering rule as auth stack.
|
||||
HOST1_PARTNERSHIP_ARR_STACK=(
|
||||
# "my-Sonarr.xml"
|
||||
# "my-Radarr.xml"
|
||||
# "my-Lidarr.xml"
|
||||
# "my-Prowlarr.xml"
|
||||
# "my-Bazarr.xml"
|
||||
)
|
||||
|
||||
# Paths HOST2 should collect during the grace window after offboard.
|
||||
# Notified on offboard — no auto-deletion, HOST2 must collect manually within PARTNERSHIP_GRACE_HOURS.
|
||||
HOST1_PARTNERSHIP_MIRROR_BACKUPS=(
|
||||
# "/mnt/user/appdata-Fallback/Jayred365-Emby"
|
||||
)
|
||||
|
||||
# Containers parked on this server when partnership is active.
|
||||
# Stopped on onboard (owner deploys its stack instead), restarted on offboard.
|
||||
HOST1_PARTNERSHIP_OWN_CONTAINERS=(
|
||||
# "Emby"
|
||||
# "NginxProxyManager"
|
||||
)
|
||||
|
||||
# Emby admin provisioning — toggle is owner-only, credentials are per-host.
|
||||
# Owner enables/disables the feature. Each host sets the account they want on the shared Emby.
|
||||
# On onboard: owner reads mirror's HOST*_PARTNERSHIP_EMBY_ADMIN_* and creates that account.
|
||||
# On offboard: account is deleted. Username collision → onboard exits with error.
|
||||
HOST1_PARTNERSHIP_PROVISION_EMBY_ADMIN=false # owner controls whether Emby is shared
|
||||
HOST1_PARTNERSHIP_EMBY_PORT=8096
|
||||
HOST1_PARTNERSHIP_EMBY_ADMIN_USER="" # this server's desired Emby username
|
||||
HOST1_PARTNERSHIP_EMBY_ADMIN_PASS="" # this server's desired Emby password
|
||||
|
||||
# ==============================================================================================
|
||||
# ── RSYNC ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Daily Sync Shares ━━━
|
||||
# Shares HOST1 pushes to all other nodes every night (1am via daily_sync_maintenance.sh).
|
||||
# Mesh model: every node pushes every media share — no ownership, no mirrors.
|
||||
# arr_sync ensures all arr libraries converge (union). rsync spreads files (additive, no --delete).
|
||||
# arr_cleanup removes true orphans based on local arr state.
|
||||
# Any node can download content to any share — it propagates to all nodes on the next cycle.
|
||||
# Nextcloud is intentionally one-directional (HOST1→HOST2 offsite backup — not arr-managed).
|
||||
# Uses DEFAULT_RSYNC_OPTS from master.conf — no profile needed.
|
||||
# For shares needing container stops or custom options — add a profile in master.conf.
|
||||
HOST1_DAILY_SYNC_SHARES=(
|
||||
/mnt/user/Books
|
||||
/mnt/user/Intros
|
||||
/mnt/user/Kids_Movies
|
||||
/mnt/user/Kids_Tv_Shows
|
||||
/mnt/user/Movies
|
||||
/mnt/user/Music
|
||||
/mnt/user/Music_Videos
|
||||
/mnt/user/Nextcloud
|
||||
/mnt/user/stand-up_comedy
|
||||
/mnt/user/Sports
|
||||
# /mnt/user/Tv_Shows
|
||||
/mnt/user/Anime_Shows-Old
|
||||
/mnt/user/Anime_Movies-Old
|
||||
/mnt/user/Anime_Movies
|
||||
/mnt/user/Anime_Shows
|
||||
)
|
||||
|
||||
# Personal encrypted shares — synced for offsite backup, independent of media shares.
|
||||
# ZFS encrypted at dataset level — remote receives encrypted blocks, cannot read content.
|
||||
# See README-Rsync_Setup.md for ZFS encryption setup before uncommenting.
|
||||
HOST1_PERSONAL_SHARES=(
|
||||
# /mnt/user/HOST1-Personal # uncomment after creating encrypted dataset
|
||||
)
|
||||
|
||||
# ━━━ Weekly Sync Shares ━━━
|
||||
# Appdata shares synced during the weekly maintenance window (Sunday 2:30am).
|
||||
# Containers stopped both sides before sync — full clean state guaranteed.
|
||||
# Profiles drive container stops, excludes, and options — configured in master.conf RSYNC section.
|
||||
# Order matters — Emby first (larger transfer), then Critical-Data (auth stack).
|
||||
HOST1_WEEKLY_SYNC_SHARES=(
|
||||
"/mnt/user/Media_Server/Emby" # emby profile — full clean mirror
|
||||
"/mnt/user/appdata-Fallback/Critical-Data" # critical-data profile — auth stack
|
||||
)
|
||||
|
||||
# ━━━ Intermediate Sync Shares ━━━
|
||||
# Shares synced every 4 hours by intermediate_sync_maintenance.sh.
|
||||
# Uses DEFAULT_RSYNC_OPTS (no --delete) — for sub-daily propagation of metadata or watch state.
|
||||
# Full media share sync stays in the daily window. Leave empty to skip mid-day rsync entirely.
|
||||
HOST1_INTERMEDIATE_SYNC_SHARES=(
|
||||
# Add shares here to enable mid-day rsync
|
||||
# Example: "/mnt/user/Emby_Metadata"
|
||||
)
|
||||
|
||||
# ━━━ Critical Sync Shares ━━━
|
||||
# Appdata shares synced every 30 minutes by critical_sync_maintenance.sh.
|
||||
# Format: "/path/to/share" or "/path/to/share|profile-name"
|
||||
# Order matters — Critical-Data first (auth stack), then Emby dirty sync.
|
||||
HOST1_CRITICAL_SYNC_SHARES=(
|
||||
"/mnt/user/appdata-Fallback/Critical-Data|critical-fallback" # auth dirty sync — stays running
|
||||
"/mnt/user/Media_Server/Emby|emby-fallback" # Emby dirty sync — stays running
|
||||
)
|
||||
|
||||
# ━━━ Backup Verify ━━━
|
||||
# Shares verified by backup_verify.sh — random file checksum comparison against remote.
|
||||
# Leave empty to use HOST1_DAILY_SYNC_SHARES automatically.
|
||||
# Sample size and minimum file size defined in master.conf.
|
||||
HOST1_BACKUP_VERIFY_SHARES=(
|
||||
# leave empty to use HOST1_DAILY_SYNC_SHARES automatically
|
||||
)
|
||||
|
||||
# ━━━ HOST1 Rsync Profile — host1-appdata ━━━
|
||||
# HOST1-specific appdata sync profile — extends the shared PROFILE_* arrays in master.conf.
|
||||
# Use for appdata unique to HOST1 (Organizrv2, VaultWarden, UptimeKuma etc.)
|
||||
# Shared appdata (auth stack, Emby) use dedicated profiles defined in master.conf.
|
||||
# Run manually: bash Rsync/rsync.sh /mnt/user/appdata-Fallback/HOST1-Appdata --profile=host1-appdata
|
||||
PROFILE_RSYNC_OPTS[host1-appdata]="-av --info=progress2 --bwlimit=${PROFILE_BW_LIMIT[host1-appdata]:-8000}"
|
||||
PROFILE_BW_LIMIT[host1-appdata]=8000
|
||||
PROFILE_RETRY_COUNT[host1-appdata]=3
|
||||
PROFILE_SLEEP[host1-appdata]=300
|
||||
PROFILE_CRITICAL_CONTAINER_NAMES[host1-appdata]="Organizrv2-Gmer4Lfe UptimeKuma-Gmer4Lfe VaultWarden-Gmer4Lfe"
|
||||
PROFILE_DELAYED_CONTAINERS[host1-appdata]=""
|
||||
PROFILE_CONTAINER_DELAY[host1-appdata]=5
|
||||
PROFILE_EXCLUDE_DIRS[host1-appdata]="logs *.tmp"
|
||||
|
||||
# ==============================================================================================
|
||||
# ── DOCKER ────────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Docker Daily Restart ━━━
|
||||
# Containers restarted every day via DAILY_MAINTENANCE_SCRIPTS.
|
||||
# Dispatcharr degrades over time without restart — daily is intentional, not just housekeeping.
|
||||
# Order matters — auth stack first, then media services.
|
||||
HOST1_DAILY_RESTART_CONTAINERS=(
|
||||
"NginxProxyManager"
|
||||
"Lldap-Gmer4Lfe"
|
||||
"Authelia"
|
||||
"Authelia-Secondary"
|
||||
"Dispatcharr-Iptv-Users"
|
||||
"Dispatcharr" # Live TV scheduler — degrades without daily restart
|
||||
"Dispatcharr-Basic"
|
||||
"ErsatzTV-Emby"
|
||||
"Slskd" # Soulseek connection drops after extended uptime; restart refreshes share index
|
||||
)
|
||||
|
||||
# ━━━ Docker Weekly Restart ━━━
|
||||
# Less critical services restarted weekly via WEEKLY_MAINTENANCE_SCRIPTS (Sunday 2:30am).
|
||||
# Containers already stopped for weekly sync — restart adds zero extra downtime.
|
||||
HOST1_WEEKLY_RESTART_CONTAINERS=(
|
||||
"NextCloud"
|
||||
"Organizrv2-Gmer4Lfe"
|
||||
"AdGuard-Home"
|
||||
"Immich-Gmer4Lfe"
|
||||
)
|
||||
|
||||
# ━━━ Docker Watchdog ━━━
|
||||
# Per-HOST1 container configuration for docker_watchdog.sh.
|
||||
# Shared thresholds and toggles live in master.conf.
|
||||
|
||||
# Memory hard limits in MB — immediate restart if exceeded.
|
||||
# Set at "container is clearly broken" not "container is busy".
|
||||
# 20GB=20480 18GB=18432 16GB=16384 12GB=12288 8GB=8192 4GB=4096 2GB=2048 1GB=1024
|
||||
declare -A HOST1_WATCHDOG_CONTAINERS=(
|
||||
["Emby"]=20480 # 20GB — large library + active transcodes
|
||||
["LidaTube"]=6144 # 6GB — memory leak over time
|
||||
["Tdarr"]=6144 # 6GB — encoding is memory intensive
|
||||
["Code-Server"]=1024 # 1GB — should never need more
|
||||
)
|
||||
|
||||
# HTTP health check URLs — checked every cycle, strike system before restart.
|
||||
# Only add containers with a meaningful web interface to check.
|
||||
declare -A HOST1_WATCHDOG_CONTAINER_URLS=(
|
||||
["Emby"]="http://localhost:8096"
|
||||
)
|
||||
|
||||
# Required containers — must always be running on HOST1.
|
||||
# Strike system before restart — repeated failures go on skip list, auto-clears on recovery.
|
||||
# Listed in dependency order — dependencies before dependents.
|
||||
HOST1_WATCHDOG_REQUIRED_CONTAINERS=(
|
||||
"NginxProxyManager"
|
||||
"Lldap-Gmer4Lfe"
|
||||
"Mariadb-Authelia"
|
||||
"Mariadb-Authelia-Secondary"
|
||||
"Redis-Authelia"
|
||||
"Redis-Authelia-Secondary"
|
||||
"Authelia"
|
||||
"Authelia-Secondary"
|
||||
)
|
||||
|
||||
# Containers to skip in Tier 2 global scan — legitimately stopped or frequently restarting.
|
||||
# Watchdog leaves these alone entirely — no restart attempts, no crash loop tracking.
|
||||
HOST1_WATCHDOG_SCAN_IGNORE=(
|
||||
"DashGate"
|
||||
"PIA-WG-Config-Generator"
|
||||
"Aperture"
|
||||
"Aperture-Kids"
|
||||
"pgvector-18-Apeture-Kids"
|
||||
"Pgvector18-Aperture"
|
||||
"emby-test" # broken test container (exit 127 — bad image)
|
||||
)
|
||||
|
||||
# Dependency ordering — skip restarting a container if its dependency is also down.
|
||||
# Prevents watchdog from restarting Authelia before Mariadb is back up.
|
||||
# SPACE-SEPARATED STRINGS — converted to array at runtime.
|
||||
declare -A HOST1_WATCHDOG_DEPENDENCIES=(
|
||||
["Authelia"]="Mariadb-Authelia Redis-Authelia"
|
||||
["Authelia-Secondary"]="Mariadb-Authelia Redis-Authelia-Secondary"
|
||||
["NextCloud"]="Postgres-NextCloud"
|
||||
)
|
||||
|
||||
# Per-container appdata growth suppress ceilings in MB.
|
||||
# ONLY needed in specific cases — growth rate detection covers all containers automatically.
|
||||
# Use this when a container legitimately has large stable data and you want to guarantee
|
||||
# it never triggers a false-positive growth alert. Growth warnings are suppressed while the
|
||||
# container's dir stays below this ceiling; above it, warnings resume as normal.
|
||||
# 50GB=51200 25GB=25600 20GB=20480 15GB=15360 10GB=10240 5GB=5120
|
||||
declare -A HOST1_WATCHDOG_APPDATA_SIZES=(
|
||||
["Tdarr"]="25600" # 25GB — transcode cache grows legitimately during active jobs
|
||||
["7dtd"]="20480" # 20GB — game server world data, expected to be large
|
||||
)
|
||||
|
||||
# ━━━ Network Watchdog ━━━
|
||||
# Host-specific connectivity config for Watchdogs/System/network_watchdog.sh.
|
||||
HOST1_NETWORK_WATCHDOG_DDNS_DOMAIN="gmer4lfe.com"
|
||||
HOST1_NETWORK_WATCHDOG_DDNS_CONTAINER="Gmer4Lfe.com"
|
||||
HOST1_NETWORK_WATCHDOG_NPM_URL="https://gmer4lfe.com"
|
||||
|
||||
# ━━━ Docker Network Connect ━━━
|
||||
# Containers connected to custom networks at array start by docker_network_connect.sh.
|
||||
# Networks created if they don't exist — idempotent, safe to re-run.
|
||||
HOST1_NETWORK_CONNECT_CONTAINERS=(
|
||||
"memcached"
|
||||
"Npm-CrowdSec"
|
||||
)
|
||||
|
||||
HOST1_NETWORK_CONNECT_NETWORKS=(
|
||||
"high-availability"
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── FALLBACK ──────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ DDNS ━━━
|
||||
# DDNS containers HOST1 manages — started/stopped by fallback.sh per DDNS absolute rules:
|
||||
# Internet loss → stop immediately
|
||||
# Failover → HOST2 starts HOST1's DDNS as Tier 1 (before any other containers)
|
||||
# Handback → stop HOST1's DDNS on HOST2 → rsync → start containers → start local DDNS last
|
||||
HOST1_DDNS_CONTAINERS=(
|
||||
"Gmer4Lfe.com"
|
||||
)
|
||||
|
||||
# ━━━ Internet Loss ━━━
|
||||
# Containers stopped immediately on HOST1 when internet connection is lost.
|
||||
# Prevents external-facing services from operating without connectivity.
|
||||
FALLBACK_HOST1_STOP_ON_NO_NET=(
|
||||
"Gmer4Lfe.com"
|
||||
)
|
||||
|
||||
# ━━━ Fallback Tiers — HOST1 Runs for HOST2 ━━━
|
||||
# Containers HOST1 starts when HOST2 goes down.
|
||||
# Tier 1 is always immediate — vital services cannot wait.
|
||||
# Higher tiers activate after HOST2_TIER*_DELAY minutes (set in host2.conf).
|
||||
FALLBACK_HOST1_COVERS_HOST2_TIER1=(
|
||||
"Gmer4Lfe.us"
|
||||
"VaultWarden-Jayred365"
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_COVERS_HOST2_TIER2=(
|
||||
# "container-placeholder"
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_COVERS_HOST2_TIER3=(
|
||||
# "container-placeholder"
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_COVERS_HOST2_TIER4=(
|
||||
# "container-placeholder"
|
||||
)
|
||||
|
||||
# ━━━ Tier Delays — HOST1's Containers on HOST2 ━━━
|
||||
# How long HOST1 must be down before each tier activates on HOST2 — in minutes.
|
||||
# Tier 1 is always immediate — no delay var needed.
|
||||
HOST1_TIER2_DELAY=240 # 4 hours — NextCloud, Immich
|
||||
HOST1_TIER3_DELAY=720 # 12 hours — secondary services
|
||||
HOST1_TIER4_DELAY=1440 # 24 hours — arrs + downloaders
|
||||
|
||||
# ━━━ Rsync Writeback — HOST1 Appdata Back on Handback ━━━
|
||||
# Syncs HOST1 appdata BACK to HOST1 when it comes back online after a failover.
|
||||
# Containers stopped before writeback — clean source, no competing writes.
|
||||
#
|
||||
# HOST1_TIER1_WRITEBACK_DELAY: short outages skip Tier 1 writeback — primary state
|
||||
# is more reliable than dirty sync data for brief outages.
|
||||
HOST1_TIER1_WRITEBACK_DELAY=60 # skip Emby writeback if outage under 1hr
|
||||
|
||||
# Tier 4 automatically syncs HOST1_DAILY_SYNC_SHARES — only list paths NOT in that array.
|
||||
FALLBACK_HOST1_WRITEBACK_TIER1=(
|
||||
"/mnt/user/Media_Server/Emby" # watch states built up during outage
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_WRITEBACK_TIER2=(
|
||||
"/mnt/user/appdata-Fallback/Important-Data" # NextCloud + Postgres — files added during outage
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_WRITEBACK_TIER3=(
|
||||
# "location-placeholder"
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_WRITEBACK_TIER4=(
|
||||
"/mnt/user/appdata-Fallback/Arrs_Stack" # arr databases — downloads queued during outage
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── MEDIA ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Media Permissions ━━━
|
||||
# Shares that media_shares_permissions.sh applies PERMISSIONS_MODE and PERMISSIONS_OWNER to.
|
||||
# Runs first in DAILY_MAINTENANCE_SCRIPTS — arr cleanup depends on correct ownership.
|
||||
HOST1_MEDIA_PERMISSION_SHARES=(
|
||||
/mnt/user/Anime_Movies
|
||||
/mnt/user/Anime_Movies-Old
|
||||
/mnt/user/Anime_Shows
|
||||
/mnt/user/Anime_Shows-Old
|
||||
/mnt/user/appcache
|
||||
/mnt/user/Books
|
||||
/mnt/user/Downloads
|
||||
/mnt/user/Games
|
||||
/mnt/user/Intros
|
||||
/mnt/user/Kids_Movies
|
||||
/mnt/user/Kids_Tv_Shows
|
||||
/mnt/user/Movie_Recordings
|
||||
/mnt/user/Movies
|
||||
/mnt/user/Music
|
||||
/mnt/user/Music_Videos
|
||||
/mnt/user/Photo
|
||||
/mnt/user/Sports
|
||||
/mnt/user/stand-up_comedy
|
||||
/mnt/user/Tv_Recordings
|
||||
/mnt/user/Tv_Shows
|
||||
/mnt/user/YouTube
|
||||
)
|
||||
|
||||
# ━━━ Media Cleaner ━━━
|
||||
# Folder lists for media_cleaner.sh — two profiles: anime and media.
|
||||
# File patterns shared across all servers — defined in master.conf.
|
||||
# Called via DAILY_MAINTENANCE_SCRIPTS. Run manually: Media/media_cleaner.sh anime|media
|
||||
HOST1_ANIME_CLEAN_FOLDERS=(
|
||||
/mnt/user/Anime_Movies
|
||||
/mnt/user/Anime_Movies-Old
|
||||
/mnt/user/Anime_Shows
|
||||
/mnt/user/Anime_Shows-Old
|
||||
)
|
||||
|
||||
HOST1_MEDIA_CLEAN_FOLDERS=(
|
||||
/mnt/user/Kids_Movies
|
||||
/mnt/user/Kids_Tv_Shows
|
||||
/mnt/user/Movies
|
||||
/mnt/user/Music
|
||||
/mnt/user/Sports
|
||||
/mnt/user/stand-up_comedy
|
||||
/mnt/user/Tv_Shows
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── MONITORS ──────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Certificate Monitor ━━━
|
||||
# Domains checked via direct openssl connection — not relying on NPM's certificate state.
|
||||
# Checks the actual certificate served, not what NPM thinks it has.
|
||||
# Thresholds (CERT_WARN_DAYS, CERT_CRIT_DAYS) defined in master.conf.
|
||||
HOST1_CERT_MONITOR_DOMAINS=(
|
||||
"Gmer4Lfe.com"
|
||||
"Gmer4Lfe.us"
|
||||
)
|
||||
|
||||
# ━━━ SMART Health ━━━
|
||||
# Drives skipped in SMART attribute monitoring — hardware is server-specific.
|
||||
# Thresholds read from dynamix.cfg at runtime — fallbacks in master.conf.
|
||||
HOST1_SMART_IGNORE_DRIVES=(
|
||||
"sda" # boot USB — SMART not meaningful on flash drives
|
||||
)
|
||||
|
||||
# ━━━ ZFS Report ━━━
|
||||
# Pools excluded from the weekly ZFS health report — reduces noise from single-disk array pools.
|
||||
# These are individual array disks formatted as ZFS — converting to XFS over time via unBalance.
|
||||
# Pool health thresholds defined in master.conf.
|
||||
HOST1_ZFS_REPORT_IGNORE_POOLS=(
|
||||
"disk5"
|
||||
"disk6"
|
||||
"disk8"
|
||||
"disk9"
|
||||
"disk10"
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── TRANSCODES ────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# Ramdisk size ceiling — tmpfs only uses RAM actually needed, not the full size upfront.
|
||||
# Real-world: 9 streams peaked at ~5.5GB — 10G gives generous headroom on 128GB RAM.
|
||||
HOST1_RAMDISK_SIZE="10G"
|
||||
|
||||
# Usage thresholds — coupled to HOST1_RAMDISK_SIZE, adjust all three together if size changes.
|
||||
# Hysteresis gap (8.5 - 7 = 1.5GB) prevents flip-flop between ramdisk and SSD.
|
||||
HOST1_RAMDISK_WARN_GB=8.5 # flip to SSD when ramdisk usage reaches this
|
||||
HOST1_RAMDISK_LOW_GB=7 # flip back to ramdisk when usage drops to this
|
||||
|
||||
# SSD fallback path — where transcodes land when ramdisk exceeds HOST1_RAMDISK_WARN_GB.
|
||||
# Must be on cache pool — array disks too slow for active transcode writes.
|
||||
HOST1_TRANSCODE_SSD="/mnt/cache/Temp_Storage/Emby/Transcodes/"
|
||||
|
||||
# Media servers sharing the ramdisk transcode space on HOST1.
|
||||
# Format: "ContainerName|URL|APIKey|Type" — Type: emby | jellyfin | plex
|
||||
# Entries with placeholder API keys are skipped automatically.
|
||||
# ⚠️ Tdarr does NOT belong here — keep Tdarr on SSD, not ramdisk.
|
||||
HOST1_TRANSCODE_SERVERS=(
|
||||
"${HOST1_EMBY_CONTAINER}|${HOST1_EMBY_URL}|${HOST1_EMBY_API_KEY}|emby"
|
||||
"${HOST1_JELLYFIN_CONTAINER}|${HOST1_JELLYFIN_URL}|${HOST1_JELLYFIN_API_KEY}|jellyfin"
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── ARR STACK ─────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Used by arr cleanup scripts and arrs_failed_stalled_recovery.sh.
|
||||
# detect_hosts() selects HOST1 vars when running on HOST1.
|
||||
#
|
||||
# PATH MAPS — container path → host path translation.
|
||||
# Arr stores file paths using container-internal paths — scripts need host paths to scan.
|
||||
# Add one entry per root folder in arr Settings → Media Management → Root Folders.
|
||||
|
||||
# ━━━ Downloaders ━━━
|
||||
# Used by downloaders_reset.sh — runs every 30min via CRITICAL_MAINTENANCE_SCRIPTS.
|
||||
# Clears stuck states, purges old history, prepares each client for a clean cycle.
|
||||
|
||||
# slskd — clears stuck searches, dead transfers, purges expired failed imports.
|
||||
# SLSKD_FAILED_IMPORTS_DIR: where Soularr moves albums Lidarr rejected.
|
||||
HOST1_SLSKD_URL="http://localhost:8980"
|
||||
HOST1_SLSKD_API_KEY="4bF9kL2mNpQrT7vWxYz1A3dEgHjKoRsU"
|
||||
HOST1_SLSKD_FAILED_IMPORTS_DIR="/mnt/user/Temp_Storage/Slskd/completed/failed_imports"
|
||||
|
||||
# SABnzbd
|
||||
HOST1_SABNZBD_URL="http://localhost:8180"
|
||||
HOST1_SABNZBD_API_KEY="8bfefe41d83b4d50883e32859b55ca9a"
|
||||
|
||||
# qBittorrent — deleteFiles=false removes torrent from qBit but leaves files on disk.
|
||||
# Radarr/Sonarr manage actual files independently.
|
||||
HOST1_QBIT_URL="http://localhost:8080"
|
||||
HOST1_QBIT_USERNAME="root"
|
||||
HOST1_QBIT_PASSWORD="Stay0utD!ck"
|
||||
|
||||
# ━━━ Lidarr — HOST1 only ━━━
|
||||
# HOST2 does not run Lidarr — HOST1_LIDARR_RECOVERY flag handles the exit cleanly.
|
||||
HOST1_LIDARR_URL="http://localhost:8686"
|
||||
HOST1_LIDARR_API_KEY="b2977e71ef074bc0a0529d9fcce3b2dc"
|
||||
HOST1_LIDARR_MUSIC_ROOT="/mnt/user/Music-New"
|
||||
HOST1_FANART_API_KEY="Yd7147a43b692df0b364b94dc47efb81"
|
||||
HOST1_LASTFM_API_KEY="be6dc169c33ae263e690c30d18b7491d"
|
||||
|
||||
declare -A HOST1_LIDARR_PATH_MAP=(
|
||||
["/ext-music"]="/mnt/user/Music-New"
|
||||
)
|
||||
|
||||
# ━━━ Sonarr ━━━
|
||||
HOST1_SONARR_URL="http://localhost:8989"
|
||||
HOST1_SONARR_API_KEY="130decd3db5b4c25afad64864cd03f9f"
|
||||
HOST1_SONARR_TV_ROOT="/mnt/user/Tv_Shows"
|
||||
|
||||
# /ext-standup-comedy maps the stand-up root; /series subdir is the Sonarr root folder
|
||||
declare -A HOST1_SONARR_PATH_MAP=(
|
||||
["/tv"]="/mnt/user/Tv_Shows"
|
||||
["/ext-standup-comedy"]="/mnt/user/stand-up_comedy"
|
||||
["/kids tv"]="/mnt/user/Kids_Tv_Shows"
|
||||
["/ext-anime-shows"]="/mnt/user/Anime_Shows-Old"
|
||||
)
|
||||
|
||||
# ━━━ Radarr ━━━
|
||||
HOST1_RADARR_URL="http://localhost:7878"
|
||||
HOST1_RADARR_API_KEY="d43a3ec6cf1549edb4af0cc63f98b2a9"
|
||||
HOST1_TMDB_API_KEY="3dac5e2e49b5540472d2eafec4f01260"
|
||||
HOST1_RADARR_MOVIES_ROOT="/mnt/user/Movies"
|
||||
|
||||
# /ext-stand-up-comedy maps the stand-up root; /specials subdir is the Radarr root folder
|
||||
declare -A HOST1_RADARR_PATH_MAP=(
|
||||
["/movies"]="/mnt/user/Movies"
|
||||
["/kids movies"]="/mnt/user/Kids_Movies"
|
||||
["/ext-stand-up-comedy"]="/mnt/user/stand-up_comedy"
|
||||
["/anime-movies"]="/mnt/user/Anime_Movies-Old"
|
||||
)
|
||||
|
||||
# ━━━ Arr Recovery Toggles ━━━
|
||||
# false = skip that arr on this host — exits cleanly without error
|
||||
HOST1_SONARR_RECOVERY=true
|
||||
HOST1_RADARR_RECOVERY=true
|
||||
HOST1_LIDARR_RECOVERY=true # HOST1 only — exits cleanly on HOST2
|
||||
|
||||
# ==============================================================================================
|
||||
# ── SYSTEM WATCHDOG ───────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Per-host check toggles and NIC config for system_watchdog.sh.
|
||||
# Aliased by detect_hosts() — script uses unprefixed SYS_WATCHDOG_* names.
|
||||
# HOST1: TR1950X 128GB — full media server, active transcoding, ZFS cache pools.
|
||||
#
|
||||
# Three-tier response — all critical checks enabled by default on HOST1:
|
||||
# Tier 1 (bypass strikes, reboot now): docker daemon, rootfs full, kernel oops, FD, /boot
|
||||
# Tier 2 (bypass strikes with OOM): RAM critical + OOM kills in cycle
|
||||
# Tier 3 (standard strike system): everything else
|
||||
#
|
||||
# RAM tiers, OOM limits, and reboot loop settings in master.conf System Watchdog section.
|
||||
|
||||
# ━━━ Primary NIC ━━━
|
||||
# Network interface for NIC state check — verify with: ip link show | grep "^[0-9]"
|
||||
# Common values: eth0, bond0, br0, eno1
|
||||
HOST1_SYS_WATCHDOG_NIC="eth0"
|
||||
|
||||
# ━━━ Tier 1 — Critical Checks ━━━
|
||||
# These bypass the strike system — a single hit triggers immediate reboot.
|
||||
# Disabling any of these is not recommended — they protect against acute system failure.
|
||||
|
||||
# Docker daemon unresponsive → try restart, reboot if restart fails.
|
||||
# Without a working daemon docker_watchdog.sh is blind and containers cannot be managed.
|
||||
HOST1_SYS_WATCHDOG_CHECK_DOCKER_DAEMON=true
|
||||
|
||||
# rootfs at critical threshold (ROOTFS_CRITICAL_PCT=99) → reboot immediately.
|
||||
# At 99% rootfs writes fail silently — logs stop, Docker errors out, SSH may stop working.
|
||||
# Standard 95% threshold still uses strike system — only 99%+ is critical tier.
|
||||
HOST1_SYS_WATCHDOG_CHECK_ROOTFS=true
|
||||
|
||||
# Kernel BUG/Oops in dmesg delta since last cycle → reboot immediately.
|
||||
# A kernel oops means the kernel ran with a corrupted state — stability is not guaranteed.
|
||||
HOST1_SYS_WATCHDOG_CHECK_KERNEL_OOPS=true
|
||||
|
||||
# File descriptor exhaustion at FD_CRITICAL_PCT (95%) → reboot immediately.
|
||||
# At 95% FD: new connections fail, Docker can't spawn processes, SSH drops.
|
||||
HOST1_SYS_WATCHDOG_CHECK_FD=true
|
||||
|
||||
# /boot read-only detected → reboot immediately.
|
||||
# Unexpected read-only /boot means state files and config writes are silently failing.
|
||||
# Fallback state, watchdog reboot log, and lock files all go stale silently.
|
||||
HOST1_SYS_WATCHDOG_CHECK_BOOT=true
|
||||
|
||||
# ━━━ Tier 2 — Urgent OOM Check ━━━
|
||||
# Bypass strikes when RAM is critically low AND OOM kill rate confirms active crisis.
|
||||
# Both must be enabled for Tier 2 bypass to function — disable either to always use strikes.
|
||||
|
||||
# Track kernel OOM kills each cycle via /proc/vmstat oom_kill delta.
|
||||
# Also provides diagnostic context in reboot messages (which processes were killed).
|
||||
HOST1_SYS_WATCHDOG_CHECK_OOM=true
|
||||
|
||||
# Free RAM check — required for both Tier 2 bypass and RAM tier logic.
|
||||
# Tiers: MEM_WARN_GB(10) → notify | MEM_SHUTDOWN_GB(6) → stop containers | MEM_GB(4) → strikes
|
||||
HOST1_SYS_WATCHDOG_CHECK_RAM=true
|
||||
|
||||
# ━━━ Tier 3 — Standard Checks (strike system) ━━━
|
||||
# Each check must fail SYS_WATCHDOG_STRIKE_LIMIT consecutive cycles before action is taken.
|
||||
# Single spikes are ignored — sustained problems trigger reboot.
|
||||
|
||||
# /var/log filesystem usage above SYS_WATCHDOG_LOG_PCT.
|
||||
# Log spam (Docker log storms, syslog loops) fills rootfs — indicates something broken.
|
||||
HOST1_SYS_WATCHDOG_CHECK_LOG=true
|
||||
|
||||
# ZFS ARC memory pinned above SYS_WATCHDOG_ARC_PINNED_PCT after cache drop.
|
||||
# Enabled on HOST1 — ZFS cache pools actively used. Disable on hosts without ZFS.
|
||||
HOST1_SYS_WATCHDOG_CHECK_ARC=true
|
||||
|
||||
# CPU temperature above SYS_WATCHDOG_CPU_TEMP_MAX (95°C).
|
||||
# Sustained high temp causes kernel throttling or panic. Requires lm-sensors.
|
||||
HOST1_SYS_WATCHDOG_CHECK_CPU_TEMP=true
|
||||
|
||||
# Load average above SYS_WATCHDOG_LOAD_MULTIPLIER × core count.
|
||||
# DISABLED on HOST1 — Tdarr and Emby cause legitimate sustained load spikes during encoding.
|
||||
# Enable on idle servers or adjust SYS_WATCHDOG_LOAD_MULTIPLIER if load is always high.
|
||||
HOST1_SYS_WATCHDOG_CHECK_LOAD=false
|
||||
|
||||
# Zombie process count above SYS_WATCHDOG_ZOMBIE_LIMIT (50).
|
||||
# Large zombie counts indicate serious process management failure — something is stuck.
|
||||
HOST1_SYS_WATCHDOG_CHECK_ZOMBIES=true
|
||||
|
||||
# Check docker_watchdog.sh persistent skip list — required containers on skip list.
|
||||
# Cross-watchdog coordination: if docker_watchdog gave up, system_watchdog escalates.
|
||||
# ENABLED — HOST1 fully built and operational, skip list is meaningful.
|
||||
HOST1_SYS_WATCHDOG_CHECK_CONTAINERS=true
|
||||
|
||||
# /tmp filesystem usage above SYS_WATCHDOG_TMP_PCT with auto-clear attempt.
|
||||
# Script tries to clear aged /tmp files first — only strikes if clear fails.
|
||||
# Lock files, rsync temp files, and Docker ops use /tmp — 100% means lock failures.
|
||||
HOST1_SYS_WATCHDOG_CHECK_TMP=true
|
||||
|
||||
# Array disk error count delta in /proc/mdstat — accumulating errors = disk failing now.
|
||||
# Triggers on SYS_WATCHDOG_MDSTAT_ERROR_LIMIT new errors in one cycle.
|
||||
HOST1_SYS_WATCHDOG_CHECK_MDSTAT=true
|
||||
|
||||
# Primary NIC operstate — detects NIC going down (physical or driver failure).
|
||||
# Uses HOST1_SYS_WATCHDOG_NIC above. Strike system — brief flaps don't trigger reboot.
|
||||
HOST1_SYS_WATCHDOG_CHECK_NETWORK=true
|
||||
|
||||
# sshd running check — attempts restart before escalating.
|
||||
# sshd down = no remote access. Script tries rc.sshd start, notifies, strikes on failure.
|
||||
HOST1_SYS_WATCHDOG_CHECK_SSHD=true
|
||||
|
||||
# Runaway process detection — single process above SYS_WATCHDOG_RUNAWAY_CPU_PCT sustained.
|
||||
# DISABLED — Tdarr encoding and Emby transcoding legitimately peg CPU for extended periods.
|
||||
# Enable only if HOST1 has no CPU-intensive workloads.
|
||||
HOST1_SYS_WATCHDOG_CHECK_RUNAWAY=false
|
||||
|
||||
# ==============================================================================================
|
||||
# ── RESOURCE MANAGER ──────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Containers to manage under pressure — see master.conf RW_CRITICAL_CONTAINERS for exclusions.
|
||||
|
||||
# docker pause at medium pressure (RAM < RW_RAM_MEDIUM_GB or load > medium threshold)
|
||||
# Suspended in-place — instant to pause/unpause, no state lost, no restart delay.
|
||||
HOST1_RW_PAUSE_CONTAINERS=(
|
||||
"Huntarr" # arr search automation — safe to suspend
|
||||
"Cleanuparr" # download cleanup — safe to suspend
|
||||
"Healarr" # arr health checks — safe to suspend
|
||||
"Soularr" # Slskd automation — background only
|
||||
"ChannelTube" # YouTube archiver — background only
|
||||
"Pinchflat" # YouTube archiver — background only
|
||||
)
|
||||
|
||||
# docker stop at hard pressure (RAM < RW_RAM_HARD_GB)
|
||||
# Full stop — these are optional/heavy services that free significant RAM when stopped.
|
||||
# resource_watchdog.sh restarts them when pressure fully clears (RAM >= RW_RAM_RECOVER_GB).
|
||||
HOST1_RW_STOP_CONTAINERS=(
|
||||
"LocalAI" # GPU/CPU heavy — largest RAM consumer when idle
|
||||
"7DaysToDie" # game server — optional
|
||||
"V-Rising" # game server — optional
|
||||
"Code-Server" # IDE — not needed during pressure events
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── AUTH STACK ────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Credentials for the Varaverk Auth Stack page (NPM, lldap, Authelia).
|
||||
|
||||
# ━━━ NginxProxyManager ━━━
|
||||
HOST1_NPM_URL="http://localhost:7818"
|
||||
HOST1_NPM_USER="failedproxy@gmail.com"
|
||||
HOST1_NPM_PASS="183134\$eanHess"
|
||||
|
||||
# ━━━ lldap ━━━
|
||||
HOST1_LLDAP_URL="http://localhost:17170"
|
||||
HOST1_LLDAP_USER="admin"
|
||||
HOST1_LLDAP_PASS="183134\$eanHess"
|
||||
|
||||
# ━━━ Authelia ━━━
|
||||
HOST1_AUTHELIA_CONFIG="/mnt/user/appdata-Fallback/Critical-Data/Authelia/configuration.yml"
|
||||
HOST1_AUTHELIA_CONTAINER="Authelia"
|
||||
|
||||
# ==============================================================================================
|
||||
# ──────────────────────── End Of HOST1 Variables ──────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
HOST1_UNRAID_API_KEY="1825c3a2e03ea5089974f4da2e171aa2d5907a1dea23cc479c33e492c8ff4dbb"
|
||||
@@ -0,0 +1,787 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ========================== HOST1 CONFIGURATION — unRAID-Gmer4Lfe ============================
|
||||
# ==============================================================================================
|
||||
# HOST1-specific variables — credentials, container names, share paths, failover lists.
|
||||
# Sourced after master.conf — values here extend shared profile arrays and add HOST1-specific
|
||||
# identity, credentials, and container configuration.
|
||||
#
|
||||
# Sparse checkout (git) ensures HOST2 never receives this file.
|
||||
# HOST2 never sees HOST1 credentials — clean separation at the file level.
|
||||
#
|
||||
# DO NOT put shared config here — thresholds, toggles, profiles belong in master.conf.
|
||||
# DO NOT put HOST2 variables here — they belong in host2.conf.
|
||||
#
|
||||
# ── INDEX ─────────────────────────────────────────────────────────────────────────────────────
|
||||
#
|
||||
# ── IDENTITY & CONNECTIVITY ────────────────────────────────────────────────────────────────
|
||||
# IDENTITY hostname, SSH key
|
||||
# EMBY container name, URL, API key
|
||||
# NOTIFICATIONS Discord webhook
|
||||
# PARTNERSHIP auth containers, backup paths
|
||||
#
|
||||
# ── RSYNC ──────────────────────────────────────────────────────────────────────────────────
|
||||
# DAILY SYNC SHARES media shares HOST1 owns and pushes to HOST2
|
||||
# WEEKLY SYNC SHARES appdata shares synced weekly (Sunday 2:30am)
|
||||
# CRITICAL SYNC SHARES appdata shares synced every 30 minutes
|
||||
# BACKUP VERIFY shares for checksum verification against remote
|
||||
# HOST1 RSYNC PROFILE host1-appdata profile for HOST1-specific appdata syncs
|
||||
#
|
||||
# ── DOCKER ─────────────────────────────────────────────────────────────────────────────────
|
||||
# DOCKER DAILY RESTART containers restarted daily
|
||||
# DOCKER WEEKLY RESTART containers restarted weekly
|
||||
# DOCKER WATCHDOG memory limits, health URLs, required containers, ignore list
|
||||
# DOCKER NETWORK CONNECT networks and containers for docker_network_connect.sh
|
||||
#
|
||||
# ── FALLBACK ───────────────────────────────────────────────────────────────────────────────
|
||||
# DDNS DDNS containers managed by HOST1
|
||||
# INTERNET LOSS containers stopped when internet is lost
|
||||
# FALLBACK TIERS what HOST1 runs for HOST2 per tier
|
||||
# TIER DELAYS how long HOST1 must be down before each tier activates on HOST2
|
||||
# RSYNC WRITEBACK HOST1 appdata synced back on handback
|
||||
#
|
||||
# ── MEDIA ──────────────────────────────────────────────────────────────────────────────────
|
||||
# MEDIA PERMISSIONS share list for media_shares_permissions.sh
|
||||
# MEDIA CLEANER folder lists for media_cleaner.sh
|
||||
#
|
||||
# ── MONITORS ───────────────────────────────────────────────────────────────────────────────
|
||||
# CERTIFICATE MONITOR domains checked for SSL expiry
|
||||
# SMART HEALTH drives to skip in SMART monitoring
|
||||
# ZFS REPORT pools to exclude from ZFS health report
|
||||
#
|
||||
# ── TRANSCODES ─────────────────────────────────────────────────────────────────────────────
|
||||
# TRANSCODES ramdisk size, thresholds, SSD path, server array
|
||||
#
|
||||
# ── ARR STACK ──────────────────────────────────────────────────────────────────────────────
|
||||
# DOWNLOADERS slskd, SABnzbd, qBittorrent credentials and URLs
|
||||
# LIDARR URL, API key, path map
|
||||
# SONARR URL, API key, path map
|
||||
# RADARR URL, API key, path map
|
||||
# ARR RECOVERY per-arr recovery toggles
|
||||
#
|
||||
# ── SYSTEM WATCHDOG ────────────────────────────────────────────────────────────────────────
|
||||
# SYSTEM WATCHDOG per-host check toggles and NIC configuration
|
||||
#
|
||||
# ── RESOURCE MANAGER ───────────────────────────────────────────────────────────────────────
|
||||
# RESOURCE MANAGER containers paused/stopped under memory pressure
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
# ==============================================================================================
|
||||
# ── IDENTITY & CONNECTIVITY ───────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Identity ━━━
|
||||
# HOST1 hostname lives in master.conf (not a credential — safe for all servers).
|
||||
# SSH key used for all server-to-server operations — rsync, failover container commands.
|
||||
# Must be in /root/.ssh/ and authorised in HOST2's /root/.ssh/authorized_keys.
|
||||
HOST1_SSH_KEY="/root/.ssh/gmer4lfe_rsync_automation"
|
||||
HOST1_OWNER="gmer4lfe"
|
||||
HOST1_OWNER_EMAIL="gmer4lfe@gmail.com"
|
||||
|
||||
# ━━━ Emby ━━━
|
||||
# Referenced by transcode_manager.sh, emby_session_report.sh, emby_database_repair.sh,
|
||||
# weekly_sync_maintenance.sh, and HOST1_TRANSCODE_SERVERS below.
|
||||
# API key: Emby Dashboard → API Keys → + New Key
|
||||
HOST1_EMBY_CONTAINER="Emby"
|
||||
HOST1_EMBY_URL="http://localhost:8096"
|
||||
HOST1_EMBY_API_KEY="0c27448d93a7431f9ac63569f7655829"
|
||||
|
||||
# ━━━ Jellyfin ━━━
|
||||
# API key: Jellyfin Dashboard → Administration → API Keys → + New Key
|
||||
HOST1_JELLYFIN_CONTAINER="Jellyfin"
|
||||
HOST1_JELLYFIN_URL="http://localhost:8095"
|
||||
HOST1_JELLYFIN_API_KEY="4e820e7df74c4933acec212b1996314e"
|
||||
|
||||
# ━━━ Gitea ━━━
|
||||
# Personal access token for gitea_ssh_setup.sh — registers this server's SSH public key
|
||||
# with Gitea so git operations use key auth instead of passwords.
|
||||
# Create in Gitea: Settings → Applications → Generate Token → scope: write:user
|
||||
HOST1_GITEA_API_TOKEN=""
|
||||
|
||||
# ━━━ Notifications ━━━
|
||||
# Discord webhook — leave blank to disable.
|
||||
# Per-host so HOST1 and HOST2 can post to different channels or only one server notifies.
|
||||
HOST1_DISCORD_WEBHOOK=""
|
||||
|
||||
# ━━━ Partnership ━━━
|
||||
# HOST1 is always the owner (source of truth) unless --transfer has been run.
|
||||
# See README-Partnership.md and master.conf PARTNERSHIP section for full lifecycle docs.
|
||||
|
||||
# Auth containers reconfigured on onboard/offboard.
|
||||
# Format: "ContainerName|WebUIPort"
|
||||
# On onboard → WebUI pointed at owner's Tailscale IP (mirror clicks NPM, gets owner's auth)
|
||||
# On offboard → WebUI pointed back at localhost
|
||||
HOST1_PARTNERSHIP_AUTH_WEBUIS=(
|
||||
"NginxProxyManager|81"
|
||||
"Lldap-Gmer4Lfe|17170"
|
||||
"Authelia|9091"
|
||||
"Authelia-Secondary|9092"
|
||||
)
|
||||
|
||||
# XML templates (from this server's templates-user/) pushed to mirror during onboard.
|
||||
# These become the mirror's active auth stack, backed by the rsync-synced appdata.
|
||||
# Update filename if Lldap is renamed to drop the host suffix.
|
||||
HOST1_PARTNERSHIP_AUTH_STACK=(
|
||||
# Dependencies first — Mariadb/Redis must be healthy before Authelia starts
|
||||
"my-Mariadb-Authelia.xml"
|
||||
"my-Mariadb-Authelia-Secondary.xml"
|
||||
"my-Redis-Authelia.xml"
|
||||
"my-Redis-Authelia-Secondary.xml"
|
||||
# Auth apps — deployed after their deps are confirmed healthy
|
||||
"my-Authelia.xml"
|
||||
"my-Authelia-Secondary.xml"
|
||||
"my-NginxProxyManager.xml"
|
||||
"my-Lldap-Gmer4Lfe.xml"
|
||||
# Source of truth — must be available on HOST2 independently of the auth stack
|
||||
"my-Gitea.xml"
|
||||
)
|
||||
|
||||
# XML templates pushed to mirror for the arr stack during onboard.
|
||||
# Deps (e.g. databases) first if any — same ordering rule as auth stack.
|
||||
HOST1_PARTNERSHIP_ARR_STACK=(
|
||||
# "my-Sonarr.xml"
|
||||
# "my-Radarr.xml"
|
||||
# "my-Lidarr.xml"
|
||||
# "my-Prowlarr.xml"
|
||||
# "my-Bazarr.xml"
|
||||
)
|
||||
|
||||
# Paths HOST2 should collect during the grace window after offboard.
|
||||
# Notified on offboard — no auto-deletion, HOST2 must collect manually within PARTNERSHIP_GRACE_HOURS.
|
||||
HOST1_PARTNERSHIP_MIRROR_BACKUPS=(
|
||||
# "/mnt/user/appdata-Fallback/Jayred365-Emby"
|
||||
)
|
||||
|
||||
# Containers parked on this server when partnership is active.
|
||||
# Stopped on onboard (owner deploys its stack instead), restarted on offboard.
|
||||
HOST1_PARTNERSHIP_OWN_CONTAINERS=(
|
||||
# "Emby"
|
||||
# "NginxProxyManager"
|
||||
)
|
||||
|
||||
# Emby admin provisioning — toggle is owner-only, credentials are per-host.
|
||||
# Owner enables/disables the feature. Each host sets the account they want on the shared Emby.
|
||||
# On onboard: owner reads mirror's HOST*_PARTNERSHIP_EMBY_ADMIN_* and creates that account.
|
||||
# On offboard: account is deleted. Username collision → onboard exits with error.
|
||||
HOST1_PARTNERSHIP_PROVISION_EMBY_ADMIN=false # owner controls whether Emby is shared
|
||||
HOST1_PARTNERSHIP_EMBY_PORT=8096
|
||||
HOST1_PARTNERSHIP_EMBY_ADMIN_USER="" # this server's desired Emby username
|
||||
HOST1_PARTNERSHIP_EMBY_ADMIN_PASS="" # this server's desired Emby password
|
||||
|
||||
# ==============================================================================================
|
||||
# ── RSYNC ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Daily Sync Shares ━━━
|
||||
# Shares HOST1 pushes to all other nodes every night (1am via daily_sync_maintenance.sh).
|
||||
# Mesh model: every node pushes every media share — no ownership, no mirrors.
|
||||
# arr_sync ensures all arr libraries converge (union). rsync spreads files (additive, no --delete).
|
||||
# arr_cleanup removes true orphans based on local arr state.
|
||||
# Any node can download content to any share — it propagates to all nodes on the next cycle.
|
||||
# Nextcloud is intentionally one-directional (HOST1→HOST2 offsite backup — not arr-managed).
|
||||
# Uses DEFAULT_RSYNC_OPTS from master.conf — no profile needed.
|
||||
# For shares needing container stops or custom options — add a profile in master.conf.
|
||||
HOST1_DAILY_SYNC_SHARES=(
|
||||
/mnt/user/Books
|
||||
/mnt/user/Intros
|
||||
/mnt/user/Kids_Movies
|
||||
/mnt/user/Kids_Tv_Shows
|
||||
/mnt/user/Movies
|
||||
/mnt/user/Music
|
||||
/mnt/user/Music_Videos
|
||||
/mnt/user/Nextcloud
|
||||
/mnt/user/stand-up_comedy
|
||||
/mnt/user/Sports
|
||||
# /mnt/user/Tv_Shows
|
||||
/mnt/user/Anime_Shows-Old
|
||||
/mnt/user/Anime_Movies-Old
|
||||
/mnt/user/Anime_Movies
|
||||
/mnt/user/Anime_Shows
|
||||
)
|
||||
|
||||
# Personal encrypted shares — synced for offsite backup, independent of media shares.
|
||||
# ZFS encrypted at dataset level — remote receives encrypted blocks, cannot read content.
|
||||
# See README-Rsync_Setup.md for ZFS encryption setup before uncommenting.
|
||||
HOST1_PERSONAL_SHARES=(
|
||||
# /mnt/user/HOST1-Personal # uncomment after creating encrypted dataset
|
||||
)
|
||||
|
||||
# ━━━ Weekly Sync Shares ━━━
|
||||
# Appdata shares synced during the weekly maintenance window (Sunday 2:30am).
|
||||
# Containers stopped both sides before sync — full clean state guaranteed.
|
||||
# Profiles drive container stops, excludes, and options — configured in master.conf RSYNC section.
|
||||
# Order matters — Emby first (larger transfer), then Critical-Data (auth stack).
|
||||
HOST1_WEEKLY_SYNC_SHARES=(
|
||||
"/mnt/user/Media_Server/Emby" # emby profile — full clean mirror
|
||||
"/mnt/user/appdata-Fallback/Critical-Data" # critical-data profile — auth stack
|
||||
)
|
||||
|
||||
# ━━━ Intermediate Sync Shares ━━━
|
||||
# Shares synced every 4 hours by intermediate_sync_maintenance.sh.
|
||||
# Uses DEFAULT_RSYNC_OPTS (no --delete) — for sub-daily propagation of metadata or watch state.
|
||||
# Full media share sync stays in the daily window. Leave empty to skip mid-day rsync entirely.
|
||||
HOST1_INTERMEDIATE_SYNC_SHARES=(
|
||||
# Add shares here to enable mid-day rsync
|
||||
# Example: "/mnt/user/Emby_Metadata"
|
||||
)
|
||||
|
||||
# ━━━ Critical Sync Shares ━━━
|
||||
# Appdata shares synced every 30 minutes by critical_sync_maintenance.sh.
|
||||
# Format: "/path/to/share" or "/path/to/share|profile-name"
|
||||
# Order matters — Critical-Data first (auth stack), then Emby dirty sync.
|
||||
HOST1_CRITICAL_SYNC_SHARES=(
|
||||
"/mnt/user/appdata-Fallback/Critical-Data|critical-fallback" # auth dirty sync — stays running
|
||||
"/mnt/user/Media_Server/Emby|emby-fallback" # Emby dirty sync — stays running
|
||||
)
|
||||
|
||||
# ━━━ Backup Verify ━━━
|
||||
# Shares verified by backup_verify.sh — random file checksum comparison against remote.
|
||||
# Leave empty to use HOST1_DAILY_SYNC_SHARES automatically.
|
||||
# Sample size and minimum file size defined in master.conf.
|
||||
HOST1_BACKUP_VERIFY_SHARES=(
|
||||
# leave empty to use HOST1_DAILY_SYNC_SHARES automatically
|
||||
)
|
||||
|
||||
# ━━━ HOST1 Rsync Profile — host1-appdata ━━━
|
||||
# HOST1-specific appdata sync profile — extends the shared PROFILE_* arrays in master.conf.
|
||||
# Use for appdata unique to HOST1 (Organizrv2, VaultWarden, UptimeKuma etc.)
|
||||
# Shared appdata (auth stack, Emby) use dedicated profiles defined in master.conf.
|
||||
# Run manually: bash Rsync/rsync.sh /mnt/user/appdata-Fallback/HOST1-Appdata --profile=host1-appdata
|
||||
PROFILE_RSYNC_OPTS[host1-appdata]="-av --info=progress2 --bwlimit=${PROFILE_BW_LIMIT[host1-appdata]:-8000}"
|
||||
PROFILE_BW_LIMIT[host1-appdata]=8000
|
||||
PROFILE_RETRY_COUNT[host1-appdata]=3
|
||||
PROFILE_SLEEP[host1-appdata]=300
|
||||
PROFILE_CRITICAL_CONTAINER_NAMES[host1-appdata]="Organizrv2-Gmer4Lfe UptimeKuma-Gmer4Lfe VaultWarden-Gmer4Lfe"
|
||||
PROFILE_DELAYED_CONTAINERS[host1-appdata]=""
|
||||
PROFILE_CONTAINER_DELAY[host1-appdata]=5
|
||||
PROFILE_EXCLUDE_DIRS[host1-appdata]="logs *.tmp"
|
||||
|
||||
# ==============================================================================================
|
||||
# ── DOCKER ────────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Docker Daily Restart ━━━
|
||||
# Containers restarted every day via DAILY_MAINTENANCE_SCRIPTS.
|
||||
# Dispatcharr degrades over time without restart — daily is intentional, not just housekeeping.
|
||||
# Order matters — auth stack first, then media services.
|
||||
HOST1_DAILY_RESTART_CONTAINERS=(
|
||||
"NginxProxyManager"
|
||||
"Lldap-Gmer4Lfe"
|
||||
"Authelia"
|
||||
"Authelia-Secondary"
|
||||
"Dispatcharr-Iptv-Users"
|
||||
"Dispatcharr" # Live TV scheduler — degrades without daily restart
|
||||
"Dispatcharr-Basic"
|
||||
"ErsatzTV-Emby"
|
||||
"Slskd" # Soulseek connection drops after extended uptime; restart refreshes share index
|
||||
)
|
||||
|
||||
# ━━━ Docker Weekly Restart ━━━
|
||||
# Less critical services restarted weekly via WEEKLY_MAINTENANCE_SCRIPTS (Sunday 2:30am).
|
||||
# Containers already stopped for weekly sync — restart adds zero extra downtime.
|
||||
HOST1_WEEKLY_RESTART_CONTAINERS=(
|
||||
"NextCloud"
|
||||
"Organizrv2-Gmer4Lfe"
|
||||
"AdGuard-Home"
|
||||
"Immich-Gmer4Lfe"
|
||||
)
|
||||
|
||||
# ━━━ Docker Watchdog ━━━
|
||||
# Per-HOST1 container configuration for docker_watchdog.sh.
|
||||
# Shared thresholds and toggles live in master.conf.
|
||||
|
||||
# Memory hard limits in MB — immediate restart if exceeded.
|
||||
# Set at "container is clearly broken" not "container is busy".
|
||||
# 20GB=20480 18GB=18432 16GB=16384 12GB=12288 8GB=8192 4GB=4096 2GB=2048 1GB=1024
|
||||
declare -A HOST1_WATCHDOG_CONTAINERS=(
|
||||
["Emby"]=20480 # 20GB — large library + active transcodes
|
||||
["LidaTube"]=6144 # 6GB — memory leak over time
|
||||
["Tdarr"]=6144 # 6GB — encoding is memory intensive
|
||||
["Code-Server"]=1024 # 1GB — should never need more
|
||||
)
|
||||
|
||||
# HTTP health check URLs — checked every cycle, strike system before restart.
|
||||
# Only add containers with a meaningful web interface to check.
|
||||
declare -A HOST1_WATCHDOG_CONTAINER_URLS=(
|
||||
["Emby"]="http://localhost:8096"
|
||||
)
|
||||
|
||||
# Required containers — must always be running on HOST1.
|
||||
# Strike system before restart — repeated failures go on skip list, auto-clears on recovery.
|
||||
# Listed in dependency order — dependencies before dependents.
|
||||
HOST1_WATCHDOG_REQUIRED_CONTAINERS=(
|
||||
"NginxProxyManager"
|
||||
"Lldap-Gmer4Lfe"
|
||||
"Mariadb-Authelia"
|
||||
"Mariadb-Authelia-Secondary"
|
||||
"Redis-Authelia"
|
||||
"Redis-Authelia-Secondary"
|
||||
"Authelia"
|
||||
"Authelia-Secondary"
|
||||
)
|
||||
|
||||
# Containers to skip in Tier 2 global scan — legitimately stopped or frequently restarting.
|
||||
# Watchdog leaves these alone entirely — no restart attempts, no crash loop tracking.
|
||||
HOST1_WATCHDOG_SCAN_IGNORE=(
|
||||
"DashGate"
|
||||
"PIA-WG-Config-Generator"
|
||||
"Aperture"
|
||||
"Aperture-Kids"
|
||||
"pgvector-18-Apeture-Kids"
|
||||
"Pgvector18-Aperture"
|
||||
"emby-test" # broken test container (exit 127 — bad image)
|
||||
)
|
||||
|
||||
# Dependency ordering — skip restarting a container if its dependency is also down.
|
||||
# Prevents watchdog from restarting Authelia before Mariadb is back up.
|
||||
# SPACE-SEPARATED STRINGS — converted to array at runtime.
|
||||
declare -A HOST1_WATCHDOG_DEPENDENCIES=(
|
||||
["Authelia"]="Mariadb-Authelia Redis-Authelia"
|
||||
["Authelia-Secondary"]="Mariadb-Authelia Redis-Authelia-Secondary"
|
||||
["NextCloud"]="Postgres-NextCloud"
|
||||
)
|
||||
|
||||
# Per-container appdata growth suppress ceilings in MB.
|
||||
# ONLY needed in specific cases — growth rate detection covers all containers automatically.
|
||||
# Use this when a container legitimately has large stable data and you want to guarantee
|
||||
# it never triggers a false-positive growth alert. Growth warnings are suppressed while the
|
||||
# container's dir stays below this ceiling; above it, warnings resume as normal.
|
||||
# 50GB=51200 25GB=25600 20GB=20480 15GB=15360 10GB=10240 5GB=5120
|
||||
declare -A HOST1_WATCHDOG_APPDATA_SIZES=(
|
||||
["Tdarr"]="25600" # 25GB — transcode cache grows legitimately during active jobs
|
||||
["7dtd"]="20480" # 20GB — game server world data, expected to be large
|
||||
)
|
||||
|
||||
# ━━━ Network Watchdog ━━━
|
||||
# Host-specific connectivity config for Watchdogs/System/network_watchdog.sh.
|
||||
HOST1_NETWORK_WATCHDOG_DDNS_DOMAIN="gmer4lfe.com"
|
||||
HOST1_NETWORK_WATCHDOG_DDNS_CONTAINER="Gmer4Lfe.com"
|
||||
HOST1_NETWORK_WATCHDOG_NPM_URL="https://gmer4lfe.com"
|
||||
|
||||
# ━━━ Docker Network Connect ━━━
|
||||
# Containers connected to custom networks at array start by docker_network_connect.sh.
|
||||
# Networks created if they don't exist — idempotent, safe to re-run.
|
||||
HOST1_NETWORK_CONNECT_CONTAINERS=(
|
||||
"memcached"
|
||||
"Npm-CrowdSec"
|
||||
)
|
||||
|
||||
HOST1_NETWORK_CONNECT_NETWORKS=(
|
||||
"high-availability"
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── FALLBACK ──────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ DDNS ━━━
|
||||
# DDNS containers HOST1 manages — started/stopped by fallback.sh per DDNS absolute rules:
|
||||
# Internet loss → stop immediately
|
||||
# Failover → HOST2 starts HOST1's DDNS as Tier 1 (before any other containers)
|
||||
# Handback → stop HOST1's DDNS on HOST2 → rsync → start containers → start local DDNS last
|
||||
HOST1_DDNS_CONTAINERS=(
|
||||
"Gmer4Lfe.com"
|
||||
)
|
||||
|
||||
# ━━━ Internet Loss ━━━
|
||||
# Containers stopped immediately on HOST1 when internet connection is lost.
|
||||
# Prevents external-facing services from operating without connectivity.
|
||||
FALLBACK_HOST1_STOP_ON_NO_NET=(
|
||||
"Gmer4Lfe.com"
|
||||
)
|
||||
|
||||
# ━━━ Fallback Tiers — HOST1 Runs for HOST2 ━━━
|
||||
# Containers HOST1 starts when HOST2 goes down.
|
||||
# Tier 1 is always immediate — vital services cannot wait.
|
||||
# Higher tiers activate after HOST2_TIER*_DELAY minutes (set in host2.conf).
|
||||
FALLBACK_HOST1_COVERS_HOST2_TIER1=(
|
||||
"Gmer4Lfe.us"
|
||||
"VaultWarden-Jayred365"
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_COVERS_HOST2_TIER2=(
|
||||
# "container-placeholder"
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_COVERS_HOST2_TIER3=(
|
||||
# "container-placeholder"
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_COVERS_HOST2_TIER4=(
|
||||
# "container-placeholder"
|
||||
)
|
||||
|
||||
# ━━━ Tier Delays — HOST1's Containers on HOST2 ━━━
|
||||
# How long HOST1 must be down before each tier activates on HOST2 — in minutes.
|
||||
# Tier 1 is always immediate — no delay var needed.
|
||||
HOST1_TIER2_DELAY=240 # 4 hours — NextCloud, Immich
|
||||
HOST1_TIER3_DELAY=720 # 12 hours — secondary services
|
||||
HOST1_TIER4_DELAY=1440 # 24 hours — arrs + downloaders
|
||||
|
||||
# ━━━ Rsync Writeback — HOST1 Appdata Back on Handback ━━━
|
||||
# Syncs HOST1 appdata BACK to HOST1 when it comes back online after a failover.
|
||||
# Containers stopped before writeback — clean source, no competing writes.
|
||||
#
|
||||
# HOST1_TIER1_WRITEBACK_DELAY: short outages skip Tier 1 writeback — primary state
|
||||
# is more reliable than dirty sync data for brief outages.
|
||||
HOST1_TIER1_WRITEBACK_DELAY=60 # skip Emby writeback if outage under 1hr
|
||||
|
||||
# Tier 4 automatically syncs HOST1_DAILY_SYNC_SHARES — only list paths NOT in that array.
|
||||
FALLBACK_HOST1_WRITEBACK_TIER1=(
|
||||
"/mnt/user/Media_Server/Emby" # watch states built up during outage
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_WRITEBACK_TIER2=(
|
||||
"/mnt/user/appdata-Fallback/Important-Data" # NextCloud + Postgres — files added during outage
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_WRITEBACK_TIER3=(
|
||||
# "location-placeholder"
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_WRITEBACK_TIER4=(
|
||||
"/mnt/user/appdata-Fallback/Arrs_Stack" # arr databases — downloads queued during outage
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── MEDIA ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Media Permissions ━━━
|
||||
# Shares that media_shares_permissions.sh applies PERMISSIONS_MODE and PERMISSIONS_OWNER to.
|
||||
# Runs first in DAILY_MAINTENANCE_SCRIPTS — arr cleanup depends on correct ownership.
|
||||
HOST1_MEDIA_PERMISSION_SHARES=(
|
||||
/mnt/user/Anime_Movies
|
||||
/mnt/user/Anime_Movies-Old
|
||||
/mnt/user/Anime_Shows
|
||||
/mnt/user/Anime_Shows-Old
|
||||
/mnt/user/appcache
|
||||
/mnt/user/Books
|
||||
/mnt/user/Downloads
|
||||
/mnt/user/Games
|
||||
/mnt/user/Intros
|
||||
/mnt/user/Kids_Movies
|
||||
/mnt/user/Kids_Tv_Shows
|
||||
/mnt/user/Movie_Recordings
|
||||
/mnt/user/Movies
|
||||
/mnt/user/Music
|
||||
/mnt/user/Music_Videos
|
||||
/mnt/user/Photo
|
||||
/mnt/user/Sports
|
||||
/mnt/user/stand-up_comedy
|
||||
/mnt/user/Tv_Recordings
|
||||
/mnt/user/Tv_Shows
|
||||
/mnt/user/YouTube
|
||||
)
|
||||
|
||||
# ━━━ Media Cleaner ━━━
|
||||
# Folder lists for media_cleaner.sh — two profiles: anime and media.
|
||||
# File patterns shared across all servers — defined in master.conf.
|
||||
# Called via DAILY_MAINTENANCE_SCRIPTS. Run manually: Media/media_cleaner.sh anime|media
|
||||
HOST1_ANIME_CLEAN_FOLDERS=(
|
||||
/mnt/user/Anime_Movies
|
||||
/mnt/user/Anime_Movies-Old
|
||||
/mnt/user/Anime_Shows
|
||||
/mnt/user/Anime_Shows-Old
|
||||
)
|
||||
|
||||
HOST1_MEDIA_CLEAN_FOLDERS=(
|
||||
/mnt/user/Kids_Movies
|
||||
/mnt/user/Kids_Tv_Shows
|
||||
/mnt/user/Movies
|
||||
/mnt/user/Music
|
||||
/mnt/user/Sports
|
||||
/mnt/user/stand-up_comedy
|
||||
/mnt/user/Tv_Shows
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── MONITORS ──────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Certificate Monitor ━━━
|
||||
# Domains checked via direct openssl connection — not relying on NPM's certificate state.
|
||||
# Checks the actual certificate served, not what NPM thinks it has.
|
||||
# Thresholds (CERT_WARN_DAYS, CERT_CRIT_DAYS) defined in master.conf.
|
||||
HOST1_CERT_MONITOR_DOMAINS=(
|
||||
"Gmer4Lfe.com"
|
||||
"Gmer4Lfe.us"
|
||||
)
|
||||
|
||||
# ━━━ SMART Health ━━━
|
||||
# Drives skipped in SMART attribute monitoring — hardware is server-specific.
|
||||
# Thresholds read from dynamix.cfg at runtime — fallbacks in master.conf.
|
||||
HOST1_SMART_IGNORE_DRIVES=(
|
||||
"sda" # boot USB — SMART not meaningful on flash drives
|
||||
)
|
||||
|
||||
# ━━━ ZFS Report ━━━
|
||||
# Pools excluded from the weekly ZFS health report — reduces noise from single-disk array pools.
|
||||
# These are individual array disks formatted as ZFS — converting to XFS over time via unBalance.
|
||||
# Pool health thresholds defined in master.conf.
|
||||
HOST1_ZFS_REPORT_IGNORE_POOLS=(
|
||||
"disk5"
|
||||
"disk6"
|
||||
"disk8"
|
||||
"disk9"
|
||||
"disk10"
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── TRANSCODES ────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# Ramdisk size ceiling — tmpfs only uses RAM actually needed, not the full size upfront.
|
||||
# Real-world: 9 streams peaked at ~5.5GB — 10G gives generous headroom on 128GB RAM.
|
||||
HOST1_RAMDISK_SIZE="10G"
|
||||
|
||||
# Usage thresholds — coupled to HOST1_RAMDISK_SIZE, adjust all three together if size changes.
|
||||
# Hysteresis gap (8.5 - 7 = 1.5GB) prevents flip-flop between ramdisk and SSD.
|
||||
HOST1_RAMDISK_WARN_GB=8.5 # flip to SSD when ramdisk usage reaches this
|
||||
HOST1_RAMDISK_LOW_GB=7 # flip back to ramdisk when usage drops to this
|
||||
|
||||
# SSD fallback path — where transcodes land when ramdisk exceeds HOST1_RAMDISK_WARN_GB.
|
||||
# Must be on cache pool — array disks too slow for active transcode writes.
|
||||
HOST1_TRANSCODE_SSD="/mnt/cache/Temp_Storage/Emby/Transcodes/"
|
||||
|
||||
# Media servers sharing the ramdisk transcode space on HOST1.
|
||||
# Format: "ContainerName|URL|APIKey|Type" — Type: emby | jellyfin | plex
|
||||
# Entries with placeholder API keys are skipped automatically.
|
||||
# ⚠️ Tdarr does NOT belong here — keep Tdarr on SSD, not ramdisk.
|
||||
HOST1_TRANSCODE_SERVERS=(
|
||||
"${HOST1_EMBY_CONTAINER}|${HOST1_EMBY_URL}|${HOST1_EMBY_API_KEY}|emby"
|
||||
"${HOST1_JELLYFIN_CONTAINER}|${HOST1_JELLYFIN_URL}|${HOST1_JELLYFIN_API_KEY}|jellyfin"
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── ARR STACK ─────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Used by arr cleanup scripts and arrs_failed_stalled_recovery.sh.
|
||||
# detect_hosts() selects HOST1 vars when running on HOST1.
|
||||
#
|
||||
# PATH MAPS — container path → host path translation.
|
||||
# Arr stores file paths using container-internal paths — scripts need host paths to scan.
|
||||
# Add one entry per root folder in arr Settings → Media Management → Root Folders.
|
||||
|
||||
# ━━━ Downloaders ━━━
|
||||
# Used by downloaders_reset.sh — runs every 30min via CRITICAL_MAINTENANCE_SCRIPTS.
|
||||
# Clears stuck states, purges old history, prepares each client for a clean cycle.
|
||||
|
||||
# slskd — clears stuck searches, dead transfers, purges expired failed imports.
|
||||
# SLSKD_FAILED_IMPORTS_DIR: where Soularr moves albums Lidarr rejected.
|
||||
HOST1_SLSKD_URL="http://localhost:8980"
|
||||
HOST1_SLSKD_API_KEY="4bF9kL2mNpQrT7vWxYz1A3dEgHjKoRsU"
|
||||
HOST1_SLSKD_FAILED_IMPORTS_DIR="/mnt/user/Temp_Storage/Slskd/completed/failed_imports"
|
||||
|
||||
# SABnzbd
|
||||
HOST1_SABNZBD_URL="http://localhost:8180"
|
||||
HOST1_SABNZBD_API_KEY="8bfefe41d83b4d50883e32859b55ca9a"
|
||||
|
||||
# qBittorrent — deleteFiles=false removes torrent from qBit but leaves files on disk.
|
||||
# Radarr/Sonarr manage actual files independently.
|
||||
HOST1_QBIT_URL="http://localhost:8080"
|
||||
HOST1_QBIT_USERNAME="root"
|
||||
HOST1_QBIT_PASSWORD="Stay0utD!ck"
|
||||
|
||||
# ━━━ Lidarr — HOST1 only ━━━
|
||||
# HOST2 does not run Lidarr — HOST1_LIDARR_RECOVERY flag handles the exit cleanly.
|
||||
HOST1_LIDARR_URL="http://localhost:8686"
|
||||
HOST1_LIDARR_API_KEY="b2977e71ef074bc0a0529d9fcce3b2dc"
|
||||
HOST1_LIDARR_MUSIC_ROOT="/mnt/user/Music-New"
|
||||
HOST1_FANART_API_KEY="Yd7147a43b692df0b364b94dc47efb81"
|
||||
HOST1_LASTFM_API_KEY="be6dc169c33ae263e690c30d18b7491d"
|
||||
|
||||
declare -A HOST1_LIDARR_PATH_MAP=(
|
||||
["/ext-music"]="/mnt/user/Music-New"
|
||||
)
|
||||
|
||||
# ━━━ Sonarr ━━━
|
||||
HOST1_SONARR_URL="http://localhost:8989"
|
||||
HOST1_SONARR_API_KEY="130decd3db5b4c25afad64864cd03f9f"
|
||||
HOST1_SONARR_TV_ROOT="/mnt/user/Tv_Shows"
|
||||
|
||||
declare -A HOST1_SONARR_PATH_MAP=(
|
||||
["/tv"]="/mnt/user/Tv_Shows"
|
||||
["/ext-standup-comedy/series"]="/mnt/user/stand-up_comedy/series"
|
||||
["/kids tv"]="/mnt/user/Kids_Tv_Shows"
|
||||
["/ext-anime-shows"]="/mnt/user/Anime_Shows-Old"
|
||||
)
|
||||
|
||||
# ━━━ Radarr ━━━
|
||||
HOST1_RADARR_URL="http://localhost:7878"
|
||||
HOST1_RADARR_API_KEY="d43a3ec6cf1549edb4af0cc63f98b2a9"
|
||||
HOST1_TMDB_API_KEY="3dac5e2e49b5540472d2eafec4f01260"
|
||||
HOST1_RADARR_MOVIES_ROOT="/mnt/user/Movies"
|
||||
|
||||
declare -A HOST1_RADARR_PATH_MAP=(
|
||||
["/movies"]="/mnt/user/Movies"
|
||||
["/kids movies"]="/mnt/user/Kids_Movies"
|
||||
["/ext-stand-up-comedy/specials"]="/mnt/user/stand-up_comedy/specials"
|
||||
["/anime-movies"]="/mnt/user/Anime_Movies-Old"
|
||||
)
|
||||
|
||||
# ━━━ Arr Recovery Toggles ━━━
|
||||
# false = skip that arr on this host — exits cleanly without error
|
||||
HOST1_SONARR_RECOVERY=true
|
||||
HOST1_RADARR_RECOVERY=true
|
||||
HOST1_LIDARR_RECOVERY=true # HOST1 only — exits cleanly on HOST2
|
||||
|
||||
# ==============================================================================================
|
||||
# ── SYSTEM WATCHDOG ───────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Per-host check toggles and NIC config for system_watchdog.sh.
|
||||
# Aliased by detect_hosts() — script uses unprefixed SYS_WATCHDOG_* names.
|
||||
# HOST1: TR1950X 128GB — full media server, active transcoding, ZFS cache pools.
|
||||
#
|
||||
# Three-tier response — all critical checks enabled by default on HOST1:
|
||||
# Tier 1 (bypass strikes, reboot now): docker daemon, rootfs full, kernel oops, FD, /boot
|
||||
# Tier 2 (bypass strikes with OOM): RAM critical + OOM kills in cycle
|
||||
# Tier 3 (standard strike system): everything else
|
||||
#
|
||||
# RAM tiers, OOM limits, and reboot loop settings in master.conf System Watchdog section.
|
||||
|
||||
# ━━━ Primary NIC ━━━
|
||||
# Network interface for NIC state check — verify with: ip link show | grep "^[0-9]"
|
||||
# Common values: eth0, bond0, br0, eno1
|
||||
HOST1_SYS_WATCHDOG_NIC="eth0"
|
||||
|
||||
# ━━━ Tier 1 — Critical Checks ━━━
|
||||
# These bypass the strike system — a single hit triggers immediate reboot.
|
||||
# Disabling any of these is not recommended — they protect against acute system failure.
|
||||
|
||||
# Docker daemon unresponsive → try restart, reboot if restart fails.
|
||||
# Without a working daemon docker_watchdog.sh is blind and containers cannot be managed.
|
||||
HOST1_SYS_WATCHDOG_CHECK_DOCKER_DAEMON=true
|
||||
|
||||
# rootfs at critical threshold (ROOTFS_CRITICAL_PCT=99) → reboot immediately.
|
||||
# At 99% rootfs writes fail silently — logs stop, Docker errors out, SSH may stop working.
|
||||
# Standard 95% threshold still uses strike system — only 99%+ is critical tier.
|
||||
HOST1_SYS_WATCHDOG_CHECK_ROOTFS=true
|
||||
|
||||
# Kernel BUG/Oops in dmesg delta since last cycle → reboot immediately.
|
||||
# A kernel oops means the kernel ran with a corrupted state — stability is not guaranteed.
|
||||
HOST1_SYS_WATCHDOG_CHECK_KERNEL_OOPS=true
|
||||
|
||||
# File descriptor exhaustion at FD_CRITICAL_PCT (95%) → reboot immediately.
|
||||
# At 95% FD: new connections fail, Docker can't spawn processes, SSH drops.
|
||||
HOST1_SYS_WATCHDOG_CHECK_FD=true
|
||||
|
||||
# /boot read-only detected → reboot immediately.
|
||||
# Unexpected read-only /boot means state files and config writes are silently failing.
|
||||
# Fallback state, watchdog reboot log, and lock files all go stale silently.
|
||||
HOST1_SYS_WATCHDOG_CHECK_BOOT=true
|
||||
|
||||
# ━━━ Tier 2 — Urgent OOM Check ━━━
|
||||
# Bypass strikes when RAM is critically low AND OOM kill rate confirms active crisis.
|
||||
# Both must be enabled for Tier 2 bypass to function — disable either to always use strikes.
|
||||
|
||||
# Track kernel OOM kills each cycle via /proc/vmstat oom_kill delta.
|
||||
# Also provides diagnostic context in reboot messages (which processes were killed).
|
||||
HOST1_SYS_WATCHDOG_CHECK_OOM=true
|
||||
|
||||
# Free RAM check — required for both Tier 2 bypass and RAM tier logic.
|
||||
# Tiers: MEM_WARN_GB(10) → notify | MEM_SHUTDOWN_GB(6) → stop containers | MEM_GB(4) → strikes
|
||||
HOST1_SYS_WATCHDOG_CHECK_RAM=true
|
||||
|
||||
# ━━━ Tier 3 — Standard Checks (strike system) ━━━
|
||||
# Each check must fail SYS_WATCHDOG_STRIKE_LIMIT consecutive cycles before action is taken.
|
||||
# Single spikes are ignored — sustained problems trigger reboot.
|
||||
|
||||
# /var/log filesystem usage above SYS_WATCHDOG_LOG_PCT.
|
||||
# Log spam (Docker log storms, syslog loops) fills rootfs — indicates something broken.
|
||||
HOST1_SYS_WATCHDOG_CHECK_LOG=true
|
||||
|
||||
# ZFS ARC memory pinned above SYS_WATCHDOG_ARC_PINNED_PCT after cache drop.
|
||||
# Enabled on HOST1 — ZFS cache pools actively used. Disable on hosts without ZFS.
|
||||
HOST1_SYS_WATCHDOG_CHECK_ARC=true
|
||||
|
||||
# CPU temperature above SYS_WATCHDOG_CPU_TEMP_MAX (95°C).
|
||||
# Sustained high temp causes kernel throttling or panic. Requires lm-sensors.
|
||||
HOST1_SYS_WATCHDOG_CHECK_CPU_TEMP=true
|
||||
|
||||
# Load average above SYS_WATCHDOG_LOAD_MULTIPLIER × core count.
|
||||
# DISABLED on HOST1 — Tdarr and Emby cause legitimate sustained load spikes during encoding.
|
||||
# Enable on idle servers or adjust SYS_WATCHDOG_LOAD_MULTIPLIER if load is always high.
|
||||
HOST1_SYS_WATCHDOG_CHECK_LOAD=false
|
||||
|
||||
# Zombie process count above SYS_WATCHDOG_ZOMBIE_LIMIT (50).
|
||||
# Large zombie counts indicate serious process management failure — something is stuck.
|
||||
HOST1_SYS_WATCHDOG_CHECK_ZOMBIES=true
|
||||
|
||||
# Check docker_watchdog.sh persistent skip list — required containers on skip list.
|
||||
# Cross-watchdog coordination: if docker_watchdog gave up, system_watchdog escalates.
|
||||
# ENABLED — HOST1 fully built and operational, skip list is meaningful.
|
||||
HOST1_SYS_WATCHDOG_CHECK_CONTAINERS=true
|
||||
|
||||
# /tmp filesystem usage above SYS_WATCHDOG_TMP_PCT with auto-clear attempt.
|
||||
# Script tries to clear aged /tmp files first — only strikes if clear fails.
|
||||
# Lock files, rsync temp files, and Docker ops use /tmp — 100% means lock failures.
|
||||
HOST1_SYS_WATCHDOG_CHECK_TMP=true
|
||||
|
||||
# Array disk error count delta in /proc/mdstat — accumulating errors = disk failing now.
|
||||
# Triggers on SYS_WATCHDOG_MDSTAT_ERROR_LIMIT new errors in one cycle.
|
||||
HOST1_SYS_WATCHDOG_CHECK_MDSTAT=true
|
||||
|
||||
# Primary NIC operstate — detects NIC going down (physical or driver failure).
|
||||
# Uses HOST1_SYS_WATCHDOG_NIC above. Strike system — brief flaps don't trigger reboot.
|
||||
HOST1_SYS_WATCHDOG_CHECK_NETWORK=true
|
||||
|
||||
# sshd running check — attempts restart before escalating.
|
||||
# sshd down = no remote access. Script tries rc.sshd start, notifies, strikes on failure.
|
||||
HOST1_SYS_WATCHDOG_CHECK_SSHD=true
|
||||
|
||||
# Runaway process detection — single process above SYS_WATCHDOG_RUNAWAY_CPU_PCT sustained.
|
||||
# DISABLED — Tdarr encoding and Emby transcoding legitimately peg CPU for extended periods.
|
||||
# Enable only if HOST1 has no CPU-intensive workloads.
|
||||
HOST1_SYS_WATCHDOG_CHECK_RUNAWAY=false
|
||||
|
||||
# ==============================================================================================
|
||||
# ── RESOURCE MANAGER ──────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Containers to manage under pressure — see master.conf RW_CRITICAL_CONTAINERS for exclusions.
|
||||
|
||||
# docker pause at medium pressure (RAM < RW_RAM_MEDIUM_GB or load > medium threshold)
|
||||
# Suspended in-place — instant to pause/unpause, no state lost, no restart delay.
|
||||
HOST1_RW_PAUSE_CONTAINERS=(
|
||||
"Huntarr" # arr search automation — safe to suspend
|
||||
"Cleanuparr" # download cleanup — safe to suspend
|
||||
"Healarr" # arr health checks — safe to suspend
|
||||
"Soularr" # Slskd automation — background only
|
||||
"ChannelTube" # YouTube archiver — background only
|
||||
"Pinchflat" # YouTube archiver — background only
|
||||
)
|
||||
|
||||
# docker stop at hard pressure (RAM < RW_RAM_HARD_GB)
|
||||
# Full stop — these are optional/heavy services that free significant RAM when stopped.
|
||||
# resource_watchdog.sh restarts them when pressure fully clears (RAM >= RW_RAM_RECOVER_GB).
|
||||
HOST1_RW_STOP_CONTAINERS=(
|
||||
"LocalAI" # GPU/CPU heavy — largest RAM consumer when idle
|
||||
"7DaysToDie" # game server — optional
|
||||
"V-Rising" # game server — optional
|
||||
"Code-Server" # IDE — not needed during pressure events
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── AUTH STACK ────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Credentials for the Varaverk Auth Stack page (NPM, lldap, Authelia).
|
||||
|
||||
# ━━━ NginxProxyManager ━━━
|
||||
HOST1_NPM_URL="http://localhost:7818"
|
||||
HOST1_NPM_USER="failedproxy@gmail.com"
|
||||
HOST1_NPM_PASS="183134\$eanHess"
|
||||
|
||||
# ━━━ lldap ━━━
|
||||
HOST1_LLDAP_URL="http://localhost:17170"
|
||||
HOST1_LLDAP_USER="admin"
|
||||
HOST1_LLDAP_PASS="183134\$eanHess"
|
||||
|
||||
# ━━━ Authelia ━━━
|
||||
HOST1_AUTHELIA_CONFIG="/mnt/user/appdata-Fallback/Critical-Data/Authelia/configuration.yml"
|
||||
HOST1_AUTHELIA_CONTAINER="Authelia"
|
||||
|
||||
# ==============================================================================================
|
||||
# ──────────────────────── End Of HOST1 Variables ──────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
HOST1_UNRAID_API_KEY="1825c3a2e03ea5089974f4da2e171aa2d5907a1dea23cc479c33e492c8ff4dbb"
|
||||
@@ -0,0 +1,788 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ========================== HOST1 CONFIGURATION — unRAID-Gmer4Lfe ============================
|
||||
# ==============================================================================================
|
||||
# HOST1-specific variables — credentials, container names, share paths, failover lists.
|
||||
# Sourced after master.conf — values here extend shared profile arrays and add HOST1-specific
|
||||
# identity, credentials, and container configuration.
|
||||
#
|
||||
# Sparse checkout (git) ensures HOST2 never receives this file.
|
||||
# HOST2 never sees HOST1 credentials — clean separation at the file level.
|
||||
#
|
||||
# DO NOT put shared config here — thresholds, toggles, profiles belong in master.conf.
|
||||
# DO NOT put HOST2 variables here — they belong in host2.conf.
|
||||
#
|
||||
# ── INDEX ─────────────────────────────────────────────────────────────────────────────────────
|
||||
#
|
||||
# ── IDENTITY & CONNECTIVITY ────────────────────────────────────────────────────────────────
|
||||
# IDENTITY hostname, SSH key
|
||||
# EMBY container name, URL, API key
|
||||
# NOTIFICATIONS Discord webhook
|
||||
# PARTNERSHIP auth containers, backup paths
|
||||
#
|
||||
# ── RSYNC ──────────────────────────────────────────────────────────────────────────────────
|
||||
# DAILY SYNC SHARES media shares HOST1 owns and pushes to HOST2
|
||||
# WEEKLY SYNC SHARES appdata shares synced weekly (Sunday 2:30am)
|
||||
# CRITICAL SYNC SHARES appdata shares synced every 30 minutes
|
||||
# BACKUP VERIFY shares for checksum verification against remote
|
||||
# HOST1 RSYNC PROFILE host1-appdata profile for HOST1-specific appdata syncs
|
||||
#
|
||||
# ── DOCKER ─────────────────────────────────────────────────────────────────────────────────
|
||||
# DOCKER DAILY RESTART containers restarted daily
|
||||
# DOCKER WEEKLY RESTART containers restarted weekly
|
||||
# DOCKER WATCHDOG memory limits, health URLs, required containers, ignore list
|
||||
# DOCKER NETWORK CONNECT networks and containers for docker_network_connect.sh
|
||||
#
|
||||
# ── FALLBACK ───────────────────────────────────────────────────────────────────────────────
|
||||
# DDNS DDNS containers managed by HOST1
|
||||
# INTERNET LOSS containers stopped when internet is lost
|
||||
# FALLBACK TIERS what HOST1 runs for HOST2 per tier
|
||||
# TIER DELAYS how long HOST1 must be down before each tier activates on HOST2
|
||||
# RSYNC WRITEBACK HOST1 appdata synced back on handback
|
||||
#
|
||||
# ── MEDIA ──────────────────────────────────────────────────────────────────────────────────
|
||||
# MEDIA PERMISSIONS share list for media_shares_permissions.sh
|
||||
# MEDIA CLEANER folder lists for media_cleaner.sh
|
||||
#
|
||||
# ── MONITORS ───────────────────────────────────────────────────────────────────────────────
|
||||
# CERTIFICATE MONITOR domains checked for SSL expiry
|
||||
# SMART HEALTH drives to skip in SMART monitoring
|
||||
# ZFS REPORT pools to exclude from ZFS health report
|
||||
#
|
||||
# ── TRANSCODES ─────────────────────────────────────────────────────────────────────────────
|
||||
# TRANSCODES ramdisk size, thresholds, SSD path, server array
|
||||
#
|
||||
# ── ARR STACK ──────────────────────────────────────────────────────────────────────────────
|
||||
# DOWNLOADERS slskd, SABnzbd, qBittorrent credentials and URLs
|
||||
# LIDARR URL, API key, path map
|
||||
# SONARR URL, API key, path map
|
||||
# RADARR URL, API key, path map
|
||||
# ARR RECOVERY per-arr recovery toggles
|
||||
#
|
||||
# ── SYSTEM WATCHDOG ────────────────────────────────────────────────────────────────────────
|
||||
# SYSTEM WATCHDOG per-host check toggles and NIC configuration
|
||||
#
|
||||
# ── RESOURCE MANAGER ───────────────────────────────────────────────────────────────────────
|
||||
# RESOURCE MANAGER containers paused/stopped under memory pressure
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
# ==============================================================================================
|
||||
# ── IDENTITY & CONNECTIVITY ───────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Identity ━━━
|
||||
# HOST1 hostname lives in master.conf (not a credential — safe for all servers).
|
||||
# SSH key used for all server-to-server operations — rsync, failover container commands.
|
||||
# Must be in /root/.ssh/ and authorised in HOST2's /root/.ssh/authorized_keys.
|
||||
HOST1_SSH_KEY="/root/.ssh/gmer4lfe_rsync_automation"
|
||||
HOST1_OWNER="gmer4lfe"
|
||||
HOST1_OWNER_EMAIL="gmer4lfe@gmail.com"
|
||||
|
||||
# ━━━ Emby ━━━
|
||||
# Referenced by transcode_manager.sh, emby_session_report.sh, emby_database_repair.sh,
|
||||
# weekly_sync_maintenance.sh, and HOST1_TRANSCODE_SERVERS below.
|
||||
# API key: Emby Dashboard → API Keys → + New Key
|
||||
HOST1_EMBY_CONTAINER="Emby"
|
||||
HOST1_EMBY_URL="http://localhost:8096"
|
||||
HOST1_EMBY_API_KEY="0c27448d93a7431f9ac63569f7655829"
|
||||
|
||||
# ━━━ Jellyfin ━━━
|
||||
# API key: Jellyfin Dashboard → Administration → API Keys → + New Key
|
||||
HOST1_JELLYFIN_CONTAINER="Jellyfin"
|
||||
HOST1_JELLYFIN_URL="http://localhost:8095"
|
||||
HOST1_JELLYFIN_API_KEY="4e820e7df74c4933acec212b1996314e"
|
||||
|
||||
# ━━━ Gitea ━━━
|
||||
# Personal access token for gitea_ssh_setup.sh — registers this server's SSH public key
|
||||
# with Gitea so git operations use key auth instead of passwords.
|
||||
# Create in Gitea: Settings → Applications → Generate Token → scope: write:user
|
||||
HOST1_GITEA_API_TOKEN=""
|
||||
|
||||
# ━━━ Notifications ━━━
|
||||
# Discord webhook — leave blank to disable.
|
||||
# Per-host so HOST1 and HOST2 can post to different channels or only one server notifies.
|
||||
HOST1_DISCORD_WEBHOOK=""
|
||||
|
||||
# ━━━ Partnership ━━━
|
||||
# HOST1 is always the owner (source of truth) unless --transfer has been run.
|
||||
# See README-Partnership.md and master.conf PARTNERSHIP section for full lifecycle docs.
|
||||
|
||||
# Auth containers reconfigured on onboard/offboard.
|
||||
# Format: "ContainerName|WebUIPort"
|
||||
# On onboard → WebUI pointed at owner's Tailscale IP (mirror clicks NPM, gets owner's auth)
|
||||
# On offboard → WebUI pointed back at localhost
|
||||
HOST1_PARTNERSHIP_AUTH_WEBUIS=(
|
||||
"NginxProxyManager|81"
|
||||
"Lldap-Gmer4Lfe|17170"
|
||||
"Authelia|9091"
|
||||
"Authelia-Secondary|9092"
|
||||
)
|
||||
|
||||
# XML templates (from this server's templates-user/) pushed to mirror during onboard.
|
||||
# These become the mirror's active auth stack, backed by the rsync-synced appdata.
|
||||
# Update filename if Lldap is renamed to drop the host suffix.
|
||||
HOST1_PARTNERSHIP_AUTH_STACK=(
|
||||
# Dependencies first — Mariadb/Redis must be healthy before Authelia starts
|
||||
"my-Mariadb-Authelia.xml"
|
||||
"my-Mariadb-Authelia-Secondary.xml"
|
||||
"my-Redis-Authelia.xml"
|
||||
"my-Redis-Authelia-Secondary.xml"
|
||||
# Auth apps — deployed after their deps are confirmed healthy
|
||||
"my-Authelia.xml"
|
||||
"my-Authelia-Secondary.xml"
|
||||
"my-NginxProxyManager.xml"
|
||||
"my-Lldap-Gmer4Lfe.xml"
|
||||
# Source of truth — must be available on HOST2 independently of the auth stack
|
||||
"my-Gitea.xml"
|
||||
)
|
||||
|
||||
# XML templates pushed to mirror for the arr stack during onboard.
|
||||
# Deps (e.g. databases) first if any — same ordering rule as auth stack.
|
||||
HOST1_PARTNERSHIP_ARR_STACK=(
|
||||
# "my-Sonarr.xml"
|
||||
# "my-Radarr.xml"
|
||||
# "my-Lidarr.xml"
|
||||
# "my-Prowlarr.xml"
|
||||
# "my-Bazarr.xml"
|
||||
)
|
||||
|
||||
# Paths HOST2 should collect during the grace window after offboard.
|
||||
# Notified on offboard — no auto-deletion, HOST2 must collect manually within PARTNERSHIP_GRACE_HOURS.
|
||||
HOST1_PARTNERSHIP_MIRROR_BACKUPS=(
|
||||
# "/mnt/user/appdata-Fallback/Jayred365-Emby"
|
||||
)
|
||||
|
||||
# Containers parked on this server when partnership is active.
|
||||
# Stopped on onboard (owner deploys its stack instead), restarted on offboard.
|
||||
HOST1_PARTNERSHIP_OWN_CONTAINERS=(
|
||||
# "Emby"
|
||||
# "NginxProxyManager"
|
||||
)
|
||||
|
||||
# Emby admin provisioning — toggle is owner-only, credentials are per-host.
|
||||
# Owner enables/disables the feature. Each host sets the account they want on the shared Emby.
|
||||
# On onboard: owner reads mirror's HOST*_PARTNERSHIP_EMBY_ADMIN_* and creates that account.
|
||||
# On offboard: account is deleted. Username collision → onboard exits with error.
|
||||
HOST1_PARTNERSHIP_PROVISION_EMBY_ADMIN=false # owner controls whether Emby is shared
|
||||
HOST1_PARTNERSHIP_EMBY_PORT=8096
|
||||
HOST1_PARTNERSHIP_EMBY_ADMIN_USER="" # this server's desired Emby username
|
||||
HOST1_PARTNERSHIP_EMBY_ADMIN_PASS="" # this server's desired Emby password
|
||||
|
||||
# ==============================================================================================
|
||||
# ── RSYNC ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Daily Sync Shares ━━━
|
||||
# Shares HOST1 pushes to all other nodes every night (1am via daily_sync_maintenance.sh).
|
||||
# Mesh model: every node pushes every media share — no ownership, no mirrors.
|
||||
# arr_sync ensures all arr libraries converge (union). rsync spreads files (additive, no --delete).
|
||||
# arr_cleanup removes true orphans based on local arr state.
|
||||
# Any node can download content to any share — it propagates to all nodes on the next cycle.
|
||||
# Nextcloud is intentionally one-directional (HOST1→HOST2 offsite backup — not arr-managed).
|
||||
# Uses DEFAULT_RSYNC_OPTS from master.conf — no profile needed.
|
||||
# For shares needing container stops or custom options — add a profile in master.conf.
|
||||
HOST1_DAILY_SYNC_SHARES=(
|
||||
/mnt/user/Books
|
||||
/mnt/user/Intros
|
||||
/mnt/user/Kids_Movies
|
||||
/mnt/user/Kids_Tv_Shows
|
||||
/mnt/user/Movies
|
||||
/mnt/user/Music
|
||||
/mnt/user/Music_Videos
|
||||
/mnt/user/Nextcloud
|
||||
/mnt/user/stand-up_comedy
|
||||
/mnt/user/Sports
|
||||
# /mnt/user/Tv_Shows
|
||||
/mnt/user/Anime_Shows-Old
|
||||
/mnt/user/Anime_Movies-Old
|
||||
/mnt/user/Anime_Movies
|
||||
/mnt/user/Anime_Shows
|
||||
)
|
||||
|
||||
# Personal encrypted shares — synced for offsite backup, independent of media shares.
|
||||
# ZFS encrypted at dataset level — remote receives encrypted blocks, cannot read content.
|
||||
# See README-Rsync_Setup.md for ZFS encryption setup before uncommenting.
|
||||
HOST1_PERSONAL_SHARES=(
|
||||
# /mnt/user/HOST1-Personal # uncomment after creating encrypted dataset
|
||||
)
|
||||
|
||||
# ━━━ Weekly Sync Shares ━━━
|
||||
# Appdata shares synced during the weekly maintenance window (Sunday 2:30am).
|
||||
# Containers stopped both sides before sync — full clean state guaranteed.
|
||||
# Profiles drive container stops, excludes, and options — configured in master.conf RSYNC section.
|
||||
# Order matters — Emby first (larger transfer), then Critical-Data (auth stack).
|
||||
HOST1_WEEKLY_SYNC_SHARES=(
|
||||
"/mnt/user/Media_Server/Emby" # emby profile — full clean mirror
|
||||
"/mnt/user/appdata-Fallback/Critical-Data" # critical-data profile — auth stack
|
||||
)
|
||||
|
||||
# ━━━ Intermediate Sync Shares ━━━
|
||||
# Shares synced every 4 hours by intermediate_sync_maintenance.sh.
|
||||
# Uses DEFAULT_RSYNC_OPTS (no --delete) — for sub-daily propagation of metadata or watch state.
|
||||
# Full media share sync stays in the daily window. Leave empty to skip mid-day rsync entirely.
|
||||
HOST1_INTERMEDIATE_SYNC_SHARES=(
|
||||
# Add shares here to enable mid-day rsync
|
||||
# Example: "/mnt/user/Emby_Metadata"
|
||||
)
|
||||
|
||||
# ━━━ Critical Sync Shares ━━━
|
||||
# Appdata shares synced every 30 minutes by critical_sync_maintenance.sh.
|
||||
# Format: "/path/to/share" or "/path/to/share|profile-name"
|
||||
# Order matters — Critical-Data first (auth stack), then Emby dirty sync.
|
||||
HOST1_CRITICAL_SYNC_SHARES=(
|
||||
"/mnt/user/appdata-Fallback/Critical-Data|critical-fallback" # auth dirty sync — stays running
|
||||
"/mnt/user/Media_Server/Emby|emby-fallback" # Emby dirty sync — stays running
|
||||
)
|
||||
|
||||
# ━━━ Backup Verify ━━━
|
||||
# Shares verified by backup_verify.sh — random file checksum comparison against remote.
|
||||
# Leave empty to use HOST1_DAILY_SYNC_SHARES automatically.
|
||||
# Sample size and minimum file size defined in master.conf.
|
||||
HOST1_BACKUP_VERIFY_SHARES=(
|
||||
# leave empty to use HOST1_DAILY_SYNC_SHARES automatically
|
||||
)
|
||||
|
||||
# ━━━ HOST1 Rsync Profile — host1-appdata ━━━
|
||||
# HOST1-specific appdata sync profile — extends the shared PROFILE_* arrays in master.conf.
|
||||
# Use for appdata unique to HOST1 (Organizrv2, VaultWarden, UptimeKuma etc.)
|
||||
# Shared appdata (auth stack, Emby) use dedicated profiles defined in master.conf.
|
||||
# Run manually: bash Rsync/rsync.sh /mnt/user/appdata-Fallback/HOST1-Appdata --profile=host1-appdata
|
||||
PROFILE_RSYNC_OPTS[host1-appdata]="-av --info=progress2 --bwlimit=${PROFILE_BW_LIMIT[host1-appdata]:-8000}"
|
||||
PROFILE_BW_LIMIT[host1-appdata]=8000
|
||||
PROFILE_RETRY_COUNT[host1-appdata]=3
|
||||
PROFILE_SLEEP[host1-appdata]=300
|
||||
PROFILE_CRITICAL_CONTAINER_NAMES[host1-appdata]="Organizrv2-Gmer4Lfe UptimeKuma-Gmer4Lfe VaultWarden-Gmer4Lfe"
|
||||
PROFILE_DELAYED_CONTAINERS[host1-appdata]=""
|
||||
PROFILE_CONTAINER_DELAY[host1-appdata]=5
|
||||
PROFILE_EXCLUDE_DIRS[host1-appdata]="logs *.tmp"
|
||||
|
||||
# ==============================================================================================
|
||||
# ── DOCKER ────────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Docker Daily Restart ━━━
|
||||
# Containers restarted every day via DAILY_MAINTENANCE_SCRIPTS.
|
||||
# Dispatcharr degrades over time without restart — daily is intentional, not just housekeeping.
|
||||
# Order matters — auth stack first, then media services.
|
||||
HOST1_DAILY_RESTART_CONTAINERS=(
|
||||
"NginxProxyManager"
|
||||
"Lldap-Gmer4Lfe"
|
||||
"Authelia"
|
||||
"Authelia-Secondary"
|
||||
"Dispatcharr-Iptv-Users"
|
||||
"Dispatcharr" # Live TV scheduler — degrades without daily restart
|
||||
"Dispatcharr-Basic"
|
||||
"ErsatzTV-Emby"
|
||||
"Slskd" # Soulseek connection drops after extended uptime; restart refreshes share index
|
||||
)
|
||||
|
||||
# ━━━ Docker Weekly Restart ━━━
|
||||
# Less critical services restarted weekly via WEEKLY_MAINTENANCE_SCRIPTS (Sunday 2:30am).
|
||||
# Containers already stopped for weekly sync — restart adds zero extra downtime.
|
||||
HOST1_WEEKLY_RESTART_CONTAINERS=(
|
||||
"NextCloud"
|
||||
"Organizrv2-Gmer4Lfe"
|
||||
"AdGuard-Home"
|
||||
"Immich-Gmer4Lfe"
|
||||
)
|
||||
|
||||
# ━━━ Docker Watchdog ━━━
|
||||
# Per-HOST1 container configuration for docker_watchdog.sh.
|
||||
# Shared thresholds and toggles live in master.conf.
|
||||
|
||||
# Memory hard limits in MB — immediate restart if exceeded.
|
||||
# Set at "container is clearly broken" not "container is busy".
|
||||
# 20GB=20480 18GB=18432 16GB=16384 12GB=12288 8GB=8192 4GB=4096 2GB=2048 1GB=1024
|
||||
declare -A HOST1_WATCHDOG_CONTAINERS=(
|
||||
["Emby"]=20480 # 20GB — large library + active transcodes
|
||||
["LidaTube"]=6144 # 6GB — memory leak over time
|
||||
["Tdarr"]=6144 # 6GB — encoding is memory intensive
|
||||
["Code-Server"]=1024 # 1GB — should never need more
|
||||
)
|
||||
|
||||
# HTTP health check URLs — checked every cycle, strike system before restart.
|
||||
# Only add containers with a meaningful web interface to check.
|
||||
declare -A HOST1_WATCHDOG_CONTAINER_URLS=(
|
||||
["Emby"]="http://localhost:8096"
|
||||
)
|
||||
|
||||
# Required containers — must always be running on HOST1.
|
||||
# Strike system before restart — repeated failures go on skip list, auto-clears on recovery.
|
||||
# Listed in dependency order — dependencies before dependents.
|
||||
HOST1_WATCHDOG_REQUIRED_CONTAINERS=(
|
||||
"NginxProxyManager"
|
||||
"Lldap-Gmer4Lfe"
|
||||
"Mariadb-Authelia"
|
||||
"Mariadb-Authelia-Secondary"
|
||||
"Redis-Authelia"
|
||||
"Redis-Authelia-Secondary"
|
||||
"Authelia"
|
||||
"Authelia-Secondary"
|
||||
)
|
||||
|
||||
# Containers to skip in Tier 2 global scan — legitimately stopped or frequently restarting.
|
||||
# Watchdog leaves these alone entirely — no restart attempts, no crash loop tracking.
|
||||
HOST1_WATCHDOG_SCAN_IGNORE=(
|
||||
"DashGate"
|
||||
"PIA-WG-Config-Generator"
|
||||
"Aperture"
|
||||
"Aperture-Kids"
|
||||
"pgvector-18-Apeture-Kids"
|
||||
"Pgvector18-Aperture"
|
||||
"emby-test" # broken test container (exit 127 — bad image)
|
||||
)
|
||||
|
||||
# Dependency ordering — skip restarting a container if its dependency is also down.
|
||||
# Prevents watchdog from restarting Authelia before Mariadb is back up.
|
||||
# SPACE-SEPARATED STRINGS — converted to array at runtime.
|
||||
declare -A HOST1_WATCHDOG_DEPENDENCIES=(
|
||||
["Authelia"]="Mariadb-Authelia Redis-Authelia"
|
||||
["Authelia-Secondary"]="Mariadb-Authelia Redis-Authelia-Secondary"
|
||||
["NextCloud"]="Postgres-NextCloud"
|
||||
)
|
||||
|
||||
# Per-container appdata growth suppress ceilings in MB.
|
||||
# ONLY needed in specific cases — growth rate detection covers all containers automatically.
|
||||
# Use this when a container legitimately has large stable data and you want to guarantee
|
||||
# it never triggers a false-positive growth alert. Growth warnings are suppressed while the
|
||||
# container's dir stays below this ceiling; above it, warnings resume as normal.
|
||||
# 50GB=51200 25GB=25600 20GB=20480 15GB=15360 10GB=10240 5GB=5120
|
||||
declare -A HOST1_WATCHDOG_APPDATA_SIZES=(
|
||||
["Tdarr"]="25600" # 25GB — transcode cache grows legitimately during active jobs
|
||||
["7dtd"]="20480" # 20GB — game server world data, expected to be large
|
||||
)
|
||||
|
||||
# ━━━ Network Watchdog ━━━
|
||||
# Host-specific connectivity config for Watchdogs/System/network_watchdog.sh.
|
||||
HOST1_NETWORK_WATCHDOG_DDNS_DOMAIN="gmer4lfe.com"
|
||||
HOST1_NETWORK_WATCHDOG_DDNS_CONTAINER="Gmer4Lfe.com"
|
||||
HOST1_NETWORK_WATCHDOG_NPM_URL="https://gmer4lfe.com"
|
||||
|
||||
# ━━━ Docker Network Connect ━━━
|
||||
# Containers connected to custom networks at array start by docker_network_connect.sh.
|
||||
# Networks created if they don't exist — idempotent, safe to re-run.
|
||||
HOST1_NETWORK_CONNECT_CONTAINERS=(
|
||||
"memcached"
|
||||
"Npm-CrowdSec"
|
||||
)
|
||||
|
||||
HOST1_NETWORK_CONNECT_NETWORKS=(
|
||||
"high-availability"
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── FALLBACK ──────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ DDNS ━━━
|
||||
# DDNS containers HOST1 manages — started/stopped by fallback.sh per DDNS absolute rules:
|
||||
# Internet loss → stop immediately
|
||||
# Failover → HOST2 starts HOST1's DDNS as Tier 1 (before any other containers)
|
||||
# Handback → stop HOST1's DDNS on HOST2 → rsync → start containers → start local DDNS last
|
||||
HOST1_DDNS_CONTAINERS=(
|
||||
"Gmer4Lfe.com"
|
||||
)
|
||||
|
||||
# ━━━ Internet Loss ━━━
|
||||
# Containers stopped immediately on HOST1 when internet connection is lost.
|
||||
# Prevents external-facing services from operating without connectivity.
|
||||
FALLBACK_HOST1_STOP_ON_NO_NET=(
|
||||
"Gmer4Lfe.com"
|
||||
)
|
||||
|
||||
# ━━━ Fallback Tiers — HOST1 Runs for HOST2 ━━━
|
||||
# Containers HOST1 starts when HOST2 goes down.
|
||||
# Tier 1 is always immediate — vital services cannot wait.
|
||||
# Higher tiers activate after HOST2_TIER*_DELAY minutes (set in host2.conf).
|
||||
FALLBACK_HOST1_COVERS_HOST2_TIER1=(
|
||||
"Gmer4Lfe.us"
|
||||
"VaultWarden-Jayred365"
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_COVERS_HOST2_TIER2=(
|
||||
# "container-placeholder"
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_COVERS_HOST2_TIER3=(
|
||||
# "container-placeholder"
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_COVERS_HOST2_TIER4=(
|
||||
# "container-placeholder"
|
||||
)
|
||||
|
||||
# ━━━ Tier Delays — HOST1's Containers on HOST2 ━━━
|
||||
# How long HOST1 must be down before each tier activates on HOST2 — in minutes.
|
||||
# Tier 1 is always immediate — no delay var needed.
|
||||
HOST1_TIER2_DELAY=240 # 4 hours — NextCloud, Immich
|
||||
HOST1_TIER3_DELAY=720 # 12 hours — secondary services
|
||||
HOST1_TIER4_DELAY=1440 # 24 hours — arrs + downloaders
|
||||
|
||||
# ━━━ Rsync Writeback — HOST1 Appdata Back on Handback ━━━
|
||||
# Syncs HOST1 appdata BACK to HOST1 when it comes back online after a failover.
|
||||
# Containers stopped before writeback — clean source, no competing writes.
|
||||
#
|
||||
# HOST1_TIER1_WRITEBACK_DELAY: short outages skip Tier 1 writeback — primary state
|
||||
# is more reliable than dirty sync data for brief outages.
|
||||
HOST1_TIER1_WRITEBACK_DELAY=60 # skip Emby writeback if outage under 1hr
|
||||
|
||||
# Tier 4 automatically syncs HOST1_DAILY_SYNC_SHARES — only list paths NOT in that array.
|
||||
FALLBACK_HOST1_WRITEBACK_TIER1=(
|
||||
"/mnt/user/Media_Server/Emby" # watch states built up during outage
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_WRITEBACK_TIER2=(
|
||||
"/mnt/user/appdata-Fallback/Important-Data" # NextCloud + Postgres — files added during outage
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_WRITEBACK_TIER3=(
|
||||
# "location-placeholder"
|
||||
)
|
||||
|
||||
FALLBACK_HOST1_WRITEBACK_TIER4=(
|
||||
"/mnt/user/appdata-Fallback/Arrs_Stack" # arr databases — downloads queued during outage
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── MEDIA ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Media Permissions ━━━
|
||||
# Shares that media_shares_permissions.sh applies PERMISSIONS_MODE and PERMISSIONS_OWNER to.
|
||||
# Runs first in DAILY_MAINTENANCE_SCRIPTS — arr cleanup depends on correct ownership.
|
||||
HOST1_MEDIA_PERMISSION_SHARES=(
|
||||
/mnt/user/Anime_Movies
|
||||
/mnt/user/Anime_Movies-Old
|
||||
/mnt/user/Anime_Shows
|
||||
/mnt/user/Anime_Shows-Old
|
||||
/mnt/user/appcache
|
||||
/mnt/user/Books
|
||||
/mnt/user/Downloads
|
||||
/mnt/user/Games
|
||||
/mnt/user/Intros
|
||||
/mnt/user/Kids_Movies
|
||||
/mnt/user/Kids_Tv_Shows
|
||||
/mnt/user/Movie_Recordings
|
||||
/mnt/user/Movies
|
||||
/mnt/user/Music
|
||||
/mnt/user/Music_Videos
|
||||
/mnt/user/Photo
|
||||
/mnt/user/Sports
|
||||
/mnt/user/stand-up_comedy
|
||||
/mnt/user/Tv_Recordings
|
||||
/mnt/user/Tv_Shows
|
||||
/mnt/user/YouTube
|
||||
)
|
||||
|
||||
# ━━━ Media Cleaner ━━━
|
||||
# Folder lists for media_cleaner.sh — two profiles: anime and media.
|
||||
# File patterns shared across all servers — defined in master.conf.
|
||||
# Called via DAILY_MAINTENANCE_SCRIPTS. Run manually: Media/media_cleaner.sh anime|media
|
||||
HOST1_ANIME_CLEAN_FOLDERS=(
|
||||
/mnt/user/Anime_Movies
|
||||
/mnt/user/Anime_Movies-Old
|
||||
/mnt/user/Anime_Shows
|
||||
/mnt/user/Anime_Shows-Old
|
||||
)
|
||||
|
||||
HOST1_MEDIA_CLEAN_FOLDERS=(
|
||||
/mnt/user/Kids_Movies
|
||||
/mnt/user/Kids_Tv_Shows
|
||||
/mnt/user/Movies
|
||||
/mnt/user/Music
|
||||
/mnt/user/Sports
|
||||
/mnt/user/stand-up_comedy
|
||||
/mnt/user/Tv_Shows
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── MONITORS ──────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Certificate Monitor ━━━
|
||||
# Domains checked via direct openssl connection — not relying on NPM's certificate state.
|
||||
# Checks the actual certificate served, not what NPM thinks it has.
|
||||
# Thresholds (CERT_WARN_DAYS, CERT_CRIT_DAYS) defined in master.conf.
|
||||
HOST1_CERT_MONITOR_DOMAINS=(
|
||||
"Gmer4Lfe.com"
|
||||
"Gmer4Lfe.us"
|
||||
)
|
||||
|
||||
# ━━━ SMART Health ━━━
|
||||
# Drives skipped in SMART attribute monitoring — hardware is server-specific.
|
||||
# Thresholds read from dynamix.cfg at runtime — fallbacks in master.conf.
|
||||
HOST1_SMART_IGNORE_DRIVES=(
|
||||
"sda" # boot USB — SMART not meaningful on flash drives
|
||||
)
|
||||
|
||||
# ━━━ ZFS Report ━━━
|
||||
# Pools excluded from the weekly ZFS health report — reduces noise from single-disk array pools.
|
||||
# These are individual array disks formatted as ZFS — converting to XFS over time via unBalance.
|
||||
# Pool health thresholds defined in master.conf.
|
||||
HOST1_ZFS_REPORT_IGNORE_POOLS=(
|
||||
"disk5"
|
||||
"disk6"
|
||||
"disk8"
|
||||
"disk9"
|
||||
"disk10"
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── TRANSCODES ────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# Ramdisk size ceiling — tmpfs only uses RAM actually needed, not the full size upfront.
|
||||
# Real-world: 9 streams peaked at ~5.5GB — 10G gives generous headroom on 128GB RAM.
|
||||
HOST1_RAMDISK_SIZE="10G"
|
||||
|
||||
# Usage thresholds — coupled to HOST1_RAMDISK_SIZE, adjust all three together if size changes.
|
||||
# Hysteresis gap (8.5 - 7 = 1.5GB) prevents flip-flop between ramdisk and SSD.
|
||||
HOST1_RAMDISK_WARN_GB=8.5 # flip to SSD when ramdisk usage reaches this
|
||||
HOST1_RAMDISK_LOW_GB=7 # flip back to ramdisk when usage drops to this
|
||||
|
||||
# SSD fallback path — where transcodes land when ramdisk exceeds HOST1_RAMDISK_WARN_GB.
|
||||
# Must be on cache pool — array disks too slow for active transcode writes.
|
||||
HOST1_TRANSCODE_SSD="/mnt/cache/Temp_Storage/Emby/Transcodes/"
|
||||
|
||||
# Media servers sharing the ramdisk transcode space on HOST1.
|
||||
# Format: "ContainerName|URL|APIKey|Type" — Type: emby | jellyfin | plex
|
||||
# Entries with placeholder API keys are skipped automatically.
|
||||
# ⚠️ Tdarr does NOT belong here — keep Tdarr on SSD, not ramdisk.
|
||||
HOST1_TRANSCODE_SERVERS=(
|
||||
"${HOST1_EMBY_CONTAINER}|${HOST1_EMBY_URL}|${HOST1_EMBY_API_KEY}|emby"
|
||||
"${HOST1_JELLYFIN_CONTAINER}|${HOST1_JELLYFIN_URL}|${HOST1_JELLYFIN_API_KEY}|jellyfin"
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── ARR STACK ─────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Used by arr cleanup scripts and arrs_failed_stalled_recovery.sh.
|
||||
# detect_hosts() selects HOST1 vars when running on HOST1.
|
||||
#
|
||||
# PATH MAPS — container path → host path translation.
|
||||
# Arr stores file paths using container-internal paths — scripts need host paths to scan.
|
||||
# Add one entry per root folder in arr Settings → Media Management → Root Folders.
|
||||
|
||||
# ━━━ Downloaders ━━━
|
||||
# Used by downloaders_reset.sh — runs every 30min via CRITICAL_MAINTENANCE_SCRIPTS.
|
||||
# Clears stuck states, purges old history, prepares each client for a clean cycle.
|
||||
|
||||
# slskd — clears stuck searches, dead transfers, purges expired failed imports.
|
||||
# SLSKD_FAILED_IMPORTS_DIR: where Soularr moves albums Lidarr rejected.
|
||||
HOST1_SLSKD_URL="http://localhost:8980"
|
||||
HOST1_SLSKD_API_KEY="4bF9kL2mNpQrT7vWxYz1A3dEgHjKoRsU"
|
||||
HOST1_SLSKD_FAILED_IMPORTS_DIR="/mnt/user/Temp_Storage/Slskd/completed/failed_imports"
|
||||
|
||||
# SABnzbd
|
||||
HOST1_SABNZBD_URL="http://localhost:8180"
|
||||
HOST1_SABNZBD_API_KEY="8bfefe41d83b4d50883e32859b55ca9a"
|
||||
|
||||
# qBittorrent — deleteFiles=false removes torrent from qBit but leaves files on disk.
|
||||
# Radarr/Sonarr manage actual files independently.
|
||||
HOST1_QBIT_URL="http://localhost:8080"
|
||||
HOST1_QBIT_USERNAME="root"
|
||||
HOST1_QBIT_PASSWORD="Stay0utD!ck"
|
||||
|
||||
# ━━━ Lidarr — HOST1 only ━━━
|
||||
# HOST2 does not run Lidarr — HOST1_LIDARR_RECOVERY flag handles the exit cleanly.
|
||||
HOST1_LIDARR_URL="http://localhost:8686"
|
||||
HOST1_LIDARR_API_KEY="b2977e71ef074bc0a0529d9fcce3b2dc"
|
||||
HOST1_LIDARR_MUSIC_ROOT="/mnt/user/Music-New"
|
||||
HOST1_FANART_API_KEY="Yd7147a43b692df0b364b94dc47efb81"
|
||||
HOST1_LASTFM_API_KEY="be6dc169c33ae263e690c30d18b7491d"
|
||||
|
||||
declare -A HOST1_LIDARR_PATH_MAP=(
|
||||
["/ext-music"]="/mnt/user/Music-New"
|
||||
)
|
||||
|
||||
# ━━━ Sonarr ━━━
|
||||
HOST1_SONARR_URL="http://localhost:8989"
|
||||
HOST1_SONARR_API_KEY="130decd3db5b4c25afad64864cd03f9f"
|
||||
HOST1_SONARR_TV_ROOT="/mnt/user/Tv_Shows"
|
||||
|
||||
declare -A HOST1_SONARR_PATH_MAP=(
|
||||
["/tv"]="/mnt/user/Tv_Shows"
|
||||
["/ext-standup-comedy"]="/mnt/user/stand-up_comedy/series"
|
||||
["/kids tv"]="/mnt/user/Kids_Tv_Shows"
|
||||
["/ext-anime-shows"]="/mnt/user/Anime_Shows-Old"
|
||||
)
|
||||
|
||||
# ━━━ Radarr ━━━
|
||||
HOST1_RADARR_URL="http://localhost:7878"
|
||||
HOST1_RADARR_API_KEY="d43a3ec6cf1549edb4af0cc63f98b2a9"
|
||||
HOST1_TMDB_API_KEY="3dac5e2e49b5540472d2eafec4f01260"
|
||||
HOST1_RADARR_MOVIES_ROOT="/mnt/user/Movies"
|
||||
|
||||
declare -A HOST1_RADARR_PATH_MAP=(
|
||||
["/movies"]="/mnt/user/Movies"
|
||||
["/kids movies"]="/mnt/user/Kids_Movies"
|
||||
["/ext-stand-up-comedy"]="/mnt/user/stand-up_comedy/specials"
|
||||
["/anime-movies"]="/mnt/user/Anime_Movies-Old"
|
||||
["/ext-anime-movies"]="/mnt/user/Anime_Movies-Old"
|
||||
)
|
||||
|
||||
# ━━━ Arr Recovery Toggles ━━━
|
||||
# false = skip that arr on this host — exits cleanly without error
|
||||
HOST1_SONARR_RECOVERY=true
|
||||
HOST1_RADARR_RECOVERY=true
|
||||
HOST1_LIDARR_RECOVERY=true # HOST1 only — exits cleanly on HOST2
|
||||
|
||||
# ==============================================================================================
|
||||
# ── SYSTEM WATCHDOG ───────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Per-host check toggles and NIC config for system_watchdog.sh.
|
||||
# Aliased by detect_hosts() — script uses unprefixed SYS_WATCHDOG_* names.
|
||||
# HOST1: TR1950X 128GB — full media server, active transcoding, ZFS cache pools.
|
||||
#
|
||||
# Three-tier response — all critical checks enabled by default on HOST1:
|
||||
# Tier 1 (bypass strikes, reboot now): docker daemon, rootfs full, kernel oops, FD, /boot
|
||||
# Tier 2 (bypass strikes with OOM): RAM critical + OOM kills in cycle
|
||||
# Tier 3 (standard strike system): everything else
|
||||
#
|
||||
# RAM tiers, OOM limits, and reboot loop settings in master.conf System Watchdog section.
|
||||
|
||||
# ━━━ Primary NIC ━━━
|
||||
# Network interface for NIC state check — verify with: ip link show | grep "^[0-9]"
|
||||
# Common values: eth0, bond0, br0, eno1
|
||||
HOST1_SYS_WATCHDOG_NIC="eth0"
|
||||
|
||||
# ━━━ Tier 1 — Critical Checks ━━━
|
||||
# These bypass the strike system — a single hit triggers immediate reboot.
|
||||
# Disabling any of these is not recommended — they protect against acute system failure.
|
||||
|
||||
# Docker daemon unresponsive → try restart, reboot if restart fails.
|
||||
# Without a working daemon docker_watchdog.sh is blind and containers cannot be managed.
|
||||
HOST1_SYS_WATCHDOG_CHECK_DOCKER_DAEMON=true
|
||||
|
||||
# rootfs at critical threshold (ROOTFS_CRITICAL_PCT=99) → reboot immediately.
|
||||
# At 99% rootfs writes fail silently — logs stop, Docker errors out, SSH may stop working.
|
||||
# Standard 95% threshold still uses strike system — only 99%+ is critical tier.
|
||||
HOST1_SYS_WATCHDOG_CHECK_ROOTFS=true
|
||||
|
||||
# Kernel BUG/Oops in dmesg delta since last cycle → reboot immediately.
|
||||
# A kernel oops means the kernel ran with a corrupted state — stability is not guaranteed.
|
||||
HOST1_SYS_WATCHDOG_CHECK_KERNEL_OOPS=true
|
||||
|
||||
# File descriptor exhaustion at FD_CRITICAL_PCT (95%) → reboot immediately.
|
||||
# At 95% FD: new connections fail, Docker can't spawn processes, SSH drops.
|
||||
HOST1_SYS_WATCHDOG_CHECK_FD=true
|
||||
|
||||
# /boot read-only detected → reboot immediately.
|
||||
# Unexpected read-only /boot means state files and config writes are silently failing.
|
||||
# Fallback state, watchdog reboot log, and lock files all go stale silently.
|
||||
HOST1_SYS_WATCHDOG_CHECK_BOOT=true
|
||||
|
||||
# ━━━ Tier 2 — Urgent OOM Check ━━━
|
||||
# Bypass strikes when RAM is critically low AND OOM kill rate confirms active crisis.
|
||||
# Both must be enabled for Tier 2 bypass to function — disable either to always use strikes.
|
||||
|
||||
# Track kernel OOM kills each cycle via /proc/vmstat oom_kill delta.
|
||||
# Also provides diagnostic context in reboot messages (which processes were killed).
|
||||
HOST1_SYS_WATCHDOG_CHECK_OOM=true
|
||||
|
||||
# Free RAM check — required for both Tier 2 bypass and RAM tier logic.
|
||||
# Tiers: MEM_WARN_GB(10) → notify | MEM_SHUTDOWN_GB(6) → stop containers | MEM_GB(4) → strikes
|
||||
HOST1_SYS_WATCHDOG_CHECK_RAM=true
|
||||
|
||||
# ━━━ Tier 3 — Standard Checks (strike system) ━━━
|
||||
# Each check must fail SYS_WATCHDOG_STRIKE_LIMIT consecutive cycles before action is taken.
|
||||
# Single spikes are ignored — sustained problems trigger reboot.
|
||||
|
||||
# /var/log filesystem usage above SYS_WATCHDOG_LOG_PCT.
|
||||
# Log spam (Docker log storms, syslog loops) fills rootfs — indicates something broken.
|
||||
HOST1_SYS_WATCHDOG_CHECK_LOG=true
|
||||
|
||||
# ZFS ARC memory pinned above SYS_WATCHDOG_ARC_PINNED_PCT after cache drop.
|
||||
# Enabled on HOST1 — ZFS cache pools actively used. Disable on hosts without ZFS.
|
||||
HOST1_SYS_WATCHDOG_CHECK_ARC=true
|
||||
|
||||
# CPU temperature above SYS_WATCHDOG_CPU_TEMP_MAX (95°C).
|
||||
# Sustained high temp causes kernel throttling or panic. Requires lm-sensors.
|
||||
HOST1_SYS_WATCHDOG_CHECK_CPU_TEMP=true
|
||||
|
||||
# Load average above SYS_WATCHDOG_LOAD_MULTIPLIER × core count.
|
||||
# DISABLED on HOST1 — Tdarr and Emby cause legitimate sustained load spikes during encoding.
|
||||
# Enable on idle servers or adjust SYS_WATCHDOG_LOAD_MULTIPLIER if load is always high.
|
||||
HOST1_SYS_WATCHDOG_CHECK_LOAD=false
|
||||
|
||||
# Zombie process count above SYS_WATCHDOG_ZOMBIE_LIMIT (50).
|
||||
# Large zombie counts indicate serious process management failure — something is stuck.
|
||||
HOST1_SYS_WATCHDOG_CHECK_ZOMBIES=true
|
||||
|
||||
# Check docker_watchdog.sh persistent skip list — required containers on skip list.
|
||||
# Cross-watchdog coordination: if docker_watchdog gave up, system_watchdog escalates.
|
||||
# ENABLED — HOST1 fully built and operational, skip list is meaningful.
|
||||
HOST1_SYS_WATCHDOG_CHECK_CONTAINERS=true
|
||||
|
||||
# /tmp filesystem usage above SYS_WATCHDOG_TMP_PCT with auto-clear attempt.
|
||||
# Script tries to clear aged /tmp files first — only strikes if clear fails.
|
||||
# Lock files, rsync temp files, and Docker ops use /tmp — 100% means lock failures.
|
||||
HOST1_SYS_WATCHDOG_CHECK_TMP=true
|
||||
|
||||
# Array disk error count delta in /proc/mdstat — accumulating errors = disk failing now.
|
||||
# Triggers on SYS_WATCHDOG_MDSTAT_ERROR_LIMIT new errors in one cycle.
|
||||
HOST1_SYS_WATCHDOG_CHECK_MDSTAT=true
|
||||
|
||||
# Primary NIC operstate — detects NIC going down (physical or driver failure).
|
||||
# Uses HOST1_SYS_WATCHDOG_NIC above. Strike system — brief flaps don't trigger reboot.
|
||||
HOST1_SYS_WATCHDOG_CHECK_NETWORK=true
|
||||
|
||||
# sshd running check — attempts restart before escalating.
|
||||
# sshd down = no remote access. Script tries rc.sshd start, notifies, strikes on failure.
|
||||
HOST1_SYS_WATCHDOG_CHECK_SSHD=true
|
||||
|
||||
# Runaway process detection — single process above SYS_WATCHDOG_RUNAWAY_CPU_PCT sustained.
|
||||
# DISABLED — Tdarr encoding and Emby transcoding legitimately peg CPU for extended periods.
|
||||
# Enable only if HOST1 has no CPU-intensive workloads.
|
||||
HOST1_SYS_WATCHDOG_CHECK_RUNAWAY=false
|
||||
|
||||
# ==============================================================================================
|
||||
# ── RESOURCE MANAGER ──────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Containers to manage under pressure — see master.conf RW_CRITICAL_CONTAINERS for exclusions.
|
||||
|
||||
# docker pause at medium pressure (RAM < RW_RAM_MEDIUM_GB or load > medium threshold)
|
||||
# Suspended in-place — instant to pause/unpause, no state lost, no restart delay.
|
||||
HOST1_RW_PAUSE_CONTAINERS=(
|
||||
"Huntarr" # arr search automation — safe to suspend
|
||||
"Cleanuparr" # download cleanup — safe to suspend
|
||||
"Healarr" # arr health checks — safe to suspend
|
||||
"Soularr" # Slskd automation — background only
|
||||
"ChannelTube" # YouTube archiver — background only
|
||||
"Pinchflat" # YouTube archiver — background only
|
||||
)
|
||||
|
||||
# docker stop at hard pressure (RAM < RW_RAM_HARD_GB)
|
||||
# Full stop — these are optional/heavy services that free significant RAM when stopped.
|
||||
# resource_watchdog.sh restarts them when pressure fully clears (RAM >= RW_RAM_RECOVER_GB).
|
||||
HOST1_RW_STOP_CONTAINERS=(
|
||||
"LocalAI" # GPU/CPU heavy — largest RAM consumer when idle
|
||||
"7DaysToDie" # game server — optional
|
||||
"V-Rising" # game server — optional
|
||||
"Code-Server" # IDE — not needed during pressure events
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── AUTH STACK ────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Credentials for the Varaverk Auth Stack page (NPM, lldap, Authelia).
|
||||
|
||||
# ━━━ NginxProxyManager ━━━
|
||||
HOST1_NPM_URL="http://localhost:7818"
|
||||
HOST1_NPM_USER="failedproxy@gmail.com"
|
||||
HOST1_NPM_PASS="183134\$eanHess"
|
||||
|
||||
# ━━━ lldap ━━━
|
||||
HOST1_LLDAP_URL="http://localhost:17170"
|
||||
HOST1_LLDAP_USER="admin"
|
||||
HOST1_LLDAP_PASS="183134\$eanHess"
|
||||
|
||||
# ━━━ Authelia ━━━
|
||||
HOST1_AUTHELIA_CONFIG="/mnt/user/appdata-Fallback/Critical-Data/Authelia/configuration.yml"
|
||||
HOST1_AUTHELIA_CONTAINER="Authelia"
|
||||
|
||||
# ==============================================================================================
|
||||
# ──────────────────────── End Of HOST1 Variables ──────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
HOST1_UNRAID_API_KEY="1825c3a2e03ea5089974f4da2e171aa2d5907a1dea23cc479c33e492c8ff4dbb"
|
||||
+585
@@ -0,0 +1,585 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ================================= Sonarr Cleanup =============================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Delete orphaned TV episode files not tracked by Sonarr. Queries the API for
|
||||
# all tracked episode file paths, walks the library on disk, and removes anything
|
||||
# untracked that is old enough to be past the import window. Triggers an Emby
|
||||
# library clean after each deletion run so ghost entries disappear immediately.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Every file encountered on disk is classified into one of five categories:
|
||||
#
|
||||
# TRACKED — Sonarr API knows this exact path → leave it alone
|
||||
# PROTECTED — matches SONARR_PROTECTED_PATTERNS → never delete
|
||||
# ORPHAN — video file, not tracked, older than SONARR_ORPHAN_AGE → delete
|
||||
# JUNK — not a video extension, not protected → delete regardless of age
|
||||
# RECENT — not tracked, under SONARR_ORPHAN_AGE → skip (may be mid-import)
|
||||
#
|
||||
# Sonarr generates show artwork (*.jpg), metadata (*.nfo), and manages subtitles
|
||||
# (*.srt, *.sub, *.ass) but does NOT include these in its tracked file API response.
|
||||
# Without PROTECTED classification these would be deleted — breaking Sonarr and
|
||||
# Emby metadata display.
|
||||
#
|
||||
# After deletions: notify_emby_scan() triggers Emby "Clean Missing Files" task.
|
||||
# Emby removes ghost entries immediately — no user-facing file-not-found errors.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Six gates — ALL must pass before any file is touched:
|
||||
# 1. Container running and not starting/unhealthy
|
||||
# 2. API reachable
|
||||
# 3. API version matches SONARR_VERSION_MAJOR in master.conf
|
||||
# 4. Series count > 0
|
||||
# 5. Tracked file count > 0
|
||||
# 6. Deletion size < SONARR_MAX_DELETE_GB — or --i-know-what-im-doing required
|
||||
#
|
||||
# acquire_lock "wait" — large scans take time, wait for previous run to finish
|
||||
# jq + curl validation — exits if either tool missing
|
||||
# DOCKER_TIMEOUT — container checks protected against daemon hangs
|
||||
# notify_emby_scan() — triggers Emby clean after deletion
|
||||
# platform_require_cmd — notify script validated before use
|
||||
# Silent by default — orphans/junk warn(), clean library logs silently
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# host*.conf
|
||||
#
|
||||
# HOST*_SONARR_URL / HOST*_SONARR_API_KEY / HOST*_SONARR_TV_ROOT
|
||||
# HOST*_SONARR_PATH_MAP — container path → host path translation
|
||||
# All aliased by detect_hosts() — script uses unprefixed names
|
||||
#
|
||||
# master.conf
|
||||
#
|
||||
# SONARR_ORPHAN_AGE — days before untracked file eligible for deletion
|
||||
# SONARR_MAX_DELETE_GB — require --i-know-what-im-doing above this
|
||||
# SONARR_EXTENSIONS — video file extensions for orphan classification
|
||||
# SONARR_PROTECTED_PATTERNS — file patterns never deleted
|
||||
# SONARR_VERSION_MAJOR — expected Sonarr major version for API safety check
|
||||
# SONARR_IMPORT_SCAN_TIMEOUT — seconds to wait for pre-flight import scan (default 600)
|
||||
# ARR_CLEANUP_STATS — stats file path (read by coffee report)
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# sonarr_cleanup.sh — normal run
|
||||
# sonarr_cleanup.sh --dry-run — preview, no deletions
|
||||
# sonarr_cleanup.sh --log — verbose output
|
||||
# sonarr_cleanup.sh --status — show config and exit
|
||||
# sonarr_cleanup.sh --i-know-what-im-doing — bypass size threshold
|
||||
# sonarr_cleanup.sh --i-know-what-im-doing --skip-strike-list — NUCLEAR MODE
|
||||
#
|
||||
# NUCLEAR MODE: both flags bypass age check AND size threshold. User accepts full
|
||||
# responsibility — the flag name is long and annoying by design.
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
# ── Special flag pre-processing ───────────────────────────────────────────────────────────────
|
||||
I_KNOW=false
|
||||
SKIP_STRIKES=false
|
||||
FILTERED_ARGS=()
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--i-know-what-im-doing) I_KNOW=true ;;
|
||||
--skip-strike-list) SKIP_STRIKES=true ;;
|
||||
*) FILTERED_ARGS+=("$arg") ;;
|
||||
esac
|
||||
done
|
||||
|
||||
parse_args "${FILTERED_ARGS[@]}"
|
||||
|
||||
# ── Nuclear mode warning ──────────────────────────────────────────────────────────────────────
|
||||
if [[ "$I_KNOW" == true ]] && [[ "$SKIP_STRIKES" == true ]] && [[ "$DRY_RUN" != true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo "⚠️ WARNING — NUCLEAR MODE ACTIVE"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo " Flags: --i-know-what-im-doing --skip-strike-list"
|
||||
echo " Strike system: BYPASSED — deletes on first pass"
|
||||
echo " Size threshold: BYPASSED — no GB limit"
|
||||
echo " Data recovery: NOT POSSIBLE after deletion"
|
||||
echo ""
|
||||
echo " Review --dry-run output before proceeding."
|
||||
echo " You have 10 seconds to cancel (Ctrl+C)..."
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
sleep 10
|
||||
echo " Proceeding..."
|
||||
echo ""
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v curl >/dev/null 2>&1; then
|
||||
error "curl not found — required for Sonarr API calls"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v jq >/dev/null 2>&1; then
|
||||
error "jq not found — required for JSON parsing"
|
||||
notify "Sonarr cleanup failed on $(hostname) — jq not installed" "Sonarr Cleanup" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
platform_require_cmd \
|
||||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||||
"" "" \
|
||||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||
|
||||
acquire_lock "wait"
|
||||
TMP_DIR="/tmp/sonarr_cleanup_$$"
|
||||
mkdir -p "$TMP_DIR"
|
||||
trap "_release_all_locks; rm -rf $TMP_DIR" EXIT
|
||||
|
||||
if ! command -v docker &>/dev/null; then
|
||||
error "Docker command not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# detect_hosts() sets MY_ID and aliases SONARR_URL, SONARR_API_KEY, SONARR_TV_ROOT
|
||||
detect_hosts
|
||||
|
||||
DOCKER_TIMEOUT=15
|
||||
SONARR_CONTAINER="Sonarr"
|
||||
|
||||
# Build path map from MY_ID's Sonarr path map
|
||||
declare -A ARR_PATH_MAP
|
||||
local_path_map_var="${MY_ID}_SONARR_PATH_MAP"
|
||||
eval "for key in \"\${!${local_path_map_var}[@]}\"; do
|
||||
ARR_PATH_MAP[\"\$key\"]=\"\${${local_path_map_var}[\$key]}\"
|
||||
done"
|
||||
|
||||
require_var SONARR_URL
|
||||
require_var SONARR_API_KEY
|
||||
require_var SONARR_TV_ROOT
|
||||
|
||||
if [[ ! -d "$SONARR_TV_ROOT" ]]; then
|
||||
error "TV root not found: $SONARR_TV_ROOT"
|
||||
notify "Sonarr cleanup failed on $(hostname) — TV root not found: $SONARR_TV_ROOT" \
|
||||
"Sonarr Cleanup" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log "$ICON_GEAR Config: url=${SONARR_URL} root=${SONARR_TV_ROOT}"
|
||||
echo " $MY_ID ($LOCAL_SERVER_NAME) — $SONARR_URL"
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be deleted"
|
||||
[[ "$I_KNOW" == true ]] && warn "OVERRIDE — --i-know-what-im-doing active"
|
||||
[[ "$SKIP_STRIKES" == true ]] && warn "OVERRIDE — --skip-strike-list active — age check bypassed"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_GEAR Sonarr URL: $SONARR_URL"
|
||||
echo "$ICON_GEAR TV root: $SONARR_TV_ROOT"
|
||||
echo "$ICON_TIME Orphan age: ${SONARR_ORPHAN_AGE} days"
|
||||
echo "$ICON_GEAR Max delete: ${SONARR_MAX_DELETE_GB}GB (requires --i-know-what-im-doing)"
|
||||
echo "$ICON_GEAR Sonarr ver: v${SONARR_VERSION_MAJOR} expected"
|
||||
echo "$ICON_GEAR Extensions: ${SONARR_EXTENSIONS[*]}"
|
||||
echo "$ICON_GEAR Protected patterns: ${SONARR_PROTECTED_PATTERNS[*]}"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "$ICON_GEAR I know: $I_KNOW"
|
||||
echo "$ICON_GEAR Skip strikes: $SKIP_STRIKES"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Safety Layer 1 — Container Health ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_SHIELD Safety Checks ━━━"
|
||||
|
||||
CONTAINER_RUNNING=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
|
||||
'{{.State.Running}}' "$SONARR_CONTAINER" 2>/dev/null)
|
||||
if [[ "$CONTAINER_RUNNING" != "true" ]]; then
|
||||
error "$SONARR_CONTAINER is not running — aborting"
|
||||
notify "Sonarr cleanup aborted on $(hostname) — container not running" \
|
||||
"Sonarr Cleanup" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
CONTAINER_HEALTH=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
|
||||
'{{.State.Health.Status}}' "$SONARR_CONTAINER" 2>/dev/null)
|
||||
case "$CONTAINER_HEALTH" in
|
||||
healthy) info "$SONARR_CONTAINER is healthy" ;;
|
||||
"") info "$SONARR_CONTAINER has no health check — proceeding" ;;
|
||||
starting)
|
||||
error "$SONARR_CONTAINER is still starting — aborting"
|
||||
notify "Sonarr cleanup aborted on $(hostname) — container still starting" \
|
||||
"Sonarr Cleanup" "warning"
|
||||
exit 1 ;;
|
||||
unhealthy)
|
||||
error "$SONARR_CONTAINER is unhealthy — aborting"
|
||||
notify "Sonarr cleanup aborted on $(hostname) — container unhealthy" \
|
||||
"Sonarr Cleanup" "warning"
|
||||
exit 1 ;;
|
||||
*) warn "$SONARR_CONTAINER health: $CONTAINER_HEALTH — proceeding with caution" ;;
|
||||
esac
|
||||
|
||||
info "Safety layer 1 passed — container healthy"
|
||||
|
||||
# ==============================================================================================
|
||||
# ── HELPER FUNCTIONS ──────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
sonarr_api() {
|
||||
local endpoint="$1"
|
||||
local response http_code body
|
||||
|
||||
response=$(curl -sf \
|
||||
--max-time 30 \
|
||||
-H "X-Api-Key: $SONARR_API_KEY" \
|
||||
-w "\n%{http_code}" \
|
||||
"${SONARR_URL}/api/v3/${endpoint}" 2>/dev/null)
|
||||
|
||||
http_code=$(echo "$response" | tail -1)
|
||||
body=$(echo "$response" | head -n -1)
|
||||
|
||||
if [[ "$http_code" != "200" ]]; then
|
||||
error "Sonarr API HTTP $http_code for: $endpoint"
|
||||
return 1
|
||||
fi
|
||||
echo "$body"
|
||||
}
|
||||
|
||||
is_video_file() {
|
||||
local ext="${1##*.}"
|
||||
ext="${ext,,}"
|
||||
for valid_ext in "${SONARR_EXTENSIONS[@]}"; do
|
||||
[[ "$ext" == "$valid_ext" ]] && return 0
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
is_protected_file() {
|
||||
local filename
|
||||
filename=$(basename "$1")
|
||||
for pattern in "${SONARR_PROTECTED_PATTERNS[@]}"; do
|
||||
# shellcheck disable=SC2254
|
||||
case "$filename" in
|
||||
$pattern) return 0 ;;
|
||||
esac
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
format_bytes() {
|
||||
local bytes=$1
|
||||
if (( bytes > 1073741824 )); then
|
||||
awk "BEGIN {printf \"%.1fGB\", $bytes / 1073741824}"
|
||||
elif (( bytes > 1048576 )); then
|
||||
awk "BEGIN {printf \"%.1fMB\", $bytes / 1048576}"
|
||||
else
|
||||
echo "${bytes}B"
|
||||
fi
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Pre-flight: Sonarr Import Scan ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Pre-flight: Sonarr Import Scan ━━━"
|
||||
|
||||
# Reverse-lookup container path from path map so Sonarr gets its own path, not the host path
|
||||
SONARR_CONTAINER_ROOT=""
|
||||
for _cp in "${!ARR_PATH_MAP[@]}"; do
|
||||
if [[ "${ARR_PATH_MAP[$_cp]}" == "$SONARR_TV_ROOT" ]]; then
|
||||
SONARR_CONTAINER_ROOT="$_cp"
|
||||
break
|
||||
fi
|
||||
done
|
||||
unset _cp
|
||||
|
||||
if [[ -n "$SONARR_CONTAINER_ROOT" ]]; then
|
||||
info "Triggering DownloadedEpisodesScan on: $SONARR_CONTAINER_ROOT"
|
||||
SCAN_PAYLOAD="{\"name\": \"DownloadedEpisodesScan\", \"path\": \"$SONARR_CONTAINER_ROOT\"}"
|
||||
else
|
||||
info "No path map match — triggering DownloadedEpisodesScan (all root folders)"
|
||||
SCAN_PAYLOAD='{"name": "DownloadedEpisodesScan"}'
|
||||
fi
|
||||
|
||||
SCAN_RESPONSE=$(curl -sf --max-time 30 -X POST \
|
||||
-H "X-Api-Key: $SONARR_API_KEY" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "$SCAN_PAYLOAD" \
|
||||
"${SONARR_URL}/api/v3/command" 2>/dev/null)
|
||||
|
||||
SCAN_CMD_ID=$(echo "$SCAN_RESPONSE" | jq -r '.id // empty' 2>/dev/null)
|
||||
|
||||
if [[ -z "$SCAN_CMD_ID" ]]; then
|
||||
warn "Could not trigger import scan — proceeding without pre-flight"
|
||||
else
|
||||
info "Import scan queued (command ID: $SCAN_CMD_ID) — waiting for completion..."
|
||||
POLL_TIMEOUT=${SONARR_IMPORT_SCAN_TIMEOUT:-600}
|
||||
POLLED=0
|
||||
while [[ "$POLLED" -lt "$POLL_TIMEOUT" ]]; do
|
||||
SCAN_STATUS=$(curl -sf --max-time 10 \
|
||||
-H "X-Api-Key: $SONARR_API_KEY" \
|
||||
"${SONARR_URL}/api/v3/command/${SCAN_CMD_ID}" 2>/dev/null | \
|
||||
jq -r '.status // empty' 2>/dev/null)
|
||||
case "$SCAN_STATUS" in
|
||||
completed) info "Import scan complete ✅"; break ;;
|
||||
failed) warn "Import scan reported failed — proceeding anyway"; break ;;
|
||||
esac
|
||||
sleep 10
|
||||
(( POLLED += 10 ))
|
||||
[[ $(( POLLED % 60 )) -eq 0 ]] && log " Still scanning... (${POLLED}s elapsed)"
|
||||
done
|
||||
[[ "$POLLED" -ge "$POLL_TIMEOUT" ]] && \
|
||||
warn "Import scan timed out after ${POLL_TIMEOUT}s — proceeding anyway"
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Fetch Sonarr Tracked Files ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Fetching Sonarr Tracked Files ━━━"
|
||||
|
||||
# Safety Layer 2 — API reachability
|
||||
if ! check_api "$SONARR_URL" "Sonarr" 10; then
|
||||
notify "Sonarr cleanup aborted on $(hostname) — API unreachable" "Sonarr Cleanup" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Safety Layer 3 — API version check
|
||||
check_arr_version "$SONARR_URL" "$SONARR_API_KEY" "v3" "$SONARR_VERSION_MAJOR" "Sonarr" || exit 1
|
||||
|
||||
info "Querying Sonarr API..."
|
||||
|
||||
# Fetch all series
|
||||
SERIES_RESPONSE=$(sonarr_api "series") || {
|
||||
error "Failed to fetch series from Sonarr"
|
||||
notify "Sonarr cleanup failed on $(hostname) — could not fetch series" \
|
||||
"Sonarr Cleanup" "warning"
|
||||
exit 1
|
||||
}
|
||||
|
||||
SERIES_IDS=$(echo "$SERIES_RESPONSE" | jq -r '.[].id' 2>/dev/null)
|
||||
SERIES_COUNT=$(echo "$SERIES_IDS" | grep -c "." 2>/dev/null || echo 0)
|
||||
|
||||
# Safety Layer 4 — series count > 0
|
||||
if [[ "$SERIES_COUNT" -eq 0 ]]; then
|
||||
error "API returned 0 series — aborting to prevent mass deletion"
|
||||
notify "Sonarr cleanup aborted on $(hostname) — 0 series returned" \
|
||||
"Sonarr Cleanup" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
info "Found $SERIES_COUNT series — fetching episode files..."
|
||||
|
||||
TRACKED_FILE="$TMP_DIR/tracked_paths.txt"
|
||||
> "$TRACKED_FILE"
|
||||
|
||||
SERIES_INDEX=0
|
||||
while IFS= read -r series_id; do
|
||||
[[ -z "$series_id" ]] && continue
|
||||
(( SERIES_INDEX++ ))
|
||||
[[ $(( SERIES_INDEX % 50 )) -eq 0 ]] && \
|
||||
log "Fetching files: $SERIES_INDEX/$SERIES_COUNT series..."
|
||||
SERIES_FILES=$(sonarr_api "episodefile?seriesId=${series_id}" 2>/dev/null)
|
||||
if [[ -n "$SERIES_FILES" ]]; then
|
||||
while IFS= read -r api_path; do
|
||||
[[ -z "$api_path" ]] && continue
|
||||
translate_path "$api_path" >> "$TRACKED_FILE"
|
||||
done < <(echo "$SERIES_FILES" | jq -r '.[].path' 2>/dev/null)
|
||||
fi
|
||||
done <<< "$SERIES_IDS"
|
||||
|
||||
sort -u "$TRACKED_FILE" -o "$TRACKED_FILE"
|
||||
|
||||
# Build in-memory lookup map — O(1) per lookup vs O(n) grep per file
|
||||
declare -A TRACKED_MAP
|
||||
while IFS= read -r _tracked_path; do
|
||||
[[ -n "$_tracked_path" ]] && TRACKED_MAP["$_tracked_path"]=1
|
||||
done < "$TRACKED_FILE"
|
||||
unset _tracked_path
|
||||
info "Built in-memory lookup map: ${#TRACKED_MAP[@]} tracked paths"
|
||||
TRACKED_COUNT=$(wc -l < "$TRACKED_FILE")
|
||||
|
||||
# Safety Layer 5 — tracked count > 0
|
||||
if [[ "$TRACKED_COUNT" -eq 0 ]]; then
|
||||
error "API returned 0 tracked files — aborting to prevent mass deletion"
|
||||
notify "Sonarr cleanup aborted on $(hostname) — 0 tracked files returned" \
|
||||
"Sonarr Cleanup" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
info "$SERIES_COUNT series | $TRACKED_COUNT tracked episode files"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Scan TV Root ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_CLEAN Scanning TV Root ━━━"
|
||||
info "Root: $SONARR_TV_ROOT | Orphan age: ${SONARR_ORPHAN_AGE} days"
|
||||
|
||||
START=$(date +%s)
|
||||
ORPHAN_COUNT=0
|
||||
JUNK_COUNT=0
|
||||
RECENT_COUNT=0
|
||||
PROTECTED_COUNT=0
|
||||
ORPHAN_BYTES=0
|
||||
JUNK_BYTES=0
|
||||
|
||||
AGE_SECONDS=$(( SONARR_ORPHAN_AGE * 86400 ))
|
||||
NOW=$(date +%s)
|
||||
MAX_DELETE_BYTES=$(awk "BEGIN {printf \"%d\", $SONARR_MAX_DELETE_GB * 1073741824}")
|
||||
|
||||
while IFS= read -r filepath; do
|
||||
[[ -z "$filepath" ]] && continue
|
||||
|
||||
if [[ -n "${TRACKED_MAP[$filepath]:-}" ]]; then
|
||||
log "TRACKED: $filepath"
|
||||
continue
|
||||
fi
|
||||
|
||||
if is_protected_file "$filepath"; then
|
||||
log "$ICON_PROTECTED PROTECTED: $filepath"
|
||||
(( PROTECTED_COUNT++ ))
|
||||
continue
|
||||
fi
|
||||
|
||||
FILE_SIZE=$(stat -c%s "$filepath" 2>/dev/null || echo 0)
|
||||
|
||||
if is_video_file "$filepath"; then
|
||||
FILE_MTIME=$(stat -c %Y "$filepath" 2>/dev/null || echo 0)
|
||||
FILE_AGE=$(( NOW - FILE_MTIME ))
|
||||
|
||||
if [[ "$FILE_AGE" -lt "$AGE_SECONDS" ]] && [[ "$SKIP_STRIKES" != true ]]; then
|
||||
log "RECENT (skipping): $filepath"
|
||||
(( RECENT_COUNT++ ))
|
||||
continue
|
||||
fi
|
||||
|
||||
warn "$ICON_TRASH ORPHAN: $filepath"
|
||||
(( ORPHAN_COUNT++ ))
|
||||
ORPHAN_BYTES=$(( ORPHAN_BYTES + FILE_SIZE ))
|
||||
else
|
||||
log "JUNK: $filepath"
|
||||
(( JUNK_COUNT++ ))
|
||||
JUNK_BYTES=$(( JUNK_BYTES + FILE_SIZE ))
|
||||
fi
|
||||
|
||||
done < <(
|
||||
for host_path in "${ARR_PATH_MAP[@]}" "$SONARR_TV_ROOT"; do
|
||||
[[ -d "$host_path" ]] && find "$host_path" -type f 2>/dev/null
|
||||
done | sort -u
|
||||
)
|
||||
|
||||
TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES ))
|
||||
TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT ))
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Safety Layer 6 — Deletion Size Threshold ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$TOTAL_DELETE_BYTES" -gt "$MAX_DELETE_BYTES" ]]; then
|
||||
TOTAL_HUMAN=$(awk "BEGIN {printf \"%.1fGB\", $TOTAL_DELETE_BYTES / 1073741824}")
|
||||
if [[ "$I_KNOW" != true ]]; then
|
||||
echo ""
|
||||
error "Deletion would exceed ${SONARR_MAX_DELETE_GB}GB — $TOTAL_HUMAN would be deleted"
|
||||
error "Review ORPHAN lines above carefully before proceeding"
|
||||
error "Rerun with: --i-know-what-im-doing"
|
||||
error "To also bypass age check: add --skip-strike-list"
|
||||
notify "Sonarr cleanup halted on $(hostname) — ${TOTAL_HUMAN} requires --i-know-what-im-doing" \
|
||||
"Sonarr Cleanup" "warning"
|
||||
exit 1
|
||||
else
|
||||
warn "OVERRIDE — deletion is $TOTAL_HUMAN — proceeding with --i-know-what-im-doing"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Execute Deletions ─────────────────────────────────────────────────────────────────────────
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
while IFS= read -r filepath; do
|
||||
[[ -z "$filepath" ]] && continue
|
||||
[[ -n "${TRACKED_MAP[$filepath]:-}" ]] && continue
|
||||
is_protected_file "$filepath" && continue
|
||||
|
||||
FILE_MTIME=$(stat -c %Y "$filepath" 2>/dev/null || echo 0)
|
||||
FILE_AGE=$(( NOW - FILE_MTIME ))
|
||||
|
||||
if is_video_file "$filepath"; then
|
||||
[[ "$FILE_AGE" -lt "$AGE_SECONDS" ]] && \
|
||||
[[ "$SKIP_STRIKES" != true ]] && continue
|
||||
fi
|
||||
|
||||
rm -f "$filepath" 2>/dev/null || error "Failed to delete: $filepath"
|
||||
|
||||
done < <(
|
||||
for host_path in "${ARR_PATH_MAP[@]}" "$SONARR_TV_ROOT"; do
|
||||
[[ -d "$host_path" ]] && find "$host_path" -type f 2>/dev/null
|
||||
done | sort -u
|
||||
)
|
||||
|
||||
info "Cleaning up empty folders..."
|
||||
for host_path in "${ARR_PATH_MAP[@]}" "$SONARR_TV_ROOT"; do
|
||||
[[ -d "$host_path" ]] && \
|
||||
find "$host_path" -mindepth 1 -type d -empty -delete 2>/dev/null
|
||||
done
|
||||
info "Empty folders removed"
|
||||
fi
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
ORPHAN_HUMAN=$(format_bytes "$ORPHAN_BYTES")
|
||||
JUNK_HUMAN=$(format_bytes "$JUNK_BYTES")
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY SONARR CLEANUP SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_SYNC Tracked: $TRACKED_COUNT files ($SERIES_COUNT series)"
|
||||
echo "$ICON_SHIELD Protected: $PROTECTED_COUNT files (artwork, subtitles, metadata)"
|
||||
echo "$ICON_TRASH Orphans: $ORPHAN_COUNT files ($ORPHAN_HUMAN)"
|
||||
echo "$ICON_TRASH Junk: $JUNK_COUNT files ($JUNK_HUMAN)"
|
||||
echo "$ICON_SKIP Recent skipped: $RECENT_COUNT files (under ${SONARR_ORPHAN_AGE} days)"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no files deleted"
|
||||
elif [[ "$TOTAL_REMOVED" -eq 0 ]]; then
|
||||
echo "$ICON_DONE Clean — nothing to remove"
|
||||
else
|
||||
warn "$ICON_DONE Removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)"
|
||||
notify "Sonarr cleanup on $(hostname) — removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" \
|
||||
"Sonarr Cleanup" "warning"
|
||||
# Notify Emby to clean missing files — removes ghost entries immediately
|
||||
notify_emby_scan
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
# Write stats for sunday_morning_coffee_report.sh
|
||||
if [[ "$DRY_RUN" == false ]] && [[ -n "${ARR_CLEANUP_STATS:-}" ]]; then
|
||||
echo "$(date '+%Y-%m-%d')|sonarr|${ORPHAN_COUNT}|${ORPHAN_BYTES}|${JUNK_COUNT}|${JUNK_BYTES}|${RECENT_COUNT}|${TRACKED_COUNT}" \
|
||||
>> "$ARR_CLEANUP_STATS" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
exit 0
|
||||
+586
@@ -0,0 +1,586 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ================================= Sonarr Cleanup =============================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Delete orphaned TV episode files not tracked by Sonarr. Queries the API for
|
||||
# all tracked episode file paths, walks the library on disk, and removes anything
|
||||
# untracked that is old enough to be past the import window. Triggers an Emby
|
||||
# library clean after each deletion run so ghost entries disappear immediately.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Every file encountered on disk is classified into one of five categories:
|
||||
#
|
||||
# TRACKED — Sonarr API knows this exact path → leave it alone
|
||||
# PROTECTED — matches SONARR_PROTECTED_PATTERNS → never delete
|
||||
# ORPHAN — video file, not tracked, older than SONARR_ORPHAN_AGE → delete
|
||||
# JUNK — not a video extension, not protected → delete regardless of age
|
||||
# RECENT — not tracked, under SONARR_ORPHAN_AGE → skip (may be mid-import)
|
||||
#
|
||||
# Sonarr generates show artwork (*.jpg), metadata (*.nfo), and manages subtitles
|
||||
# (*.srt, *.sub, *.ass) but does NOT include these in its tracked file API response.
|
||||
# Without PROTECTED classification these would be deleted — breaking Sonarr and
|
||||
# Emby metadata display.
|
||||
#
|
||||
# After deletions: notify_emby_scan() triggers Emby "Clean Missing Files" task.
|
||||
# Emby removes ghost entries immediately — no user-facing file-not-found errors.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Six gates — ALL must pass before any file is touched:
|
||||
# 1. Container running and not starting/unhealthy
|
||||
# 2. API reachable
|
||||
# 3. API version matches SONARR_VERSION_MAJOR in master.conf
|
||||
# 4. Series count > 0
|
||||
# 5. Tracked file count > 0
|
||||
# 6. Deletion size < SONARR_MAX_DELETE_GB — or --i-know-what-im-doing required
|
||||
#
|
||||
# acquire_lock "wait" — large scans take time, wait for previous run to finish
|
||||
# jq + curl validation — exits if either tool missing
|
||||
# DOCKER_TIMEOUT — container checks protected against daemon hangs
|
||||
# notify_emby_scan() — triggers Emby clean after deletion
|
||||
# platform_require_cmd — notify script validated before use
|
||||
# Silent by default — orphans/junk warn(), clean library logs silently
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# host*.conf
|
||||
#
|
||||
# HOST*_SONARR_URL / HOST*_SONARR_API_KEY / HOST*_SONARR_TV_ROOT
|
||||
# HOST*_SONARR_PATH_MAP — container path → host path translation
|
||||
# All aliased by detect_hosts() — script uses unprefixed names
|
||||
#
|
||||
# master.conf
|
||||
#
|
||||
# SONARR_ORPHAN_AGE — days before untracked file eligible for deletion
|
||||
# SONARR_MAX_DELETE_GB — require --i-know-what-im-doing above this
|
||||
# SONARR_EXTENSIONS — video file extensions for orphan classification
|
||||
# SONARR_PROTECTED_PATTERNS — file patterns never deleted
|
||||
# SONARR_VERSION_MAJOR — expected Sonarr major version for API safety check
|
||||
# SONARR_IMPORT_SCAN_TIMEOUT — seconds to wait for pre-flight import scan (default 600)
|
||||
# ARR_CLEANUP_STATS — stats file path (read by coffee report)
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# sonarr_cleanup.sh — normal run
|
||||
# sonarr_cleanup.sh --dry-run — preview, no deletions
|
||||
# sonarr_cleanup.sh --log — verbose output
|
||||
# sonarr_cleanup.sh --status — show config and exit
|
||||
# sonarr_cleanup.sh --i-know-what-im-doing — bypass size threshold
|
||||
# sonarr_cleanup.sh --i-know-what-im-doing --skip-strike-list — NUCLEAR MODE
|
||||
#
|
||||
# NUCLEAR MODE: both flags bypass age check AND size threshold. User accepts full
|
||||
# responsibility — the flag name is long and annoying by design.
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
# ── Special flag pre-processing ───────────────────────────────────────────────────────────────
|
||||
I_KNOW=false
|
||||
SKIP_STRIKES=false
|
||||
FILTERED_ARGS=()
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--i-know-what-im-doing) I_KNOW=true ;;
|
||||
--skip-strike-list) SKIP_STRIKES=true ;;
|
||||
*) FILTERED_ARGS+=("$arg") ;;
|
||||
esac
|
||||
done
|
||||
|
||||
parse_args "${FILTERED_ARGS[@]}"
|
||||
|
||||
# ── Nuclear mode warning ──────────────────────────────────────────────────────────────────────
|
||||
if [[ "$I_KNOW" == true ]] && [[ "$SKIP_STRIKES" == true ]] && [[ "$DRY_RUN" != true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo "⚠️ WARNING — NUCLEAR MODE ACTIVE"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo " Flags: --i-know-what-im-doing --skip-strike-list"
|
||||
echo " Strike system: BYPASSED — deletes on first pass"
|
||||
echo " Size threshold: BYPASSED — no GB limit"
|
||||
echo " Data recovery: NOT POSSIBLE after deletion"
|
||||
echo ""
|
||||
echo " Review --dry-run output before proceeding."
|
||||
echo " You have 10 seconds to cancel (Ctrl+C)..."
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
sleep 10
|
||||
echo " Proceeding..."
|
||||
echo ""
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v curl >/dev/null 2>&1; then
|
||||
error "curl not found — required for Sonarr API calls"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v jq >/dev/null 2>&1; then
|
||||
error "jq not found — required for JSON parsing"
|
||||
notify "Sonarr cleanup failed on $(hostname) — jq not installed" "Sonarr Cleanup" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
platform_require_cmd \
|
||||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||||
"" "" \
|
||||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||
|
||||
acquire_lock "wait"
|
||||
TMP_DIR="/tmp/sonarr_cleanup_$$"
|
||||
mkdir -p "$TMP_DIR"
|
||||
trap "_release_all_locks; rm -rf $TMP_DIR" EXIT
|
||||
|
||||
if ! command -v docker &>/dev/null; then
|
||||
error "Docker command not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# detect_hosts() sets MY_ID and aliases SONARR_URL, SONARR_API_KEY, SONARR_TV_ROOT
|
||||
detect_hosts
|
||||
|
||||
DOCKER_TIMEOUT=15
|
||||
SONARR_CONTAINER="Sonarr"
|
||||
|
||||
# Build path map from MY_ID's Sonarr path map
|
||||
declare -A ARR_PATH_MAP
|
||||
local_path_map_var="${MY_ID}_SONARR_PATH_MAP"
|
||||
eval "for key in \"\${!${local_path_map_var}[@]}\"; do
|
||||
ARR_PATH_MAP[\"\$key\"]=\"\${${local_path_map_var}[\$key]}\"
|
||||
done"
|
||||
|
||||
require_var SONARR_URL
|
||||
require_var SONARR_API_KEY
|
||||
require_var SONARR_TV_ROOT
|
||||
|
||||
if [[ ! -d "$SONARR_TV_ROOT" ]]; then
|
||||
error "TV root not found: $SONARR_TV_ROOT"
|
||||
notify "Sonarr cleanup failed on $(hostname) — TV root not found: $SONARR_TV_ROOT" \
|
||||
"Sonarr Cleanup" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log "$ICON_GEAR Config: url=${SONARR_URL} root=${SONARR_TV_ROOT}"
|
||||
echo " $MY_ID ($LOCAL_SERVER_NAME) — $SONARR_URL"
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be deleted"
|
||||
[[ "$I_KNOW" == true ]] && warn "OVERRIDE — --i-know-what-im-doing active"
|
||||
[[ "$SKIP_STRIKES" == true ]] && warn "OVERRIDE — --skip-strike-list active — age check bypassed"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_GEAR Sonarr URL: $SONARR_URL"
|
||||
echo "$ICON_GEAR TV root: $SONARR_TV_ROOT"
|
||||
echo "$ICON_TIME Orphan age: ${SONARR_ORPHAN_AGE} days"
|
||||
echo "$ICON_GEAR Max delete: ${SONARR_MAX_DELETE_GB}GB (requires --i-know-what-im-doing)"
|
||||
echo "$ICON_GEAR Sonarr ver: v${SONARR_VERSION_MAJOR} expected"
|
||||
echo "$ICON_GEAR Extensions: ${SONARR_EXTENSIONS[*]}"
|
||||
echo "$ICON_GEAR Protected patterns: ${SONARR_PROTECTED_PATTERNS[*]}"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "$ICON_GEAR I know: $I_KNOW"
|
||||
echo "$ICON_GEAR Skip strikes: $SKIP_STRIKES"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Safety Layer 1 — Container Health ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_SHIELD Safety Checks ━━━"
|
||||
|
||||
CONTAINER_RUNNING=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
|
||||
'{{.State.Running}}' "$SONARR_CONTAINER" 2>/dev/null)
|
||||
if [[ "$CONTAINER_RUNNING" != "true" ]]; then
|
||||
error "$SONARR_CONTAINER is not running — aborting"
|
||||
notify "Sonarr cleanup aborted on $(hostname) — container not running" \
|
||||
"Sonarr Cleanup" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
CONTAINER_HEALTH=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
|
||||
'{{.State.Health.Status}}' "$SONARR_CONTAINER" 2>/dev/null)
|
||||
case "$CONTAINER_HEALTH" in
|
||||
healthy) info "$SONARR_CONTAINER is healthy" ;;
|
||||
"") info "$SONARR_CONTAINER has no health check — proceeding" ;;
|
||||
starting)
|
||||
error "$SONARR_CONTAINER is still starting — aborting"
|
||||
notify "Sonarr cleanup aborted on $(hostname) — container still starting" \
|
||||
"Sonarr Cleanup" "warning"
|
||||
exit 1 ;;
|
||||
unhealthy)
|
||||
error "$SONARR_CONTAINER is unhealthy — aborting"
|
||||
notify "Sonarr cleanup aborted on $(hostname) — container unhealthy" \
|
||||
"Sonarr Cleanup" "warning"
|
||||
exit 1 ;;
|
||||
*) warn "$SONARR_CONTAINER health: $CONTAINER_HEALTH — proceeding with caution" ;;
|
||||
esac
|
||||
|
||||
info "Safety layer 1 passed — container healthy"
|
||||
|
||||
# ==============================================================================================
|
||||
# ── HELPER FUNCTIONS ──────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
sonarr_api() {
|
||||
local endpoint="$1"
|
||||
local response http_code body
|
||||
|
||||
response=$(curl -sf \
|
||||
--max-time 30 \
|
||||
-H "X-Api-Key: $SONARR_API_KEY" \
|
||||
-w "\n%{http_code}" \
|
||||
"${SONARR_URL}/api/v3/${endpoint}" 2>/dev/null)
|
||||
|
||||
http_code=$(echo "$response" | tail -1)
|
||||
body=$(echo "$response" | head -n -1)
|
||||
|
||||
if [[ "$http_code" != "200" ]]; then
|
||||
error "Sonarr API HTTP $http_code for: $endpoint"
|
||||
return 1
|
||||
fi
|
||||
echo "$body"
|
||||
}
|
||||
|
||||
is_video_file() {
|
||||
local ext="${1##*.}"
|
||||
ext="${ext,,}"
|
||||
for valid_ext in "${SONARR_EXTENSIONS[@]}"; do
|
||||
[[ "$ext" == "$valid_ext" ]] && return 0
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
is_protected_file() {
|
||||
local filename
|
||||
filename=$(basename "$1")
|
||||
for pattern in "${SONARR_PROTECTED_PATTERNS[@]}"; do
|
||||
# shellcheck disable=SC2254
|
||||
case "$filename" in
|
||||
$pattern) return 0 ;;
|
||||
esac
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
format_bytes() {
|
||||
local bytes=$1
|
||||
if (( bytes > 1073741824 )); then
|
||||
awk "BEGIN {printf \"%.1fGB\", $bytes / 1073741824}"
|
||||
elif (( bytes > 1048576 )); then
|
||||
awk "BEGIN {printf \"%.1fMB\", $bytes / 1048576}"
|
||||
else
|
||||
echo "${bytes}B"
|
||||
fi
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Pre-flight: Sonarr Import Scan ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Pre-flight: Sonarr Import Scan ━━━"
|
||||
|
||||
# Fetch root folders from Sonarr API and translate container paths to host paths
|
||||
mapfile -t SCAN_ROOTS < <(
|
||||
sonarr_api "rootfolder" | \
|
||||
jq -r '.[].path' 2>/dev/null | \
|
||||
while IFS= read -r cp; do translate_path "$cp"; done
|
||||
)
|
||||
|
||||
if [[ "${#SCAN_ROOTS[@]}" -eq 0 ]]; then
|
||||
error "No root folders returned from Sonarr API — aborting"
|
||||
notify "Sonarr cleanup aborted on $(hostname) — no root folders from API" \
|
||||
"Sonarr Cleanup" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
info "Scan targets (${#SCAN_ROOTS[@]}): ${SCAN_ROOTS[*]}"
|
||||
|
||||
info "Triggering DownloadedEpisodesScan (all root folders)"
|
||||
SCAN_PAYLOAD='{"name": "DownloadedEpisodesScan"}'
|
||||
|
||||
SCAN_RESPONSE=$(curl -sf --max-time 30 -X POST \
|
||||
-H "X-Api-Key: $SONARR_API_KEY" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "$SCAN_PAYLOAD" \
|
||||
"${SONARR_URL}/api/v3/command" 2>/dev/null)
|
||||
|
||||
SCAN_CMD_ID=$(echo "$SCAN_RESPONSE" | jq -r '.id // empty' 2>/dev/null)
|
||||
|
||||
if [[ -z "$SCAN_CMD_ID" ]]; then
|
||||
warn "Could not trigger import scan — proceeding without pre-flight"
|
||||
else
|
||||
info "Import scan queued (command ID: $SCAN_CMD_ID) — waiting for completion..."
|
||||
POLL_TIMEOUT=${SONARR_IMPORT_SCAN_TIMEOUT:-600}
|
||||
POLLED=0
|
||||
while [[ "$POLLED" -lt "$POLL_TIMEOUT" ]]; do
|
||||
SCAN_STATUS=$(curl -sf --max-time 10 \
|
||||
-H "X-Api-Key: $SONARR_API_KEY" \
|
||||
"${SONARR_URL}/api/v3/command/${SCAN_CMD_ID}" 2>/dev/null | \
|
||||
jq -r '.status // empty' 2>/dev/null)
|
||||
case "$SCAN_STATUS" in
|
||||
completed) info "Import scan complete ✅"; break ;;
|
||||
failed) warn "Import scan reported failed — proceeding anyway"; break ;;
|
||||
esac
|
||||
sleep 10
|
||||
(( POLLED += 10 ))
|
||||
[[ $(( POLLED % 60 )) -eq 0 ]] && log " Still scanning... (${POLLED}s elapsed)"
|
||||
done
|
||||
[[ "$POLLED" -ge "$POLL_TIMEOUT" ]] && \
|
||||
warn "Import scan timed out after ${POLL_TIMEOUT}s — proceeding anyway"
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Fetch Sonarr Tracked Files ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Fetching Sonarr Tracked Files ━━━"
|
||||
|
||||
# Safety Layer 2 — API reachability
|
||||
if ! check_api "$SONARR_URL" "Sonarr" 10; then
|
||||
notify "Sonarr cleanup aborted on $(hostname) — API unreachable" "Sonarr Cleanup" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Safety Layer 3 — API version check
|
||||
check_arr_version "$SONARR_URL" "$SONARR_API_KEY" "v3" "$SONARR_VERSION_MAJOR" "Sonarr" || exit 1
|
||||
|
||||
info "Querying Sonarr API..."
|
||||
|
||||
# Fetch all series
|
||||
SERIES_RESPONSE=$(sonarr_api "series") || {
|
||||
error "Failed to fetch series from Sonarr"
|
||||
notify "Sonarr cleanup failed on $(hostname) — could not fetch series" \
|
||||
"Sonarr Cleanup" "warning"
|
||||
exit 1
|
||||
}
|
||||
|
||||
SERIES_IDS=$(echo "$SERIES_RESPONSE" | jq -r '.[].id' 2>/dev/null)
|
||||
SERIES_COUNT=$(echo "$SERIES_IDS" | grep -c "." 2>/dev/null || echo 0)
|
||||
|
||||
# Safety Layer 4 — series count > 0
|
||||
if [[ "$SERIES_COUNT" -eq 0 ]]; then
|
||||
error "API returned 0 series — aborting to prevent mass deletion"
|
||||
notify "Sonarr cleanup aborted on $(hostname) — 0 series returned" \
|
||||
"Sonarr Cleanup" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
info "Found $SERIES_COUNT series — fetching episode files..."
|
||||
|
||||
TRACKED_FILE="$TMP_DIR/tracked_paths.txt"
|
||||
> "$TRACKED_FILE"
|
||||
|
||||
SERIES_INDEX=0
|
||||
while IFS= read -r series_id; do
|
||||
[[ -z "$series_id" ]] && continue
|
||||
(( SERIES_INDEX++ ))
|
||||
[[ $(( SERIES_INDEX % 50 )) -eq 0 ]] && \
|
||||
log "Fetching files: $SERIES_INDEX/$SERIES_COUNT series..."
|
||||
SERIES_FILES=$(sonarr_api "episodefile?seriesId=${series_id}" 2>/dev/null)
|
||||
if [[ -n "$SERIES_FILES" ]]; then
|
||||
while IFS= read -r api_path; do
|
||||
[[ -z "$api_path" ]] && continue
|
||||
translate_path "$api_path" >> "$TRACKED_FILE"
|
||||
done < <(echo "$SERIES_FILES" | jq -r '.[].path' 2>/dev/null)
|
||||
fi
|
||||
done <<< "$SERIES_IDS"
|
||||
|
||||
sort -u "$TRACKED_FILE" -o "$TRACKED_FILE"
|
||||
|
||||
# Build in-memory lookup map — O(1) per lookup vs O(n) grep per file
|
||||
declare -A TRACKED_MAP
|
||||
while IFS= read -r _tracked_path; do
|
||||
[[ -n "$_tracked_path" ]] && TRACKED_MAP["$_tracked_path"]=1
|
||||
done < "$TRACKED_FILE"
|
||||
unset _tracked_path
|
||||
info "Built in-memory lookup map: ${#TRACKED_MAP[@]} tracked paths"
|
||||
TRACKED_COUNT=$(wc -l < "$TRACKED_FILE")
|
||||
|
||||
# Safety Layer 5 — tracked count > 0
|
||||
if [[ "$TRACKED_COUNT" -eq 0 ]]; then
|
||||
error "API returned 0 tracked files — aborting to prevent mass deletion"
|
||||
notify "Sonarr cleanup aborted on $(hostname) — 0 tracked files returned" \
|
||||
"Sonarr Cleanup" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
info "$SERIES_COUNT series | $TRACKED_COUNT tracked episode files"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Scan TV Root ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_CLEAN Scanning TV Root ━━━"
|
||||
info "Root: $SONARR_TV_ROOT | Orphan age: ${SONARR_ORPHAN_AGE} days"
|
||||
|
||||
START=$(date +%s)
|
||||
ORPHAN_COUNT=0
|
||||
JUNK_COUNT=0
|
||||
RECENT_COUNT=0
|
||||
PROTECTED_COUNT=0
|
||||
ORPHAN_BYTES=0
|
||||
JUNK_BYTES=0
|
||||
|
||||
AGE_SECONDS=$(( SONARR_ORPHAN_AGE * 86400 ))
|
||||
NOW=$(date +%s)
|
||||
MAX_DELETE_BYTES=$(awk "BEGIN {printf \"%d\", $SONARR_MAX_DELETE_GB * 1073741824}")
|
||||
|
||||
while IFS= read -r filepath; do
|
||||
[[ -z "$filepath" ]] && continue
|
||||
|
||||
if [[ -n "${TRACKED_MAP[$filepath]:-}" ]]; then
|
||||
log "TRACKED: $filepath"
|
||||
continue
|
||||
fi
|
||||
|
||||
if is_protected_file "$filepath"; then
|
||||
log "$ICON_PROTECTED PROTECTED: $filepath"
|
||||
(( PROTECTED_COUNT++ ))
|
||||
continue
|
||||
fi
|
||||
|
||||
FILE_SIZE=$(stat -c%s "$filepath" 2>/dev/null || echo 0)
|
||||
|
||||
if is_video_file "$filepath"; then
|
||||
FILE_MTIME=$(stat -c %Y "$filepath" 2>/dev/null || echo 0)
|
||||
FILE_AGE=$(( NOW - FILE_MTIME ))
|
||||
|
||||
if [[ "$FILE_AGE" -lt "$AGE_SECONDS" ]] && [[ "$SKIP_STRIKES" != true ]]; then
|
||||
log "RECENT (skipping): $filepath"
|
||||
(( RECENT_COUNT++ ))
|
||||
continue
|
||||
fi
|
||||
|
||||
warn "$ICON_TRASH ORPHAN: $filepath"
|
||||
(( ORPHAN_COUNT++ ))
|
||||
ORPHAN_BYTES=$(( ORPHAN_BYTES + FILE_SIZE ))
|
||||
else
|
||||
log "JUNK: $filepath"
|
||||
(( JUNK_COUNT++ ))
|
||||
JUNK_BYTES=$(( JUNK_BYTES + FILE_SIZE ))
|
||||
fi
|
||||
|
||||
done < <(
|
||||
for host_path in "${SCAN_ROOTS[@]}"; do
|
||||
[[ -d "$host_path" ]] && find "$host_path" -type f 2>/dev/null
|
||||
done | sort -u
|
||||
)
|
||||
|
||||
TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES ))
|
||||
TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT ))
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Safety Layer 6 — Deletion Size Threshold ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$TOTAL_DELETE_BYTES" -gt "$MAX_DELETE_BYTES" ]]; then
|
||||
TOTAL_HUMAN=$(awk "BEGIN {printf \"%.1fGB\", $TOTAL_DELETE_BYTES / 1073741824}")
|
||||
if [[ "$I_KNOW" != true ]]; then
|
||||
echo ""
|
||||
error "Deletion would exceed ${SONARR_MAX_DELETE_GB}GB — $TOTAL_HUMAN would be deleted"
|
||||
error "Review ORPHAN lines above carefully before proceeding"
|
||||
error "Rerun with: --i-know-what-im-doing"
|
||||
error "To also bypass age check: add --skip-strike-list"
|
||||
notify "Sonarr cleanup halted on $(hostname) — ${TOTAL_HUMAN} requires --i-know-what-im-doing" \
|
||||
"Sonarr Cleanup" "warning"
|
||||
exit 1
|
||||
else
|
||||
warn "OVERRIDE — deletion is $TOTAL_HUMAN — proceeding with --i-know-what-im-doing"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Execute Deletions ─────────────────────────────────────────────────────────────────────────
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
while IFS= read -r filepath; do
|
||||
[[ -z "$filepath" ]] && continue
|
||||
[[ -n "${TRACKED_MAP[$filepath]:-}" ]] && continue
|
||||
is_protected_file "$filepath" && continue
|
||||
|
||||
FILE_MTIME=$(stat -c %Y "$filepath" 2>/dev/null || echo 0)
|
||||
FILE_AGE=$(( NOW - FILE_MTIME ))
|
||||
|
||||
if is_video_file "$filepath"; then
|
||||
[[ "$FILE_AGE" -lt "$AGE_SECONDS" ]] && \
|
||||
[[ "$SKIP_STRIKES" != true ]] && continue
|
||||
fi
|
||||
|
||||
rm -f "$filepath" 2>/dev/null || error "Failed to delete: $filepath"
|
||||
|
||||
done < <(
|
||||
for host_path in "${SCAN_ROOTS[@]}"; do
|
||||
[[ -d "$host_path" ]] && find "$host_path" -type f 2>/dev/null
|
||||
done | sort -u
|
||||
)
|
||||
|
||||
info "Cleaning up empty folders..."
|
||||
for host_path in "${SCAN_ROOTS[@]}"; do
|
||||
[[ -d "$host_path" ]] && \
|
||||
find "$host_path" -mindepth 1 -type d -empty -delete 2>/dev/null
|
||||
done
|
||||
info "Empty folders removed"
|
||||
fi
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
ORPHAN_HUMAN=$(format_bytes "$ORPHAN_BYTES")
|
||||
JUNK_HUMAN=$(format_bytes "$JUNK_BYTES")
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY SONARR CLEANUP SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_SYNC Tracked: $TRACKED_COUNT files ($SERIES_COUNT series)"
|
||||
echo "$ICON_SHIELD Protected: $PROTECTED_COUNT files (artwork, subtitles, metadata)"
|
||||
echo "$ICON_TRASH Orphans: $ORPHAN_COUNT files ($ORPHAN_HUMAN)"
|
||||
echo "$ICON_TRASH Junk: $JUNK_COUNT files ($JUNK_HUMAN)"
|
||||
echo "$ICON_SKIP Recent skipped: $RECENT_COUNT files (under ${SONARR_ORPHAN_AGE} days)"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no files deleted"
|
||||
elif [[ "$TOTAL_REMOVED" -eq 0 ]]; then
|
||||
echo "$ICON_DONE Clean — nothing to remove"
|
||||
else
|
||||
warn "$ICON_DONE Removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)"
|
||||
notify "Sonarr cleanup on $(hostname) — removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" \
|
||||
"Sonarr Cleanup" "warning"
|
||||
# Notify Emby to clean missing files — removes ghost entries immediately
|
||||
notify_emby_scan
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
# Write stats for sunday_morning_coffee_report.sh
|
||||
if [[ "$DRY_RUN" == false ]] && [[ -n "${ARR_CLEANUP_STATS:-}" ]]; then
|
||||
echo "$(date '+%Y-%m-%d')|sonarr|${ORPHAN_COUNT}|${ORPHAN_BYTES}|${JUNK_COUNT}|${JUNK_BYTES}|${RECENT_COUNT}|${TRACKED_COUNT}" \
|
||||
>> "$ARR_CLEANUP_STATS" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
exit 0
|
||||
@@ -0,0 +1,489 @@
|
||||
<style>
|
||||
.vv-wd-card { background:#161616;border:1px solid #2a2a2a;border-radius:6px;padding:10px;min-width:0; }
|
||||
.vv-wd-sec { font-size:10px;font-weight:bold;color:#444;letter-spacing:.07em;text-transform:uppercase;margin-bottom:6px; }
|
||||
.vv-wd-row { display:flex;justify-content:space-between;align-items:baseline;gap:6px;margin:2px 0; }
|
||||
.vv-wd-lbl { font-size:11px;color:#444;white-space:nowrap; }
|
||||
.vv-wd-val { font-size:12px;color:#bbb;text-align:right; }
|
||||
.vv-wd-sep { border:none;border-top:1px solid #1e1e1e;margin:6px 0; }
|
||||
.vv-wd-pill { font-size:10px;padding:1px 6px;border-radius:2px;background:#1e1e1e;color:#666;border:1px solid #272727; }
|
||||
.vv-wd-pill.ok { background:#0d1f0d;color:#4caf50;border-color:#1a3a1a; }
|
||||
.vv-wd-pill.warn { background:#1f1500;color:#ffb74d;border-color:#3a2800; }
|
||||
.vv-wd-pill.err { background:#200d0d;color:#ef5350;border-color:#3a1a1a; }
|
||||
.vv-wd-pill-row { display:flex;flex-wrap:wrap;gap:4px;margin-top:4px; }
|
||||
.vv-wd-bar { height:5px;border-radius:2px;background:#1e1e1e;margin-top:3px;overflow:hidden; }
|
||||
.vv-wd-bar-fill{ height:100%;border-radius:2px;transition:width .3s; }
|
||||
.vv-wd-badge { font-size:11px;font-weight:bold;padding:2px 8px;border-radius:3px; }
|
||||
.vv-wd-badge.ok { background:#0d1f0d;color:#4caf50; }
|
||||
.vv-wd-badge.soft { background:#1f1f00;color:#cddc39; }
|
||||
.vv-wd-badge.med { background:#1f1000;color:#ffb74d; }
|
||||
.vv-wd-badge.hard { background:#200d0d;color:#ef5350; }
|
||||
.vv-wd-node-h { display:flex;align-items:center;gap:8px;margin-bottom:10px; }
|
||||
.vv-wd-node-id { font-size:12px;font-weight:bold;color:#666;letter-spacing:.06em;text-transform:uppercase; }
|
||||
.vv-wd-dot { width:6px;height:6px;border-radius:50%;flex-shrink:0; }
|
||||
.vv-wd-strike-row { display:flex;align-items:baseline;gap:6px;margin:2px 0; }
|
||||
.vv-wd-strike-name{ font-size:11px;color:#777;flex:1; }
|
||||
.vv-wd-strike-cnt { font-size:11px;font-weight:bold;color:#ffb74d; }
|
||||
.vv-wd-reboot-ts { font-size:11px;color:#555;margin:1px 0; }
|
||||
.vv-wd-ctr-row { display:flex;justify-content:space-between;align-items:baseline;margin:2px 0; }
|
||||
.vv-wd-ctr-name{ font-size:11px;color:#888; }
|
||||
.vv-wd-ctr-lim { font-size:11px;color:#555; }
|
||||
.vv-wd-pressure{ grid-column:1/-1;border-color:#3a2000;background:#1a1000; }
|
||||
|
||||
/* One host per row — inner grid sizes all cards equally */
|
||||
.vv-wd-host-row {
|
||||
grid-column: 1 / -1;
|
||||
display: grid;
|
||||
grid-template-columns: repeat(5, 1fr);
|
||||
gap: 12px;
|
||||
}
|
||||
@media (max-width: 900px) {
|
||||
.vv-wd-host-row { grid-template-columns: 1fr; }
|
||||
}
|
||||
</style>
|
||||
|
||||
<div style="display:flex;justify-content:space-between;align-items:center;margin-bottom:12px;padding:0 2px;">
|
||||
<span style="font-size:13px;font-weight:bold;color:#888;text-transform:uppercase;letter-spacing:.06em;">Watchdog</span>
|
||||
<span style="font-size:11px;color:#3a3a3a;" id="vv-wd-ts"></span>
|
||||
</div>
|
||||
|
||||
<div id="vv-wd-grid" style="display:grid;grid-template-columns:repeat(8,1fr);gap:12px;">
|
||||
<div style="grid-column:1/-1;color:#444;font-size:12px;padding:16px 0;text-align:center;">Loading…</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
(function() {
|
||||
|
||||
function _dur(s) {
|
||||
const d = Math.floor(s / 86400), h = Math.floor((s % 86400) / 3600), m = Math.floor((s % 3600) / 60);
|
||||
if (d) return d + 'd ' + h + 'h';
|
||||
if (h) return h + 'h ' + m + 'm';
|
||||
return m + 'm';
|
||||
}
|
||||
|
||||
function _row(lbl, val) {
|
||||
return `<div class="vv-wd-row"><span class="vv-wd-lbl">${lbl}</span><span class="vv-wd-val">${val}</span></div>`;
|
||||
}
|
||||
|
||||
function _bar(pct, col) {
|
||||
return `<div class="vv-wd-bar"><div class="vv-wd-bar-fill" style="width:${Math.min(pct,100)}%;background:${col}"></div></div>`;
|
||||
}
|
||||
|
||||
function _pill(label, cls) {
|
||||
return `<span class="vv-wd-pill ${cls}">${label}</span>`;
|
||||
}
|
||||
|
||||
function _levelLabel(level) {
|
||||
return ['OK', 'SOFT', 'MEDIUM', 'HARD'][level] || '?';
|
||||
}
|
||||
function _levelCls(level) {
|
||||
return ['ok', 'soft', 'med', 'hard'][level] || 'ok';
|
||||
}
|
||||
|
||||
// ── Pressure alert card ───────────────────────────────────────────────────────
|
||||
function _pressureCard(node) {
|
||||
const st = node.states;
|
||||
if (!st || st.rw_level === 0) return '';
|
||||
const level = st.rw_level;
|
||||
const cls = _levelCls(level);
|
||||
const label = _levelLabel(level);
|
||||
|
||||
const paused = (st.rw_paused || []).filter(Boolean);
|
||||
const stopped = (st.rw_stopped || []).filter(Boolean);
|
||||
const pausedHtml = paused.length ? paused.map(c => _pill(c, 'warn')).join('') : '';
|
||||
const stoppedHtml = stopped.length ? stopped.map(c => _pill(c, 'err')).join('') : '';
|
||||
|
||||
return `<div class="vv-wd-card vv-wd-pressure" style="grid-column:1/-1">
|
||||
<div style="display:flex;align-items:center;gap:10px;margin-bottom:8px;">
|
||||
<span class="vv-wd-badge ${cls}">PRESSURE ${label}</span>
|
||||
<span style="font-size:11px;color:#7a5020;">${node.id} (${node.hostname})</span>
|
||||
${st.mem_shutdown ? `<span class="vv-wd-badge hard" style="margin-left:auto;">MEM SHUTDOWN ACTIVE</span>` : ''}
|
||||
</div>
|
||||
${paused.length ? `<div style="margin-bottom:4px;"><span class="vv-wd-lbl">Paused:</span> <span class="vv-wd-pill-row" style="display:inline-flex;">${pausedHtml}</span></div>` : ''}
|
||||
${stopped.length ? `<div><span class="vv-wd-lbl">Stopped:</span> <span class="vv-wd-pill-row" style="display:inline-flex;">${stoppedHtml}</span></div>` : ''}
|
||||
</div>`;
|
||||
}
|
||||
|
||||
// ── System health card ────────────────────────────────────────────────────────
|
||||
function _systemCard(node, cfg) {
|
||||
const sys = node.system;
|
||||
if (!sys) {
|
||||
return `<div class="vv-wd-card">
|
||||
<div class="vv-wd-node-h">
|
||||
<span class="vv-wd-dot" style="background:#444"></span>
|
||||
<span class="vv-wd-node-id">${node.id}</span>
|
||||
<span style="font-size:11px;color:#3a3a3a;">${node.hostname}</span>
|
||||
<span class="vv-wd-badge" style="margin-left:auto;background:#1a1a1a;color:#444">UNREACHABLE</span>
|
||||
</div>
|
||||
</div>`;
|
||||
}
|
||||
|
||||
const memGb = sys.mem_avail / GB;
|
||||
const memTotGb = sys.mem_total / GB;
|
||||
const usedPct = memTotGb > 0 ? ((memTotGb - memGb) / memTotGb) * 100 : 0;
|
||||
const memCol = memGb < cfg.sys_mem_gb ? '#ef5350'
|
||||
: memGb < cfg.rw_hard_gb ? '#ef5350'
|
||||
: memGb < cfg.rw_medium_gb ? '#ffb74d'
|
||||
: memGb < cfg.rw_soft_gb ? '#cddc39' : '#4caf50';
|
||||
|
||||
const loadPct = sys.cores > 0 ? (sys.load1 / (sys.cores * cfg.rw_load_med)) * 100 : 0;
|
||||
const loadCol = sys.load1 > sys.cores * cfg.rw_load_med ? '#ef5350'
|
||||
: sys.load1 > sys.cores * cfg.rw_load_soft ? '#ffb74d'
|
||||
: '#4caf50';
|
||||
|
||||
const apiOnly = sys.api_only === true;
|
||||
const daemonDot = sys.daemon_ok === null ? '#555' : sys.daemon_ok ? '#4caf50' : '#ef5350';
|
||||
const daemonTxt = sys.daemon_ok === null ? '—' : sys.daemon_ok ? 'daemon ok' : 'daemon err';
|
||||
|
||||
const st = node.states || {};
|
||||
const level = st.rw_level || 0;
|
||||
const dotCol = level >= 3 ? '#ef5350' : level >= 2 ? '#ffb74d' : level >= 1 ? '#cddc39'
|
||||
: apiOnly ? '#4a7a9b' // blue-grey: API-only, no watchdog state
|
||||
: '#4caf50';
|
||||
|
||||
return `<div class="vv-wd-card">
|
||||
<div class="vv-wd-node-h">
|
||||
<span class="vv-wd-dot" style="background:${dotCol}"></span>
|
||||
<span class="vv-wd-node-id">${node.id}</span>
|
||||
<span style="font-size:11px;color:#3a3a3a;">${node.hostname}</span>
|
||||
<span class="vv-wd-badge ${apiOnly ? '' : _levelCls(level)}"
|
||||
style="margin-left:auto;${apiOnly ? 'background:#0d1f2a;color:#4a9eff;' : ''}"
|
||||
>${apiOnly ? 'API ONLY' : _levelLabel(level)}</span>
|
||||
</div>
|
||||
<div class="vv-wd-sec">System</div>
|
||||
${_row('RAM free', `<span style="color:${memCol}">${_fmtBytes(sys.mem_avail)}</span> / ${_fmtBytes(sys.mem_total)}`)}
|
||||
${_bar(usedPct, memCol)}
|
||||
<div style="display:flex;justify-content:space-between;margin-top:1px;font-size:10px;color:#333">
|
||||
<span>free</span>
|
||||
<span>${cfg.rw_soft_gb}G soft · ${cfg.rw_hard_gb}G hard · ${cfg.sys_mem_gb}G reboot</span>
|
||||
</div>
|
||||
<div style="height:5px"></div>
|
||||
${!apiOnly ? `${_row('Load avg', `<span style="color:${loadCol}">${sys.load1.toFixed(2)}</span> / ${sys.cores} cores`)}
|
||||
${_bar(loadPct, loadCol)}
|
||||
<div style="height:5px"></div>` : ''}
|
||||
${_row('Uptime', sys.uptime ? _dur(sys.uptime) : '—')}
|
||||
${_row('Docker', `<span style="color:${daemonDot}">${daemonTxt}</span>`)}
|
||||
${!apiOnly ? (sys.oom_count > 0 ? _row('OOM kills', `<span style="color:#ef5350">${sys.oom_count}</span>`) : _row('OOM kills', '<span style="color:#333">0</span>')) : ''}
|
||||
${apiOnly ? `<div style="font-size:10px;color:#333;margin-top:6px;">Watchdog state unavailable — SSH not configured</div>` : ''}
|
||||
</div>`;
|
||||
}
|
||||
|
||||
// ── Docker watchdog state card ────────────────────────────────────────────────
|
||||
function _dockerCard(node, cfg) {
|
||||
const st = node.states;
|
||||
if (!st) return `<div class="vv-wd-card"><div class="vv-wd-sec">Docker Watchdog</div><div style="color:#3a3a3a;font-size:11px;padding:8px 0;">No data</div></div>`;
|
||||
|
||||
const strikes = Object.entries(st.ctr_strikes || {});
|
||||
const skiplist = st.skiplist || [];
|
||||
const restarts = (st.restarts || []).slice(0, 10);
|
||||
const daemonCls = st.daemon_strikes > 0 ? 'err' : 'ok';
|
||||
const allOk = strikes.length === 0 && skiplist.length === 0 && st.daemon_strikes === 0;
|
||||
|
||||
// Group restarts by container for last-24h summary
|
||||
const rCounts = {};
|
||||
for (const r of restarts) {
|
||||
rCounts[r.name] = (rCounts[r.name] || 0) + 1;
|
||||
}
|
||||
|
||||
let strikesHtml = '';
|
||||
if (strikes.length === 0 && st.daemon_strikes === 0) {
|
||||
strikesHtml = '<div style="color:#333;font-size:11px;">0 active strikes</div>';
|
||||
} else {
|
||||
if (st.daemon_strikes > 0) {
|
||||
strikesHtml += `<div class="vv-wd-strike-row"><span class="vv-wd-strike-name">daemon</span><span class="vv-wd-strike-cnt">${st.daemon_strikes}</span></div>`;
|
||||
}
|
||||
for (const [name, cnt] of strikes) {
|
||||
strikesHtml += `<div class="vv-wd-strike-row"><span class="vv-wd-strike-name">${name}</span><span class="vv-wd-strike-cnt">${cnt} / ${cfg.cpu_fail_lim}</span></div>`;
|
||||
}
|
||||
}
|
||||
|
||||
let skipHtml = '';
|
||||
if (skiplist.length === 0) {
|
||||
skipHtml = '<div style="color:#333;font-size:11px;">empty</div>';
|
||||
} else {
|
||||
skipHtml = `<div class="vv-wd-pill-row">${skiplist.map(c => _pill(c, 'err')).join('')}</div>`;
|
||||
}
|
||||
|
||||
let restartHtml = '';
|
||||
const rcEntries = Object.entries(rCounts);
|
||||
if (rcEntries.length === 0) {
|
||||
restartHtml = '<div style="color:#333;font-size:11px;">none (24h)</div>';
|
||||
} else {
|
||||
restartHtml = rcEntries.map(([n, c]) =>
|
||||
`<div class="vv-wd-row"><span class="vv-wd-lbl">${n}</span><span class="vv-wd-val" style="color:${c >= cfg.restart_limit ? '#ef5350' : '#ffb74d'}">${c}×</span></div>`
|
||||
).join('');
|
||||
}
|
||||
|
||||
return `<div class="vv-wd-card">
|
||||
<div class="vv-wd-sec">Docker Watchdog</div>
|
||||
<div style="display:flex;gap:6px;margin-bottom:8px;">
|
||||
${_pill(st.daemon_restart ? 'daemon restarted' : 'daemon ok', daemonCls)}
|
||||
${allOk ? _pill('all clear', 'ok') : ''}
|
||||
</div>
|
||||
<div class="vv-wd-sec">Strikes</div>
|
||||
${strikesHtml}
|
||||
<hr class="vv-wd-sep">
|
||||
<div class="vv-wd-sec">Skip list</div>
|
||||
${skipHtml}
|
||||
<hr class="vv-wd-sep">
|
||||
<div class="vv-wd-sec">Restarts (24h)</div>
|
||||
${restartHtml}
|
||||
</div>`;
|
||||
}
|
||||
|
||||
// ── Stability / reboot card ───────────────────────────────────────────────────
|
||||
function _stabilityCard(node, cfg) {
|
||||
const st = node.states;
|
||||
if (!st) return `<div class="vv-wd-card"><div class="vv-wd-sec">Stability</div><div style="color:#3a3a3a;font-size:11px;padding:8px 0;">No data</div></div>`;
|
||||
|
||||
const reboots = st.reboots || [];
|
||||
const sysStr = Object.entries(st.sys_strikes || {});
|
||||
const rebootCls = reboots.length >= cfg.reboot_limit ? 'err' : reboots.length > 0 ? 'warn' : 'ok';
|
||||
|
||||
let sysHtml = '';
|
||||
if (sysStr.length === 0) {
|
||||
sysHtml = '<div style="color:#333;font-size:11px;">0 active strikes</div>';
|
||||
} else {
|
||||
sysHtml = sysStr.map(([k, v]) =>
|
||||
`<div class="vv-wd-strike-row"><span class="vv-wd-strike-name" style="font-size:10px;">${k.replace(/_/g,' ')}</span><span class="vv-wd-strike-cnt">${v}</span></div>`
|
||||
).join('');
|
||||
}
|
||||
|
||||
const rebootHtml = reboots.length === 0
|
||||
? '<div style="color:#333;font-size:11px;">none (12h)</div>'
|
||||
: reboots.map(ts => `<div class="vv-wd-reboot-ts">${_relTime(ts)}</div>`).join('');
|
||||
|
||||
return `<div class="vv-wd-card">
|
||||
<div class="vv-wd-sec">Stability</div>
|
||||
<div style="display:flex;gap:6px;margin-bottom:8px;">
|
||||
${_pill(`${reboots.length} / ${cfg.reboot_limit} reboots`, rebootCls)}
|
||||
${_pill(`${cfg.reboot_window}h window`, '')}
|
||||
</div>
|
||||
<div class="vv-wd-sec">Strikes</div>
|
||||
${sysHtml}
|
||||
<hr class="vv-wd-sep">
|
||||
<div class="vv-wd-sec">Reboot history</div>
|
||||
${rebootHtml}
|
||||
</div>`;
|
||||
}
|
||||
|
||||
// ── Config inventory card ─────────────────────────────────────────────────────
|
||||
function _configCard(node) {
|
||||
const cfg = node.config || {};
|
||||
const mon = Object.entries(cfg.monitored || {});
|
||||
const req = cfg.required || [];
|
||||
const ign = cfg.ignore || [];
|
||||
const paus = cfg.pause_list|| [];
|
||||
const stop = cfg.stop_list || [];
|
||||
const crit = cfg.critical || [];
|
||||
|
||||
const monHtml = mon.length === 0
|
||||
? '<div style="color:#333;font-size:11px;">none</div>'
|
||||
: mon.map(([name, mb]) => {
|
||||
const gb = (mb / 1024).toFixed(0);
|
||||
return `<div class="vv-wd-ctr-row"><span class="vv-wd-ctr-name">${name}</span><span class="vv-wd-ctr-lim">${gb} GB</span></div>`;
|
||||
}).join('');
|
||||
|
||||
const reqHtml = req.length === 0
|
||||
? '<div style="color:#333;font-size:11px;">none</div>'
|
||||
: `<div class="vv-wd-pill-row">${req.map(c => _pill(c, crit.includes(c) ? '' : '')).join('')}</div>`;
|
||||
|
||||
const ignHtml = ign.length === 0
|
||||
? '<div style="color:#3a3a3a;font-size:11px;">none</div>'
|
||||
: `<div class="vv-wd-pill-row">${ign.map(c => _pill(c, '')).join('')}</div>`;
|
||||
|
||||
const pausHtml = paus.length === 0
|
||||
? '<div style="color:#3a3a3a;font-size:11px;">none</div>'
|
||||
: `<div class="vv-wd-pill-row">${paus.map(c => _pill(c, 'warn')).join('')}</div>`;
|
||||
|
||||
const stopHtml = stop.length === 0
|
||||
? '<div style="color:#3a3a3a;font-size:11px;">none</div>'
|
||||
: `<div class="vv-wd-pill-row">${stop.map(c => _pill(c, 'err')).join('')}</div>`;
|
||||
|
||||
return `<div class="vv-wd-card" style="grid-column:span 4;">
|
||||
<div style="display:flex;gap:6px;align-items:center;margin-bottom:8px;">
|
||||
<span class="vv-wd-node-id">${node.id}</span>
|
||||
<span style="font-size:11px;color:#3a3a3a;">${node.hostname}</span>
|
||||
</div>
|
||||
<div style="display:grid;grid-template-columns:1fr 1fr;gap:10px;">
|
||||
<div>
|
||||
<div class="vv-wd-sec">Mem limits (Tier 1)</div>
|
||||
${monHtml}
|
||||
<div style="height:8px"></div>
|
||||
<div class="vv-wd-sec">Required</div>
|
||||
${reqHtml}
|
||||
</div>
|
||||
<div>
|
||||
<div class="vv-wd-sec">Pause at medium pressure</div>
|
||||
${pausHtml}
|
||||
<div style="height:8px"></div>
|
||||
<div class="vv-wd-sec">Stop at hard pressure</div>
|
||||
${stopHtml}
|
||||
<div style="height:8px"></div>
|
||||
<div class="vv-wd-sec">Scan ignore</div>
|
||||
${ignHtml}
|
||||
</div>
|
||||
</div>
|
||||
</div>`;
|
||||
}
|
||||
|
||||
// ── Storage watchdog card ─────────────────────────────────────────────────────
|
||||
function _storageCard(node, cfg) {
|
||||
const st = node.states;
|
||||
const nodeCfg = node.config || {};
|
||||
if (!st) return `<div class="vv-wd-card"><div class="vv-wd-sec">Storage Watchdog</div><div style="color:#3a3a3a;font-size:11px;padding:8px 0;">No data</div></div>`;
|
||||
|
||||
const storWd = st.storage_wd || {};
|
||||
const growthStr = Object.entries(storWd.growth_strikes || {});
|
||||
const logStr = Object.entries(storWd.log_strikes || {});
|
||||
const totalIssues = growthStr.length + logStr.length;
|
||||
const allClear = totalIssues === 0;
|
||||
|
||||
// Baseline info
|
||||
const bCount = storWd.baseline_count ?? 0;
|
||||
const bAge = storWd.baseline_age_sec;
|
||||
let baselineNote = bCount > 0
|
||||
? `${bCount} containers tracked`
|
||||
: 'no baseline yet (builds after first cycle)';
|
||||
if (bAge != null && bCount > 0) {
|
||||
const bAgeStr = bAge < 120 ? bAge + 's ago' : bAge < 3600 ? Math.floor(bAge/60) + 'm ago' : Math.floor(bAge/3600) + 'h ago';
|
||||
baselineNote += ` · updated ${bAgeStr}`;
|
||||
}
|
||||
|
||||
// Suppress ceilings configured for this host
|
||||
const sizes = Object.entries(nodeCfg.appdata_sizes || {});
|
||||
const sizesHtml = sizes.length
|
||||
? sizes.map(([c, mb]) => _pill(`${c} <${Math.round(mb/1024)}GB`, '')).join('')
|
||||
: '';
|
||||
|
||||
let growthHtml = '';
|
||||
if (growthStr.length === 0) {
|
||||
growthHtml = '<div style="color:#333;font-size:11px;">no active strikes</div>';
|
||||
} else {
|
||||
growthHtml = growthStr.map(([name, cnt]) =>
|
||||
`<div class="vv-wd-strike-row">
|
||||
<span class="vv-wd-strike-name">${name}</span>
|
||||
<span class="vv-wd-strike-cnt">${cnt} / ${cfg.stor_strike_lim}</span>
|
||||
</div>`
|
||||
).join('');
|
||||
}
|
||||
|
||||
let logHtml = '';
|
||||
if (logStr.length === 0) {
|
||||
logHtml = '<div style="color:#333;font-size:11px;">no active strikes</div>';
|
||||
} else {
|
||||
logHtml = logStr.map(([key, cnt]) => {
|
||||
const display = key.length > 36 ? '…' + key.slice(-36) : key;
|
||||
return `<div class="vv-wd-strike-row">
|
||||
<span class="vv-wd-strike-name" style="font-size:10px;" title="${key}">${display}</span>
|
||||
<span class="vv-wd-strike-cnt">${cnt} / ${cfg.stor_strike_lim}</span>
|
||||
</div>`;
|
||||
}).join('');
|
||||
}
|
||||
|
||||
return `<div class="vv-wd-card">
|
||||
<div class="vv-wd-sec">Storage Watchdog</div>
|
||||
<div style="display:flex;gap:5px;flex-wrap:wrap;margin-bottom:6px;">
|
||||
${allClear ? _pill('all clear', 'ok') : _pill(totalIssues + ' active strike' + (totalIssues !== 1 ? 's' : ''), 'warn')}
|
||||
${_pill('growth >' + cfg.growth_gb + 'GB/cycle', '')}
|
||||
${_pill('log max ' + cfg.log_max_gb + 'GB', '')}
|
||||
${cfg.truncate_logs ? _pill('auto-truncate on', 'ok') : _pill('auto-truncate off', '')}
|
||||
</div>
|
||||
<div style="font-size:10px;color:#3a3a3a;margin-bottom:8px;">${baselineNote}</div>
|
||||
<div style="display:grid;grid-template-columns:1fr 1fr;gap:12px;">
|
||||
<div>
|
||||
<div class="vv-wd-sec">Growth strikes</div>
|
||||
${growthHtml}
|
||||
</div>
|
||||
<div>
|
||||
<div class="vv-wd-sec">Log size strikes</div>
|
||||
${logHtml}
|
||||
</div>
|
||||
</div>
|
||||
${sizes.length ? `<hr class="vv-wd-sep"><div class="vv-wd-sec">Suppress ceilings (this host)</div><div class="vv-wd-pill-row">${sizesHtml}</div>` : ''}
|
||||
</div>`;
|
||||
}
|
||||
|
||||
// ── Network watchdog card ─────────────────────────────────────────────────────
|
||||
function _networkCard(node, cfg) {
|
||||
const st = node.states;
|
||||
const nodeCfg = node.config || {};
|
||||
if (!st) return `<div class="vv-wd-card"><div class="vv-wd-sec">Network Watchdog</div><div style="color:#3a3a3a;font-size:11px;padding:8px 0;">No data</div></div>`;
|
||||
|
||||
const netWd = st.network_wd || {};
|
||||
const npmStr = netWd.npm_strikes ?? 0;
|
||||
const npmCls = npmStr >= cfg.npm_strike_lim ? 'err' : npmStr > 0 ? 'warn' : 'ok';
|
||||
const ddnsDomain = nodeCfg.ddns_domain || '';
|
||||
const ddnsCtr = nodeCfg.ddns_container || '';
|
||||
const npmUrl = nodeCfg.npm_url || '';
|
||||
|
||||
const ddnsHtml = ddnsDomain
|
||||
? `${_row('DDNS domain', `<span style="color:#888;">${ddnsDomain}</span>`)}
|
||||
${ddnsCtr ? _row('DDNS container', `<span style="color:#888;">${ddnsCtr}</span>`) : ''}`
|
||||
: _row('DDNS', '<span style="color:#444;">not configured for this host</span>');
|
||||
|
||||
const npmHtml = npmUrl
|
||||
? `${_row('NPM URL', `<span style="color:#888;font-size:10px;">${npmUrl}</span>`)}
|
||||
${_row('NPM strikes', `<span class="vv-wd-pill ${npmCls}" style="font-size:10px;">${npmStr} / ${cfg.npm_strike_lim}</span>`)}`
|
||||
: _row('NPM check', '<span style="color:#444;">not configured for this host</span>');
|
||||
|
||||
return `<div class="vv-wd-card">
|
||||
<div class="vv-wd-sec">Network Watchdog</div>
|
||||
<div style="display:flex;gap:5px;flex-wrap:wrap;margin-bottom:8px;">
|
||||
${cfg.net_wd_enabled ? _pill('enabled', 'ok') : _pill('disabled', '')}
|
||||
${_pill('Tailscale check ' + (cfg.ts_check ? 'on' : 'off'), cfg.ts_check ? '' : '')}
|
||||
${npmUrl ? _pill('NPM ' + npmStr + '/' + cfg.npm_strike_lim + ' strikes', npmCls) : ''}
|
||||
</div>
|
||||
${ddnsHtml}
|
||||
<hr class="vv-wd-sep">
|
||||
${npmHtml}
|
||||
</div>`;
|
||||
}
|
||||
|
||||
// ── Main render ───────────────────────────────────────────────────────────────
|
||||
function _render(data) {
|
||||
const nodes = data.nodes || [];
|
||||
const cfg = data.cfg || {};
|
||||
let html = '';
|
||||
|
||||
// Per-host: pressure alert (if active) then all 5 watchdog cards in one equal-spaced row
|
||||
for (const node of nodes) {
|
||||
html += _pressureCard(node);
|
||||
html += `<div class="vv-wd-host-row">
|
||||
${_systemCard(node, cfg)}
|
||||
${_dockerCard(node, cfg)}
|
||||
${_stabilityCard(node, cfg)}
|
||||
${_storageCard(node, cfg)}
|
||||
${_networkCard(node, cfg)}
|
||||
</div>`;
|
||||
}
|
||||
|
||||
// Config inventory — separate row, each node span 4 (2 nodes = full row)
|
||||
for (const node of nodes) html += _configCard(node);
|
||||
|
||||
if (!html) html = '<div style="grid-column:1/-1;color:#444;font-size:12px;padding:16px 0;text-align:center;">No nodes configured.</div>';
|
||||
|
||||
document.getElementById('vv-wd-grid').innerHTML = html;
|
||||
|
||||
const ts = data.ts
|
||||
? new Date(data.ts * 1000).toLocaleString([], {
|
||||
month:'numeric', day:'numeric', year:'numeric',
|
||||
hour:'2-digit', minute:'2-digit', second:'2-digit'})
|
||||
: '';
|
||||
document.getElementById('vv-wd-ts').textContent = ts ? 'Updated: ' + ts : '';
|
||||
}
|
||||
|
||||
function vvWdLoad() {
|
||||
fetch('/plugins/varaverk/api/watchdog.php')
|
||||
.then(r => r.json())
|
||||
.then(_render)
|
||||
.catch(() => {
|
||||
document.getElementById('vv-wd-grid').innerHTML =
|
||||
'<div style="grid-column:1/-1;color:#555;font-size:12px;padding:24px 0;text-align:center;">Error loading watchdog data — check API</div>';
|
||||
});
|
||||
}
|
||||
|
||||
vvWdLoad();
|
||||
setInterval(vvWdLoad, 30000);
|
||||
|
||||
})();
|
||||
</script>
|
||||
@@ -0,0 +1,507 @@
|
||||
<style>
|
||||
.vv-wd-card { background:#161616;border:1px solid #2a2a2a;border-radius:6px;padding:10px;min-width:0; }
|
||||
.vv-wd-sec { font-size:10px;font-weight:bold;color:#444;letter-spacing:.07em;text-transform:uppercase;margin-bottom:6px; }
|
||||
.vv-wd-row { display:flex;justify-content:space-between;align-items:baseline;gap:6px;margin:2px 0; }
|
||||
.vv-wd-lbl { font-size:11px;color:#444;white-space:nowrap; }
|
||||
.vv-wd-val { font-size:12px;color:#bbb;text-align:right; }
|
||||
.vv-wd-sep { border:none;border-top:1px solid #1e1e1e;margin:6px 0; }
|
||||
.vv-wd-pill { font-size:10px;padding:1px 6px;border-radius:2px;background:#1e1e1e;color:#666;border:1px solid #272727; }
|
||||
.vv-wd-pill.ok { background:#0d1f0d;color:#4caf50;border-color:#1a3a1a; }
|
||||
.vv-wd-pill.warn { background:#1f1500;color:#ffb74d;border-color:#3a2800; }
|
||||
.vv-wd-pill.err { background:#200d0d;color:#ef5350;border-color:#3a1a1a; }
|
||||
.vv-wd-pill-row { display:flex;flex-wrap:wrap;gap:4px;margin-top:4px; }
|
||||
.vv-wd-bar { height:5px;border-radius:2px;background:#1e1e1e;margin-top:3px;overflow:hidden; }
|
||||
.vv-wd-bar-fill{ height:100%;border-radius:2px;transition:width .3s; }
|
||||
.vv-wd-badge { font-size:11px;font-weight:bold;padding:2px 8px;border-radius:3px; }
|
||||
.vv-wd-badge.ok { background:#0d1f0d;color:#4caf50; }
|
||||
.vv-wd-badge.soft { background:#1f1f00;color:#cddc39; }
|
||||
.vv-wd-badge.med { background:#1f1000;color:#ffb74d; }
|
||||
.vv-wd-badge.hard { background:#200d0d;color:#ef5350; }
|
||||
.vv-wd-node-h { display:flex;align-items:center;gap:8px;margin-bottom:10px; }
|
||||
.vv-wd-node-id { font-size:12px;font-weight:bold;color:#666;letter-spacing:.06em;text-transform:uppercase; }
|
||||
.vv-wd-dot { width:6px;height:6px;border-radius:50%;flex-shrink:0; }
|
||||
.vv-wd-strike-row { display:flex;align-items:baseline;gap:6px;margin:2px 0; }
|
||||
.vv-wd-strike-name{ font-size:11px;color:#777;flex:1; }
|
||||
.vv-wd-strike-cnt { font-size:11px;font-weight:bold;color:#ffb74d; }
|
||||
.vv-wd-reboot-ts { font-size:11px;color:#555;margin:1px 0; }
|
||||
.vv-wd-ctr-row { display:flex;justify-content:space-between;align-items:baseline;margin:2px 0; }
|
||||
.vv-wd-ctr-name{ font-size:11px;color:#888; }
|
||||
.vv-wd-ctr-lim { font-size:11px;color:#555; }
|
||||
.vv-wd-pressure{ grid-column:1/-1;border-color:#3a2000;background:#1a1000; }
|
||||
|
||||
/* One host per row — inner grid sizes all cards equally */
|
||||
.vv-wd-host-row {
|
||||
grid-column: 1 / -1;
|
||||
display: grid;
|
||||
grid-template-columns: repeat(5, 1fr);
|
||||
gap: 12px;
|
||||
}
|
||||
@media (max-width: 900px) {
|
||||
.vv-wd-host-row { grid-template-columns: 1fr; }
|
||||
}
|
||||
</style>
|
||||
|
||||
<div style="display:flex;justify-content:space-between;align-items:center;margin-bottom:12px;padding:0 2px;">
|
||||
<span style="font-size:13px;font-weight:bold;color:#888;text-transform:uppercase;letter-spacing:.06em;">Watchdog</span>
|
||||
<span style="font-size:11px;color:#3a3a3a;" id="vv-wd-ts"></span>
|
||||
</div>
|
||||
|
||||
<div id="vv-wd-grid" style="display:grid;grid-template-columns:repeat(8,1fr);gap:12px;">
|
||||
<div style="grid-column:1/-1;color:#444;font-size:12px;padding:16px 0;text-align:center;">Loading…</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
(function() {
|
||||
|
||||
const GB = 1073741824;
|
||||
|
||||
function _fmtBytes(b) {
|
||||
if (!b) return '0';
|
||||
if (b >= GB) return (b / GB).toFixed(1) + 'G';
|
||||
if (b >= 1048576) return (b / 1048576).toFixed(0) + 'M';
|
||||
return (b / 1024).toFixed(0) + 'K';
|
||||
}
|
||||
|
||||
function _relTime(ts) {
|
||||
if (!ts) return '—';
|
||||
const s = Math.floor(Date.now() / 1000) - ts;
|
||||
if (s < 60) return 'just now';
|
||||
if (s < 3600) return Math.floor(s / 60) + 'm ago';
|
||||
if (s < 86400) return Math.floor(s / 3600) + 'h ago';
|
||||
return Math.floor(s / 86400) + 'd ago';
|
||||
}
|
||||
|
||||
function _dur(s) {
|
||||
const d = Math.floor(s / 86400), h = Math.floor((s % 86400) / 3600), m = Math.floor((s % 3600) / 60);
|
||||
if (d) return d + 'd ' + h + 'h';
|
||||
if (h) return h + 'h ' + m + 'm';
|
||||
return m + 'm';
|
||||
}
|
||||
|
||||
function _row(lbl, val) {
|
||||
return `<div class="vv-wd-row"><span class="vv-wd-lbl">${lbl}</span><span class="vv-wd-val">${val}</span></div>`;
|
||||
}
|
||||
|
||||
function _bar(pct, col) {
|
||||
return `<div class="vv-wd-bar"><div class="vv-wd-bar-fill" style="width:${Math.min(pct,100)}%;background:${col}"></div></div>`;
|
||||
}
|
||||
|
||||
function _pill(label, cls) {
|
||||
return `<span class="vv-wd-pill ${cls}">${label}</span>`;
|
||||
}
|
||||
|
||||
function _levelLabel(level) {
|
||||
return ['OK', 'SOFT', 'MEDIUM', 'HARD'][level] || '?';
|
||||
}
|
||||
function _levelCls(level) {
|
||||
return ['ok', 'soft', 'med', 'hard'][level] || 'ok';
|
||||
}
|
||||
|
||||
// ── Pressure alert card ───────────────────────────────────────────────────────
|
||||
function _pressureCard(node) {
|
||||
const st = node.states;
|
||||
if (!st || st.rw_level === 0) return '';
|
||||
const level = st.rw_level;
|
||||
const cls = _levelCls(level);
|
||||
const label = _levelLabel(level);
|
||||
|
||||
const paused = (st.rw_paused || []).filter(Boolean);
|
||||
const stopped = (st.rw_stopped || []).filter(Boolean);
|
||||
const pausedHtml = paused.length ? paused.map(c => _pill(c, 'warn')).join('') : '';
|
||||
const stoppedHtml = stopped.length ? stopped.map(c => _pill(c, 'err')).join('') : '';
|
||||
|
||||
return `<div class="vv-wd-card vv-wd-pressure" style="grid-column:1/-1">
|
||||
<div style="display:flex;align-items:center;gap:10px;margin-bottom:8px;">
|
||||
<span class="vv-wd-badge ${cls}">PRESSURE ${label}</span>
|
||||
<span style="font-size:11px;color:#7a5020;">${node.id} (${node.hostname})</span>
|
||||
${st.mem_shutdown ? `<span class="vv-wd-badge hard" style="margin-left:auto;">MEM SHUTDOWN ACTIVE</span>` : ''}
|
||||
</div>
|
||||
${paused.length ? `<div style="margin-bottom:4px;"><span class="vv-wd-lbl">Paused:</span> <span class="vv-wd-pill-row" style="display:inline-flex;">${pausedHtml}</span></div>` : ''}
|
||||
${stopped.length ? `<div><span class="vv-wd-lbl">Stopped:</span> <span class="vv-wd-pill-row" style="display:inline-flex;">${stoppedHtml}</span></div>` : ''}
|
||||
</div>`;
|
||||
}
|
||||
|
||||
// ── System health card ────────────────────────────────────────────────────────
|
||||
function _systemCard(node, cfg) {
|
||||
const sys = node.system;
|
||||
if (!sys) {
|
||||
return `<div class="vv-wd-card">
|
||||
<div class="vv-wd-node-h">
|
||||
<span class="vv-wd-dot" style="background:#444"></span>
|
||||
<span class="vv-wd-node-id">${node.id}</span>
|
||||
<span style="font-size:11px;color:#3a3a3a;">${node.hostname}</span>
|
||||
<span class="vv-wd-badge" style="margin-left:auto;background:#1a1a1a;color:#444">UNREACHABLE</span>
|
||||
</div>
|
||||
</div>`;
|
||||
}
|
||||
|
||||
const memGb = sys.mem_avail / GB;
|
||||
const memTotGb = sys.mem_total / GB;
|
||||
const usedPct = memTotGb > 0 ? ((memTotGb - memGb) / memTotGb) * 100 : 0;
|
||||
const memCol = memGb < cfg.sys_mem_gb ? '#ef5350'
|
||||
: memGb < cfg.rw_hard_gb ? '#ef5350'
|
||||
: memGb < cfg.rw_medium_gb ? '#ffb74d'
|
||||
: memGb < cfg.rw_soft_gb ? '#cddc39' : '#4caf50';
|
||||
|
||||
const loadPct = sys.cores > 0 ? (sys.load1 / (sys.cores * cfg.rw_load_med)) * 100 : 0;
|
||||
const loadCol = sys.load1 > sys.cores * cfg.rw_load_med ? '#ef5350'
|
||||
: sys.load1 > sys.cores * cfg.rw_load_soft ? '#ffb74d'
|
||||
: '#4caf50';
|
||||
|
||||
const apiOnly = sys.api_only === true;
|
||||
const daemonDot = sys.daemon_ok === null ? '#555' : sys.daemon_ok ? '#4caf50' : '#ef5350';
|
||||
const daemonTxt = sys.daemon_ok === null ? '—' : sys.daemon_ok ? 'daemon ok' : 'daemon err';
|
||||
|
||||
const st = node.states || {};
|
||||
const level = st.rw_level || 0;
|
||||
const dotCol = level >= 3 ? '#ef5350' : level >= 2 ? '#ffb74d' : level >= 1 ? '#cddc39'
|
||||
: apiOnly ? '#4a7a9b' // blue-grey: API-only, no watchdog state
|
||||
: '#4caf50';
|
||||
|
||||
return `<div class="vv-wd-card">
|
||||
<div class="vv-wd-node-h">
|
||||
<span class="vv-wd-dot" style="background:${dotCol}"></span>
|
||||
<span class="vv-wd-node-id">${node.id}</span>
|
||||
<span style="font-size:11px;color:#3a3a3a;">${node.hostname}</span>
|
||||
<span class="vv-wd-badge ${apiOnly ? '' : _levelCls(level)}"
|
||||
style="margin-left:auto;${apiOnly ? 'background:#0d1f2a;color:#4a9eff;' : ''}"
|
||||
>${apiOnly ? 'API ONLY' : _levelLabel(level)}</span>
|
||||
</div>
|
||||
<div class="vv-wd-sec">System</div>
|
||||
${_row('RAM free', `<span style="color:${memCol}">${_fmtBytes(sys.mem_avail)}</span> / ${_fmtBytes(sys.mem_total)}`)}
|
||||
${_bar(usedPct, memCol)}
|
||||
<div style="display:flex;justify-content:space-between;margin-top:1px;font-size:10px;color:#333">
|
||||
<span>free</span>
|
||||
<span>${cfg.rw_soft_gb}G soft · ${cfg.rw_hard_gb}G hard · ${cfg.sys_mem_gb}G reboot</span>
|
||||
</div>
|
||||
<div style="height:5px"></div>
|
||||
${!apiOnly ? `${_row('Load avg', `<span style="color:${loadCol}">${sys.load1.toFixed(2)}</span> / ${sys.cores} cores`)}
|
||||
${_bar(loadPct, loadCol)}
|
||||
<div style="height:5px"></div>` : ''}
|
||||
${_row('Uptime', sys.uptime ? _dur(sys.uptime) : '—')}
|
||||
${_row('Docker', `<span style="color:${daemonDot}">${daemonTxt}</span>`)}
|
||||
${!apiOnly ? (sys.oom_count > 0 ? _row('OOM kills', `<span style="color:#ef5350">${sys.oom_count}</span>`) : _row('OOM kills', '<span style="color:#333">0</span>')) : ''}
|
||||
${apiOnly ? `<div style="font-size:10px;color:#333;margin-top:6px;">Watchdog state unavailable — SSH not configured</div>` : ''}
|
||||
</div>`;
|
||||
}
|
||||
|
||||
// ── Docker watchdog state card ────────────────────────────────────────────────
|
||||
function _dockerCard(node, cfg) {
|
||||
const st = node.states;
|
||||
if (!st) return `<div class="vv-wd-card"><div class="vv-wd-sec">Docker Watchdog</div><div style="color:#3a3a3a;font-size:11px;padding:8px 0;">No data</div></div>`;
|
||||
|
||||
const strikes = Object.entries(st.ctr_strikes || {});
|
||||
const skiplist = st.skiplist || [];
|
||||
const restarts = (st.restarts || []).slice(0, 10);
|
||||
const daemonCls = st.daemon_strikes > 0 ? 'err' : 'ok';
|
||||
const allOk = strikes.length === 0 && skiplist.length === 0 && st.daemon_strikes === 0;
|
||||
|
||||
// Group restarts by container for last-24h summary
|
||||
const rCounts = {};
|
||||
for (const r of restarts) {
|
||||
rCounts[r.name] = (rCounts[r.name] || 0) + 1;
|
||||
}
|
||||
|
||||
let strikesHtml = '';
|
||||
if (strikes.length === 0 && st.daemon_strikes === 0) {
|
||||
strikesHtml = '<div style="color:#333;font-size:11px;">0 active strikes</div>';
|
||||
} else {
|
||||
if (st.daemon_strikes > 0) {
|
||||
strikesHtml += `<div class="vv-wd-strike-row"><span class="vv-wd-strike-name">daemon</span><span class="vv-wd-strike-cnt">${st.daemon_strikes}</span></div>`;
|
||||
}
|
||||
for (const [name, cnt] of strikes) {
|
||||
strikesHtml += `<div class="vv-wd-strike-row"><span class="vv-wd-strike-name">${name}</span><span class="vv-wd-strike-cnt">${cnt} / ${cfg.cpu_fail_lim}</span></div>`;
|
||||
}
|
||||
}
|
||||
|
||||
let skipHtml = '';
|
||||
if (skiplist.length === 0) {
|
||||
skipHtml = '<div style="color:#333;font-size:11px;">empty</div>';
|
||||
} else {
|
||||
skipHtml = `<div class="vv-wd-pill-row">${skiplist.map(c => _pill(c, 'err')).join('')}</div>`;
|
||||
}
|
||||
|
||||
let restartHtml = '';
|
||||
const rcEntries = Object.entries(rCounts);
|
||||
if (rcEntries.length === 0) {
|
||||
restartHtml = '<div style="color:#333;font-size:11px;">none (24h)</div>';
|
||||
} else {
|
||||
restartHtml = rcEntries.map(([n, c]) =>
|
||||
`<div class="vv-wd-row"><span class="vv-wd-lbl">${n}</span><span class="vv-wd-val" style="color:${c >= cfg.restart_limit ? '#ef5350' : '#ffb74d'}">${c}×</span></div>`
|
||||
).join('');
|
||||
}
|
||||
|
||||
return `<div class="vv-wd-card">
|
||||
<div class="vv-wd-sec">Docker Watchdog</div>
|
||||
<div style="display:flex;gap:6px;margin-bottom:8px;">
|
||||
${_pill(st.daemon_restart ? 'daemon restarted' : 'daemon ok', daemonCls)}
|
||||
${allOk ? _pill('all clear', 'ok') : ''}
|
||||
</div>
|
||||
<div class="vv-wd-sec">Strikes</div>
|
||||
${strikesHtml}
|
||||
<hr class="vv-wd-sep">
|
||||
<div class="vv-wd-sec">Skip list</div>
|
||||
${skipHtml}
|
||||
<hr class="vv-wd-sep">
|
||||
<div class="vv-wd-sec">Restarts (24h)</div>
|
||||
${restartHtml}
|
||||
</div>`;
|
||||
}
|
||||
|
||||
// ── Stability / reboot card ───────────────────────────────────────────────────
|
||||
function _stabilityCard(node, cfg) {
|
||||
const st = node.states;
|
||||
if (!st) return `<div class="vv-wd-card"><div class="vv-wd-sec">Stability</div><div style="color:#3a3a3a;font-size:11px;padding:8px 0;">No data</div></div>`;
|
||||
|
||||
const reboots = st.reboots || [];
|
||||
const sysStr = Object.entries(st.sys_strikes || {});
|
||||
const rebootCls = reboots.length >= cfg.reboot_limit ? 'err' : reboots.length > 0 ? 'warn' : 'ok';
|
||||
|
||||
let sysHtml = '';
|
||||
if (sysStr.length === 0) {
|
||||
sysHtml = '<div style="color:#333;font-size:11px;">0 active strikes</div>';
|
||||
} else {
|
||||
sysHtml = sysStr.map(([k, v]) =>
|
||||
`<div class="vv-wd-strike-row"><span class="vv-wd-strike-name" style="font-size:10px;">${k.replace(/_/g,' ')}</span><span class="vv-wd-strike-cnt">${v}</span></div>`
|
||||
).join('');
|
||||
}
|
||||
|
||||
const rebootHtml = reboots.length === 0
|
||||
? '<div style="color:#333;font-size:11px;">none (12h)</div>'
|
||||
: reboots.map(ts => `<div class="vv-wd-reboot-ts">${_relTime(ts)}</div>`).join('');
|
||||
|
||||
return `<div class="vv-wd-card">
|
||||
<div class="vv-wd-sec">Stability</div>
|
||||
<div style="display:flex;gap:6px;margin-bottom:8px;">
|
||||
${_pill(`${reboots.length} / ${cfg.reboot_limit} reboots`, rebootCls)}
|
||||
${_pill(`${cfg.reboot_window}h window`, '')}
|
||||
</div>
|
||||
<div class="vv-wd-sec">Strikes</div>
|
||||
${sysHtml}
|
||||
<hr class="vv-wd-sep">
|
||||
<div class="vv-wd-sec">Reboot history</div>
|
||||
${rebootHtml}
|
||||
</div>`;
|
||||
}
|
||||
|
||||
// ── Config inventory card ─────────────────────────────────────────────────────
|
||||
function _configCard(node) {
|
||||
const cfg = node.config || {};
|
||||
const mon = Object.entries(cfg.monitored || {});
|
||||
const req = cfg.required || [];
|
||||
const ign = cfg.ignore || [];
|
||||
const paus = cfg.pause_list|| [];
|
||||
const stop = cfg.stop_list || [];
|
||||
const crit = cfg.critical || [];
|
||||
|
||||
const monHtml = mon.length === 0
|
||||
? '<div style="color:#333;font-size:11px;">none</div>'
|
||||
: mon.map(([name, mb]) => {
|
||||
const gb = (mb / 1024).toFixed(0);
|
||||
return `<div class="vv-wd-ctr-row"><span class="vv-wd-ctr-name">${name}</span><span class="vv-wd-ctr-lim">${gb} GB</span></div>`;
|
||||
}).join('');
|
||||
|
||||
const reqHtml = req.length === 0
|
||||
? '<div style="color:#333;font-size:11px;">none</div>'
|
||||
: `<div class="vv-wd-pill-row">${req.map(c => _pill(c, crit.includes(c) ? '' : '')).join('')}</div>`;
|
||||
|
||||
const ignHtml = ign.length === 0
|
||||
? '<div style="color:#3a3a3a;font-size:11px;">none</div>'
|
||||
: `<div class="vv-wd-pill-row">${ign.map(c => _pill(c, '')).join('')}</div>`;
|
||||
|
||||
const pausHtml = paus.length === 0
|
||||
? '<div style="color:#3a3a3a;font-size:11px;">none</div>'
|
||||
: `<div class="vv-wd-pill-row">${paus.map(c => _pill(c, 'warn')).join('')}</div>`;
|
||||
|
||||
const stopHtml = stop.length === 0
|
||||
? '<div style="color:#3a3a3a;font-size:11px;">none</div>'
|
||||
: `<div class="vv-wd-pill-row">${stop.map(c => _pill(c, 'err')).join('')}</div>`;
|
||||
|
||||
return `<div class="vv-wd-card" style="grid-column:span 4;">
|
||||
<div style="display:flex;gap:6px;align-items:center;margin-bottom:8px;">
|
||||
<span class="vv-wd-node-id">${node.id}</span>
|
||||
<span style="font-size:11px;color:#3a3a3a;">${node.hostname}</span>
|
||||
</div>
|
||||
<div style="display:grid;grid-template-columns:1fr 1fr;gap:10px;">
|
||||
<div>
|
||||
<div class="vv-wd-sec">Mem limits (Tier 1)</div>
|
||||
${monHtml}
|
||||
<div style="height:8px"></div>
|
||||
<div class="vv-wd-sec">Required</div>
|
||||
${reqHtml}
|
||||
</div>
|
||||
<div>
|
||||
<div class="vv-wd-sec">Pause at medium pressure</div>
|
||||
${pausHtml}
|
||||
<div style="height:8px"></div>
|
||||
<div class="vv-wd-sec">Stop at hard pressure</div>
|
||||
${stopHtml}
|
||||
<div style="height:8px"></div>
|
||||
<div class="vv-wd-sec">Scan ignore</div>
|
||||
${ignHtml}
|
||||
</div>
|
||||
</div>
|
||||
</div>`;
|
||||
}
|
||||
|
||||
// ── Storage watchdog card ─────────────────────────────────────────────────────
|
||||
function _storageCard(node, cfg) {
|
||||
const st = node.states;
|
||||
const nodeCfg = node.config || {};
|
||||
if (!st) return `<div class="vv-wd-card"><div class="vv-wd-sec">Storage Watchdog</div><div style="color:#3a3a3a;font-size:11px;padding:8px 0;">No data</div></div>`;
|
||||
|
||||
const storWd = st.storage_wd || {};
|
||||
const growthStr = Object.entries(storWd.growth_strikes || {});
|
||||
const logStr = Object.entries(storWd.log_strikes || {});
|
||||
const totalIssues = growthStr.length + logStr.length;
|
||||
const allClear = totalIssues === 0;
|
||||
|
||||
// Baseline info
|
||||
const bCount = storWd.baseline_count ?? 0;
|
||||
const bAge = storWd.baseline_age_sec;
|
||||
let baselineNote = bCount > 0
|
||||
? `${bCount} containers tracked`
|
||||
: 'no baseline yet (builds after first cycle)';
|
||||
if (bAge != null && bCount > 0) {
|
||||
const bAgeStr = bAge < 120 ? bAge + 's ago' : bAge < 3600 ? Math.floor(bAge/60) + 'm ago' : Math.floor(bAge/3600) + 'h ago';
|
||||
baselineNote += ` · updated ${bAgeStr}`;
|
||||
}
|
||||
|
||||
// Suppress ceilings configured for this host
|
||||
const sizes = Object.entries(nodeCfg.appdata_sizes || {});
|
||||
const sizesHtml = sizes.length
|
||||
? sizes.map(([c, mb]) => _pill(`${c} <${Math.round(mb/1024)}GB`, '')).join('')
|
||||
: '';
|
||||
|
||||
let growthHtml = '';
|
||||
if (growthStr.length === 0) {
|
||||
growthHtml = '<div style="color:#333;font-size:11px;">no active strikes</div>';
|
||||
} else {
|
||||
growthHtml = growthStr.map(([name, cnt]) =>
|
||||
`<div class="vv-wd-strike-row">
|
||||
<span class="vv-wd-strike-name">${name}</span>
|
||||
<span class="vv-wd-strike-cnt">${cnt} / ${cfg.stor_strike_lim}</span>
|
||||
</div>`
|
||||
).join('');
|
||||
}
|
||||
|
||||
let logHtml = '';
|
||||
if (logStr.length === 0) {
|
||||
logHtml = '<div style="color:#333;font-size:11px;">no active strikes</div>';
|
||||
} else {
|
||||
logHtml = logStr.map(([key, cnt]) => {
|
||||
const display = key.length > 36 ? '…' + key.slice(-36) : key;
|
||||
return `<div class="vv-wd-strike-row">
|
||||
<span class="vv-wd-strike-name" style="font-size:10px;" title="${key}">${display}</span>
|
||||
<span class="vv-wd-strike-cnt">${cnt} / ${cfg.stor_strike_lim}</span>
|
||||
</div>`;
|
||||
}).join('');
|
||||
}
|
||||
|
||||
return `<div class="vv-wd-card">
|
||||
<div class="vv-wd-sec">Storage Watchdog</div>
|
||||
<div style="display:flex;gap:5px;flex-wrap:wrap;margin-bottom:6px;">
|
||||
${allClear ? _pill('all clear', 'ok') : _pill(totalIssues + ' active strike' + (totalIssues !== 1 ? 's' : ''), 'warn')}
|
||||
${_pill('growth >' + cfg.growth_gb + 'GB/cycle', '')}
|
||||
${_pill('log max ' + cfg.log_max_gb + 'GB', '')}
|
||||
${cfg.truncate_logs ? _pill('auto-truncate on', 'ok') : _pill('auto-truncate off', '')}
|
||||
</div>
|
||||
<div style="font-size:10px;color:#3a3a3a;margin-bottom:8px;">${baselineNote}</div>
|
||||
<div style="display:grid;grid-template-columns:1fr 1fr;gap:12px;">
|
||||
<div>
|
||||
<div class="vv-wd-sec">Growth strikes</div>
|
||||
${growthHtml}
|
||||
</div>
|
||||
<div>
|
||||
<div class="vv-wd-sec">Log size strikes</div>
|
||||
${logHtml}
|
||||
</div>
|
||||
</div>
|
||||
${sizes.length ? `<hr class="vv-wd-sep"><div class="vv-wd-sec">Suppress ceilings (this host)</div><div class="vv-wd-pill-row">${sizesHtml}</div>` : ''}
|
||||
</div>`;
|
||||
}
|
||||
|
||||
// ── Network watchdog card ─────────────────────────────────────────────────────
|
||||
function _networkCard(node, cfg) {
|
||||
const st = node.states;
|
||||
const nodeCfg = node.config || {};
|
||||
if (!st) return `<div class="vv-wd-card"><div class="vv-wd-sec">Network Watchdog</div><div style="color:#3a3a3a;font-size:11px;padding:8px 0;">No data</div></div>`;
|
||||
|
||||
const netWd = st.network_wd || {};
|
||||
const npmStr = netWd.npm_strikes ?? 0;
|
||||
const npmCls = npmStr >= cfg.npm_strike_lim ? 'err' : npmStr > 0 ? 'warn' : 'ok';
|
||||
const ddnsDomain = nodeCfg.ddns_domain || '';
|
||||
const ddnsCtr = nodeCfg.ddns_container || '';
|
||||
const npmUrl = nodeCfg.npm_url || '';
|
||||
|
||||
const ddnsHtml = ddnsDomain
|
||||
? `${_row('DDNS domain', `<span style="color:#888;">${ddnsDomain}</span>`)}
|
||||
${ddnsCtr ? _row('DDNS container', `<span style="color:#888;">${ddnsCtr}</span>`) : ''}`
|
||||
: _row('DDNS', '<span style="color:#444;">not configured for this host</span>');
|
||||
|
||||
const npmHtml = npmUrl
|
||||
? `${_row('NPM URL', `<span style="color:#888;font-size:10px;">${npmUrl}</span>`)}
|
||||
${_row('NPM strikes', `<span class="vv-wd-pill ${npmCls}" style="font-size:10px;">${npmStr} / ${cfg.npm_strike_lim}</span>`)}`
|
||||
: _row('NPM check', '<span style="color:#444;">not configured for this host</span>');
|
||||
|
||||
return `<div class="vv-wd-card">
|
||||
<div class="vv-wd-sec">Network Watchdog</div>
|
||||
<div style="display:flex;gap:5px;flex-wrap:wrap;margin-bottom:8px;">
|
||||
${cfg.net_wd_enabled ? _pill('enabled', 'ok') : _pill('disabled', '')}
|
||||
${_pill('Tailscale check ' + (cfg.ts_check ? 'on' : 'off'), cfg.ts_check ? '' : '')}
|
||||
${npmUrl ? _pill('NPM ' + npmStr + '/' + cfg.npm_strike_lim + ' strikes', npmCls) : ''}
|
||||
</div>
|
||||
${ddnsHtml}
|
||||
<hr class="vv-wd-sep">
|
||||
${npmHtml}
|
||||
</div>`;
|
||||
}
|
||||
|
||||
// ── Main render ───────────────────────────────────────────────────────────────
|
||||
function _render(data) {
|
||||
const nodes = data.nodes || [];
|
||||
const cfg = data.cfg || {};
|
||||
let html = '';
|
||||
|
||||
// Per-host: pressure alert (if active) then all 5 watchdog cards in one equal-spaced row
|
||||
for (const node of nodes) {
|
||||
html += _pressureCard(node);
|
||||
html += `<div class="vv-wd-host-row">
|
||||
${_systemCard(node, cfg)}
|
||||
${_dockerCard(node, cfg)}
|
||||
${_stabilityCard(node, cfg)}
|
||||
${_storageCard(node, cfg)}
|
||||
${_networkCard(node, cfg)}
|
||||
</div>`;
|
||||
}
|
||||
|
||||
// Config inventory — separate row, each node span 4 (2 nodes = full row)
|
||||
for (const node of nodes) html += _configCard(node);
|
||||
|
||||
if (!html) html = '<div style="grid-column:1/-1;color:#444;font-size:12px;padding:16px 0;text-align:center;">No nodes configured.</div>';
|
||||
|
||||
document.getElementById('vv-wd-grid').innerHTML = html;
|
||||
|
||||
const ts = data.ts
|
||||
? new Date(data.ts * 1000).toLocaleString([], {
|
||||
month:'numeric', day:'numeric', year:'numeric',
|
||||
hour:'2-digit', minute:'2-digit', second:'2-digit'})
|
||||
: '';
|
||||
document.getElementById('vv-wd-ts').textContent = ts ? 'Updated: ' + ts : '';
|
||||
}
|
||||
|
||||
function vvWdLoad() {
|
||||
fetch('/plugins/varaverk/api/watchdog.php')
|
||||
.then(r => r.json())
|
||||
.then(_render)
|
||||
.catch(() => {
|
||||
document.getElementById('vv-wd-grid').innerHTML =
|
||||
'<div style="grid-column:1/-1;color:#555;font-size:12px;padding:24px 0;text-align:center;">Error loading watchdog data — check API</div>';
|
||||
});
|
||||
}
|
||||
|
||||
vvWdLoad();
|
||||
setInterval(vvWdLoad, 30000);
|
||||
|
||||
})();
|
||||
</script>
|
||||
+230
@@ -0,0 +1,230 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ============================= Media Shares Permissions =======================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Apply nobody:users ownership and correct permissions to all media shares.
|
||||
# Runs daily as the first job in the maintenance window — arr cleanup depends
|
||||
# on correct ownership to rename and delete files.
|
||||
#
|
||||
# Now a proper daily failsafe: files arrive with wrong ownership from rsync
|
||||
# without --chown, manual admin copies, containers with unconfigured PUID/PGID,
|
||||
# or unRAID environment resets after updates.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# acquire_lock "wait" — wait if previous run still active (large share scans)
|
||||
# detect_hosts() — correct share list per host via MY_ID aliases
|
||||
# Empty array guard — warns and exits cleanly if no shares configured
|
||||
# Folder existence — skips missing shares with warning, continues others
|
||||
# Separate passes — directories and files chmod'd separately for correctness
|
||||
# platform_require_cmd — notify script validated before use
|
||||
# Silent by default — only failures produce output, success is silent
|
||||
#
|
||||
# Diagnostic — high corrected count on every run means a container has wrong PUID/PGID:
|
||||
# Correct values on unRAID: PUID=99 (nobody) PGID=100 (users)
|
||||
# Common culprits: SABnzbd, qBittorrent, slskd — check these first
|
||||
# Once fixed, this script should correct 0 files per run (pure failsafe)
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# host*.conf
|
||||
#
|
||||
# HOST*_MEDIA_PERMISSION_SHARES — shares this host applies permissions to
|
||||
# Aliased by detect_hosts() — script uses MEDIA_PERMISSION_SHARES
|
||||
#
|
||||
# master.conf
|
||||
#
|
||||
# PERMISSIONS_DIR_MODE — directory permissions (default 755)
|
||||
# PERMISSIONS_FILE_MODE — file permissions (default 664)
|
||||
# PERMISSIONS_OWNER — ownership applied to all files (default nobody:users)
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# media_shares_permissions.sh — normal run
|
||||
# media_shares_permissions.sh --dry-run — preview without making changes
|
||||
# media_shares_permissions.sh --log — verbose output
|
||||
# media_shares_permissions.sh --status — show config and exit
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
platform_require_cmd \
|
||||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||||
"" "" \
|
||||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||
|
||||
acquire_lock "wait"
|
||||
|
||||
# detect_hosts() sets MY_ID and aliases HOST*_MEDIA_PERMISSION_SHARES
|
||||
detect_hosts
|
||||
|
||||
# Empty array guard
|
||||
if [[ ${#MEDIA_PERMISSION_SHARES[@]} -eq 0 ]]; then
|
||||
warn "MEDIA_PERMISSION_SHARES is empty for $MY_ID — nothing to do"
|
||||
warn "Check HOST*_MEDIA_PERMISSION_SHARES in host*.conf"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_PERMS Dir mode: ${PERMISSIONS_DIR_MODE:-755}"
|
||||
echo "$ICON_PERMS File mode: ${PERMISSIONS_FILE_MODE:-664}"
|
||||
echo "$ICON_PERMS Owner: $PERMISSIONS_OWNER"
|
||||
echo "$ICON_PERMS Shares: ${#MEDIA_PERMISSION_SHARES[@]}"
|
||||
echo ""
|
||||
for share in "${MEDIA_PERMISSION_SHARES[@]}"; do
|
||||
local_status="missing"
|
||||
[[ -d "$share" ]] && local_status="exists"
|
||||
echo " $share — $local_status"
|
||||
done
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Apply Permissions ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_PERMS Media Permissions — $MY_ID ━━━"
|
||||
log "Dir mode: ${PERMISSIONS_DIR_MODE:-755}"
|
||||
log "File mode: ${PERMISSIONS_FILE_MODE:-664}"
|
||||
log "Owner: $PERMISSIONS_OWNER"
|
||||
log "Shares: ${#MEDIA_PERMISSION_SHARES[@]}"
|
||||
echo ""
|
||||
|
||||
START=$(date +%s)
|
||||
FAILED=()
|
||||
UPDATED=()
|
||||
SKIPPED=()
|
||||
TOTAL_DIRS_FIXED=0
|
||||
TOTAL_FILES_FIXED=0
|
||||
|
||||
for SHARE in "${MEDIA_PERMISSION_SHARES[@]}"; do
|
||||
SHARE_NAME=$(basename "$SHARE")
|
||||
|
||||
if [[ ! -d "$SHARE" ]]; then
|
||||
warn "$SHARE_NAME not found — skipping"
|
||||
SKIPPED+=("$SHARE_NAME")
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
# Count what would be changed without making changes
|
||||
DIR_COUNT=$(find "$SHARE" -type d ! -perm "${PERMISSIONS_DIR_MODE:-755}" \
|
||||
2>/dev/null | wc -l)
|
||||
FILE_COUNT=$(find "$SHARE" -type f ! -perm "${PERMISSIONS_FILE_MODE:-664}" \
|
||||
2>/dev/null | wc -l)
|
||||
OWNER_COUNT=$(find "$SHARE" ! -user nobody -o ! -group users \
|
||||
2>/dev/null | wc -l)
|
||||
warn "DRY RUN — $SHARE_NAME: $DIR_COUNT dirs, $FILE_COUNT files, $OWNER_COUNT ownership fixes needed"
|
||||
continue
|
||||
fi
|
||||
|
||||
log "Updating $SHARE_NAME..."
|
||||
|
||||
CHMOD_DIR_OK=true
|
||||
CHMOD_FILE_OK=true
|
||||
CHOWN_OK=true
|
||||
|
||||
# Count files with wrong ownership before fixing (diagnostic)
|
||||
WRONG_OWNER=$(find "$SHARE" \( ! -user nobody -o ! -group users \) \
|
||||
2>/dev/null | wc -l)
|
||||
|
||||
# Apply ownership first — affects all files and directories
|
||||
chown -R "$PERMISSIONS_OWNER" "$SHARE" 2>/dev/null || CHOWN_OK=false
|
||||
|
||||
# Apply directory permissions — separate pass for correctness
|
||||
# Directories need execute bit — different from files
|
||||
find "$SHARE" -type d -exec chmod "${PERMISSIONS_DIR_MODE:-755}" {} + \
|
||||
2>/dev/null || CHMOD_DIR_OK=false
|
||||
|
||||
# Apply file permissions — no execute bit on media files
|
||||
find "$SHARE" -type f -exec chmod "${PERMISSIONS_FILE_MODE:-664}" {} + \
|
||||
2>/dev/null || CHMOD_FILE_OK=false
|
||||
|
||||
if [[ "$CHMOD_DIR_OK" == true && \
|
||||
"$CHMOD_FILE_OK" == true && \
|
||||
"$CHOWN_OK" == true ]]; then
|
||||
log "$ICON_UNLOCKED $SHARE_NAME — permissions applied"
|
||||
UPDATED+=("$SHARE_NAME")
|
||||
# Log diagnostic if many files had wrong ownership
|
||||
if [[ "$WRONG_OWNER" -gt 0 ]]; then
|
||||
warn "$SHARE_NAME — corrected $WRONG_OWNER file(s) with wrong ownership"
|
||||
warn "If this is high, check container PUID/PGID settings (should be PUID=99 PGID=100)"
|
||||
fi
|
||||
TOTAL_DIRS_FIXED=$(( TOTAL_DIRS_FIXED + 1 ))
|
||||
TOTAL_FILES_FIXED=$(( TOTAL_FILES_FIXED + WRONG_OWNER ))
|
||||
else
|
||||
error "$SHARE_NAME — permissions failed"
|
||||
error " chown: $CHOWN_OK chmod dirs: $CHMOD_DIR_OK chmod files: $CHMOD_FILE_OK"
|
||||
FAILED+=("$SHARE_NAME")
|
||||
fi
|
||||
done
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY MEDIA PERMISSIONS SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_PERMS Dir mode: ${PERMISSIONS_DIR_MODE:-755}"
|
||||
echo "$ICON_PERMS File mode: ${PERMISSIONS_FILE_MODE:-664}"
|
||||
echo "$ICON_PERMS Owner: $PERMISSIONS_OWNER"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
[[ ${#UPDATED[@]} -gt 0 ]] && log "Updated: ${#UPDATED[@]} shares"
|
||||
[[ ${#SKIPPED[@]} -gt 0 ]] && warn "Skipped: ${SKIPPED[*]} (not found)"
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}"
|
||||
|
||||
# Diagnostic — high correction count indicates container PUID/PGID issue
|
||||
if [[ "$TOTAL_FILES_FIXED" -gt 50 ]]; then
|
||||
warn "$TOTAL_FILES_FIXED files had wrong ownership this run"
|
||||
warn "High count suggests a container is not set to PUID=99 PGID=100"
|
||||
warn "Common culprits: SABnzbd, qBittorrent, slskd — check container env vars"
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no changes made"
|
||||
elif [[ ${#FAILED[@]} -gt 0 ]]; then
|
||||
echo "$ICON_ERROR Status: SOME SHARES FAILED"
|
||||
notify "Media permissions failed on $(hostname) — ${FAILED[*]}" \
|
||||
"Media Permissions" "warning"
|
||||
else
|
||||
echo "$ICON_DONE Status: done — ${#UPDATED[@]} shares updated"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
+232
@@ -0,0 +1,232 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ============================= Media Shares Permissions =======================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Apply nobody:users ownership and correct permissions to all media shares.
|
||||
# Runs daily as the first job in the maintenance window — arr cleanup depends
|
||||
# on correct ownership to rename and delete files.
|
||||
#
|
||||
# Now a proper daily failsafe: files arrive with wrong ownership from rsync
|
||||
# without --chown, manual admin copies, containers with unconfigured PUID/PGID,
|
||||
# or unRAID environment resets after updates.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# acquire_lock "wait" — wait if previous run still active (large share scans)
|
||||
# detect_hosts() — correct share list per host via MY_ID aliases
|
||||
# Empty array guard — warns and exits cleanly if no shares configured
|
||||
# Folder existence — skips missing shares with warning, continues others
|
||||
# Separate passes — directories and files chmod'd separately for correctness
|
||||
# platform_require_cmd — notify script validated before use
|
||||
# Silent by default — only failures produce output, success is silent
|
||||
#
|
||||
# Diagnostic — high corrected count on every run means a container has wrong PUID/PGID:
|
||||
# Correct values on unRAID: PUID=99 (nobody) PGID=100 (users)
|
||||
# Common culprits: SABnzbd, qBittorrent, slskd — check these first
|
||||
# Once fixed, this script should correct 0 files per run (pure failsafe)
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# host*.conf
|
||||
#
|
||||
# HOST*_MEDIA_PERMISSION_SHARES — shares this host applies permissions to
|
||||
# Aliased by detect_hosts() — script uses MEDIA_PERMISSION_SHARES
|
||||
#
|
||||
# master.conf
|
||||
#
|
||||
# PERMISSIONS_DIR_MODE — directory permissions (default 755)
|
||||
# PERMISSIONS_FILE_MODE — file permissions (default 664)
|
||||
# PERMISSIONS_OWNER — ownership applied to all files (default nobody:users)
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# media_shares_permissions.sh — normal run
|
||||
# media_shares_permissions.sh --dry-run — preview without making changes
|
||||
# media_shares_permissions.sh --log — verbose output
|
||||
# media_shares_permissions.sh --status — show config and exit
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
platform_require_cmd \
|
||||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||||
"" "" \
|
||||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||
|
||||
acquire_lock "wait"
|
||||
|
||||
# detect_hosts() sets MY_ID and aliases HOST*_MEDIA_PERMISSION_SHARES
|
||||
detect_hosts
|
||||
|
||||
# Empty array guard
|
||||
if [[ ${#MEDIA_PERMISSION_SHARES[@]} -eq 0 ]]; then
|
||||
warn "MEDIA_PERMISSION_SHARES is empty for $MY_ID — nothing to do"
|
||||
warn "Check HOST*_MEDIA_PERMISSION_SHARES in host*.conf"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_PERMS Dir mode: ${PERMISSIONS_DIR_MODE:-755}"
|
||||
echo "$ICON_PERMS File mode: ${PERMISSIONS_FILE_MODE:-664}"
|
||||
echo "$ICON_PERMS Owner: $PERMISSIONS_OWNER"
|
||||
echo "$ICON_PERMS Shares: ${#MEDIA_PERMISSION_SHARES[@]}"
|
||||
echo ""
|
||||
for share in "${MEDIA_PERMISSION_SHARES[@]}"; do
|
||||
local_status="missing"
|
||||
[[ -d "$share" ]] && local_status="exists"
|
||||
echo " $share — $local_status"
|
||||
done
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Apply Permissions ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_PERMS Media Permissions — $MY_ID ━━━"
|
||||
log "Dir mode: ${PERMISSIONS_DIR_MODE:-755}"
|
||||
log "File mode: ${PERMISSIONS_FILE_MODE:-664}"
|
||||
log "Owner: $PERMISSIONS_OWNER"
|
||||
log "Shares: ${#MEDIA_PERMISSION_SHARES[@]}"
|
||||
echo ""
|
||||
|
||||
START=$(date +%s)
|
||||
FAILED=()
|
||||
UPDATED=()
|
||||
SKIPPED=()
|
||||
TOTAL_DIRS_FIXED=0
|
||||
TOTAL_FILES_FIXED=0
|
||||
|
||||
for SHARE in "${MEDIA_PERMISSION_SHARES[@]}"; do
|
||||
SHARE_NAME=$(basename "$SHARE")
|
||||
|
||||
if [[ ! -d "$SHARE" ]]; then
|
||||
warn "$SHARE_NAME not found — skipping"
|
||||
SKIPPED+=("$SHARE_NAME")
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
# Count what would be changed without making changes
|
||||
DIR_COUNT=$(find "$SHARE" -type d ! -perm "${PERMISSIONS_DIR_MODE:-755}" \
|
||||
2>/dev/null | wc -l)
|
||||
FILE_COUNT=$(find "$SHARE" -type f ! -perm "${PERMISSIONS_FILE_MODE:-664}" \
|
||||
2>/dev/null | wc -l)
|
||||
OWNER_COUNT=$(find "$SHARE" ! -user nobody -o ! -group users \
|
||||
2>/dev/null | wc -l)
|
||||
warn "DRY RUN — $SHARE_NAME: $DIR_COUNT dirs, $FILE_COUNT files, $OWNER_COUNT ownership fixes needed"
|
||||
continue
|
||||
fi
|
||||
|
||||
log "Updating $SHARE_NAME..."
|
||||
|
||||
CHMOD_DIR_OK=true
|
||||
CHMOD_FILE_OK=true
|
||||
CHOWN_OK=true
|
||||
|
||||
# Count files with wrong ownership before fixing (diagnostic)
|
||||
WRONG_OWNER=$(find "$SHARE" \( ! -user nobody -o ! -group users \) \
|
||||
2>/dev/null | wc -l)
|
||||
|
||||
# Apply ownership first — affects all files and directories
|
||||
chown -R "$PERMISSIONS_OWNER" "$SHARE" 2>/dev/null || CHOWN_OK=false
|
||||
|
||||
# Apply directory permissions — separate pass for correctness
|
||||
# Directories need execute bit — different from files
|
||||
find "$SHARE" -type d -exec chmod "${PERMISSIONS_DIR_MODE:-755}" {} + \
|
||||
2>/dev/null || CHMOD_DIR_OK=false
|
||||
|
||||
# Apply file permissions — no execute bit on media files
|
||||
# Ignore "No such file" errors: race condition with volatile dirs (e.g. Emby transcodes)
|
||||
_chmod_errs=$(find "$SHARE" -type f -exec chmod "${PERMISSIONS_FILE_MODE:-664}" {} + 2>&1 | \
|
||||
grep -v "No such file or directory" | grep -c "chmod:" || true)
|
||||
[[ "$_chmod_errs" -gt 0 ]] && CHMOD_FILE_OK=false
|
||||
|
||||
if [[ "$CHMOD_DIR_OK" == true && \
|
||||
"$CHMOD_FILE_OK" == true && \
|
||||
"$CHOWN_OK" == true ]]; then
|
||||
log "$ICON_UNLOCKED $SHARE_NAME — permissions applied"
|
||||
UPDATED+=("$SHARE_NAME")
|
||||
# Log diagnostic if many files had wrong ownership
|
||||
if [[ "$WRONG_OWNER" -gt 0 ]]; then
|
||||
warn "$SHARE_NAME — corrected $WRONG_OWNER file(s) with wrong ownership"
|
||||
warn "If this is high, check container PUID/PGID settings (should be PUID=99 PGID=100)"
|
||||
fi
|
||||
TOTAL_DIRS_FIXED=$(( TOTAL_DIRS_FIXED + 1 ))
|
||||
TOTAL_FILES_FIXED=$(( TOTAL_FILES_FIXED + WRONG_OWNER ))
|
||||
else
|
||||
error "$SHARE_NAME — permissions failed"
|
||||
error " chown: $CHOWN_OK chmod dirs: $CHMOD_DIR_OK chmod files: $CHMOD_FILE_OK"
|
||||
FAILED+=("$SHARE_NAME")
|
||||
fi
|
||||
done
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY MEDIA PERMISSIONS SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_PERMS Dir mode: ${PERMISSIONS_DIR_MODE:-755}"
|
||||
echo "$ICON_PERMS File mode: ${PERMISSIONS_FILE_MODE:-664}"
|
||||
echo "$ICON_PERMS Owner: $PERMISSIONS_OWNER"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
[[ ${#UPDATED[@]} -gt 0 ]] && log "Updated: ${#UPDATED[@]} shares"
|
||||
[[ ${#SKIPPED[@]} -gt 0 ]] && warn "Skipped: ${SKIPPED[*]} (not found)"
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}"
|
||||
|
||||
# Diagnostic — high correction count indicates container PUID/PGID issue
|
||||
if [[ "$TOTAL_FILES_FIXED" -gt 50 ]]; then
|
||||
warn "$TOTAL_FILES_FIXED files had wrong ownership this run"
|
||||
warn "High count suggests a container is not set to PUID=99 PGID=100"
|
||||
warn "Common culprits: SABnzbd, qBittorrent, slskd — check container env vars"
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no changes made"
|
||||
elif [[ ${#FAILED[@]} -gt 0 ]]; then
|
||||
echo "$ICON_ERROR Status: SOME SHARES FAILED"
|
||||
notify "Media permissions failed on $(hostname) — ${FAILED[*]}" \
|
||||
"Media Permissions" "warning"
|
||||
else
|
||||
echo "$ICON_DONE Status: done — ${#UPDATED[@]} shares updated"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
@@ -0,0 +1,68 @@
|
||||
<?php
|
||||
header('Content-Type: application/json');
|
||||
require_once dirname(__DIR__) . '/include/config.php';
|
||||
require_once dirname(__DIR__) . '/include/monitor.php';
|
||||
|
||||
$base = vv_rsync_status();
|
||||
$vars = vv_conf_vars();
|
||||
|
||||
$base['windows']['fallback'] = ($vars['FALLBACK_RSYNC_ENABLED'] ?? 'true') !== 'false';
|
||||
|
||||
// Bandwidth history — last 30 days
|
||||
$bwLog = DATA_DIR . '/bandwidth_history.db';
|
||||
$warnGb = (float)($vars['BANDWIDTH_WARN_GB'] ?? 50);
|
||||
$cutoff = date('Y-m-d', strtotime('-30 days'));
|
||||
$history = [];
|
||||
|
||||
if (file_exists($bwLog)) {
|
||||
foreach (file($bwLog, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES) ?: [] as $line) {
|
||||
$p = explode('|', $line);
|
||||
if (count($p) < 5) continue;
|
||||
if ($p[0] < $cutoff) continue;
|
||||
$history[] = [
|
||||
'date' => $p[0],
|
||||
'time' => $p[1],
|
||||
'profile' => $p[2],
|
||||
'duration' => (int)$p[3],
|
||||
'status' => trim($p[4]),
|
||||
'bytes' => isset($p[5]) ? (int)$p[5] : 0,
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
// Per-window orch arrays (scripts + sync shares)
|
||||
$masterRaw = vv_read_conf_raw('master.conf');
|
||||
$myId = strtoupper(vv_detect_host());
|
||||
$hostRaw = vv_read_conf_raw(vv_detect_host() . '.conf');
|
||||
|
||||
$winArrayDefs = [
|
||||
'critical' => ['CRITICAL_MAINTENANCE_SCRIPTS', "{$myId}_CRITICAL_SYNC_SHARES"],
|
||||
'intermediate' => ['INTERMEDIATE_MAINTENANCE_SCRIPTS', "{$myId}_INTERMEDIATE_SYNC_SHARES"],
|
||||
'daily' => ['DAILY_MAINTENANCE_SCRIPTS', "{$myId}_DAILY_SYNC_SHARES"],
|
||||
'weekly' => ['WEEKLY_MAINTENANCE_SCRIPTS', "{$myId}_WEEKLY_SYNC_SHARES"],
|
||||
'fallback' => [null, null],
|
||||
];
|
||||
$winArrays = [];
|
||||
foreach ($winArrayDefs as $win => [$sv, $shv]) {
|
||||
$winArrays[$win] = [
|
||||
'scripts' => $sv ? vv_parse_bash_array($masterRaw, $sv) : [],
|
||||
'shares' => $shv ? vv_parse_bash_array($hostRaw, $shv) : [],
|
||||
];
|
||||
}
|
||||
|
||||
echo json_encode([
|
||||
'enabled' => $base['enabled'],
|
||||
'windows' => $base['windows'],
|
||||
'active' => $base['active'],
|
||||
'last_sync' => $base['last_sync'],
|
||||
'bw_history' => $history,
|
||||
'bw_warn_gb' => $warnGb,
|
||||
'win_arrays' => $winArrays,
|
||||
'settings' => [
|
||||
'bw_limit' => (int)($vars['BW_LIMIT'] ?? 0),
|
||||
'retry_count' => (int)($vars['RETRY_COUNT'] ?? 3),
|
||||
'sleep' => (int)($vars['SLEEP'] ?? 300),
|
||||
'bw_warn_gb' => (float)($vars['BANDWIDTH_WARN_GB'] ?? 50),
|
||||
],
|
||||
'ts' => time(),
|
||||
]);
|
||||
@@ -0,0 +1,110 @@
|
||||
<?php
|
||||
header('Content-Type: application/json');
|
||||
require_once dirname(__DIR__) . '/include/config.php';
|
||||
require_once dirname(__DIR__) . '/include/monitor.php';
|
||||
|
||||
// ── Live sync log ─────────────────────────────────────────────────────────────
|
||||
$action = $_GET['action'] ?? '';
|
||||
if ($action === 'rsync_log') {
|
||||
$lockDir = '/tmp/unraid_locks';
|
||||
$lines = [];
|
||||
$profile = null;
|
||||
$live = false;
|
||||
$elapsed = 0;
|
||||
|
||||
// Active sync — read live log
|
||||
foreach (glob("$lockDir/rsync_*.lock") ?: [] as $lf) {
|
||||
$content = trim(@file_get_contents($lf) ?: '');
|
||||
[$pid, $locked_name] = array_pad(explode(':', $content, 2), 2, '');
|
||||
if (!$pid || !file_exists("/proc/$pid")) continue;
|
||||
$profile = preg_replace('/^rsync_/', '', $locked_name ?: basename($lf, '.lock'));
|
||||
$elapsed = time() - (int)filemtime($lf);
|
||||
$liveLog = "$lockDir/rsync_{$profile}.log";
|
||||
$raw = file_exists($liveLog) ? (file($liveLog, FILE_IGNORE_NEW_LINES) ?: []) : [];
|
||||
$live = true;
|
||||
$lines = $raw;
|
||||
break;
|
||||
}
|
||||
|
||||
// No active sync — use most recent last.log
|
||||
if (!$live) {
|
||||
$lastLogs = glob("$lockDir/rsync_*.last.log") ?: [];
|
||||
if ($lastLogs) {
|
||||
usort($lastLogs, fn($a, $b) => filemtime($b) <=> filemtime($a));
|
||||
$lastLog = $lastLogs[0];
|
||||
$profile = preg_replace('/^rsync_(.+)\.last\.log$/', '$1', basename($lastLog));
|
||||
$lines = file($lastLog, FILE_IGNORE_NEW_LINES) ?: [];
|
||||
}
|
||||
}
|
||||
|
||||
// Strip ANSI escape codes, keep last 200 lines
|
||||
$lines = array_map(fn($l) => preg_replace('/\x1b\[[0-9;]*[mGKHF]/', '', $l), $lines);
|
||||
$lines = array_slice($lines, -200);
|
||||
|
||||
echo json_encode(['ok' => true, 'live' => $live, 'profile' => $profile, 'elapsed' => $elapsed, 'lines' => array_values($lines)]);
|
||||
exit;
|
||||
}
|
||||
|
||||
$base = vv_rsync_status();
|
||||
$vars = vv_conf_vars();
|
||||
|
||||
$base['windows']['fallback'] = ($vars['FALLBACK_RSYNC_ENABLED'] ?? 'true') !== 'false';
|
||||
|
||||
// Bandwidth history — last 30 days
|
||||
$bwLog = DATA_DIR . '/bandwidth_history.db';
|
||||
$warnGb = (float)($vars['BANDWIDTH_WARN_GB'] ?? 50);
|
||||
$cutoff = date('Y-m-d', strtotime('-30 days'));
|
||||
$history = [];
|
||||
|
||||
if (file_exists($bwLog)) {
|
||||
foreach (file($bwLog, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES) ?: [] as $line) {
|
||||
$p = explode('|', $line);
|
||||
if (count($p) < 5) continue;
|
||||
if ($p[0] < $cutoff) continue;
|
||||
$history[] = [
|
||||
'date' => $p[0],
|
||||
'time' => $p[1],
|
||||
'profile' => $p[2],
|
||||
'duration' => (int)$p[3],
|
||||
'status' => trim($p[4]),
|
||||
'bytes' => isset($p[5]) ? (int)$p[5] : 0,
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
// Per-window orch arrays (scripts + sync shares)
|
||||
$masterRaw = vv_read_conf_raw('master.conf');
|
||||
$myId = strtoupper(vv_detect_host());
|
||||
$hostRaw = vv_read_conf_raw(vv_detect_host() . '.conf');
|
||||
|
||||
$winArrayDefs = [
|
||||
'critical' => ['CRITICAL_MAINTENANCE_SCRIPTS', "{$myId}_CRITICAL_SYNC_SHARES"],
|
||||
'intermediate' => ['INTERMEDIATE_MAINTENANCE_SCRIPTS', "{$myId}_INTERMEDIATE_SYNC_SHARES"],
|
||||
'daily' => ['DAILY_MAINTENANCE_SCRIPTS', "{$myId}_DAILY_SYNC_SHARES"],
|
||||
'weekly' => ['WEEKLY_MAINTENANCE_SCRIPTS', "{$myId}_WEEKLY_SYNC_SHARES"],
|
||||
'fallback' => [null, null],
|
||||
];
|
||||
$winArrays = [];
|
||||
foreach ($winArrayDefs as $win => [$sv, $shv]) {
|
||||
$winArrays[$win] = [
|
||||
'scripts' => $sv ? vv_parse_bash_array($masterRaw, $sv) : [],
|
||||
'shares' => $shv ? vv_parse_bash_array($hostRaw, $shv) : [],
|
||||
];
|
||||
}
|
||||
|
||||
echo json_encode([
|
||||
'enabled' => $base['enabled'],
|
||||
'windows' => $base['windows'],
|
||||
'active' => $base['active'],
|
||||
'last_sync' => $base['last_sync'],
|
||||
'bw_history' => $history,
|
||||
'bw_warn_gb' => $warnGb,
|
||||
'win_arrays' => $winArrays,
|
||||
'settings' => [
|
||||
'bw_limit' => (int)($vars['BW_LIMIT'] ?? 0),
|
||||
'retry_count' => (int)($vars['RETRY_COUNT'] ?? 3),
|
||||
'sleep' => (int)($vars['SLEEP'] ?? 300),
|
||||
'bw_warn_gb' => (float)($vars['BANDWIDTH_WARN_GB'] ?? 50),
|
||||
],
|
||||
'ts' => time(),
|
||||
]);
|
||||
+450
@@ -0,0 +1,450 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ============================= Rsync Core Script ==============================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Core rsync engine for the two-server ecosystem. Called per share or per
|
||||
# appdata profile by orchestrators (daily_sync_maintenance, weekly_sync_maintenance,
|
||||
# critical_sync_maintenance) and directly for manual or scheduled dirty syncs.
|
||||
#
|
||||
# Profile is inferred from the directory basename (lowercased). Override with
|
||||
# --profile=name for explicit selection. If no profile matches, global defaults
|
||||
# from master.conf apply and no containers are stopped.
|
||||
#
|
||||
# After each sync, logs transfer data to bandwidth_monitor.sh for the weekly
|
||||
# bandwidth report. Silent on success — only failures produce visible output.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Profiles define per-share behavior:
|
||||
# PROFILE_CRITICAL_CONTAINER_NAMES — containers stopped on both servers before sync
|
||||
# PROFILE_DELAYED_CONTAINERS — containers with a delay before restart after sync
|
||||
# PROFILE_CONTAINER_DELAY — seconds before delayed containers start
|
||||
# PROFILE_RSYNC_OPTS — rsync flags (does not inherit DEFAULT_RSYNC_OPTS)
|
||||
# PROFILE_BW_LIMIT — bandwidth limit in KB/s
|
||||
# PROFILE_RETRY_COUNT — retry attempts on failure
|
||||
# PROFILE_SLEEP — seconds between retry attempts
|
||||
# PROFILE_EXCLUDE_DIRS — paths excluded from transfer
|
||||
# PROFILE_REMOTE_RESTART_CONTAINERS — containers restarted on remote after dirty sync
|
||||
# Was running → restart. Was stopped → leave stopped.
|
||||
#
|
||||
# Two-tier rsync enable/disable:
|
||||
# Tier 1: RSYNC_ENABLED=false → all rsync stops immediately (checked by this script)
|
||||
# Tier 2: per-orchestrator flag (DAILY_RSYNC_ENABLED etc.) → checked by caller
|
||||
#
|
||||
# Bandwidth logging: after each sync, logs profile/duration/status/bytes to
|
||||
# bandwidth_monitor.sh --log-transfer. Bytes captured from rsync --stats via awk
|
||||
# using version-stable field names.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Global Rsync Gate
|
||||
# check_rsync_enabled() — RSYNC_ENABLED=false exits cleanly before any operation.
|
||||
#
|
||||
# Partnership Blocklist
|
||||
# Refuses to sync if REMOTE_SERVER_NAME appears in the partnership blocklist.
|
||||
# Written at offboard — prevents stale access after a partnership ends.
|
||||
#
|
||||
# Version Parity
|
||||
# check_unraid_version_parity — refuses sync if servers on incompatible unRAID versions.
|
||||
#
|
||||
# Remote Health Pre-flights
|
||||
# check_connectivity() — Tailscale IP reachable before any SSH
|
||||
# check_remote_rootfs() — aborts if remote rootfs exceeds ROOTFS_WARN_PCT
|
||||
# check_remote_share() — aborts if target directory missing or empty on remote
|
||||
# check_remote_disks() — verifies all backing disks online on remote
|
||||
#
|
||||
# Drive Temperature Check
|
||||
# check_local_disk_temps() — runs before any transfer. Exit 1 = skip this profile,
|
||||
# exit 2 = abort all remaining syncs (CRITICAL temperature).
|
||||
#
|
||||
# Remote Docker Daemon Check
|
||||
# check_remote_docker_daemon — verified before any container stop/start operations.
|
||||
# If daemon unresponsive: container operations skipped, rsync proceeds without stopping.
|
||||
#
|
||||
# Per-Profile Concurrency Lock
|
||||
# acquire_rsync_lock() — per-profile lock prevents parallel runs of the same profile.
|
||||
# Global concurrent limit prevents too many simultaneous rsync processes.
|
||||
#
|
||||
# Notification Validated
|
||||
# platform_require_cmd confirms the notify script is present before use.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# master.conf
|
||||
#
|
||||
# RSYNC_ENABLED
|
||||
# Global on/off toggle for all rsync operations. (default: true)
|
||||
#
|
||||
# DEFAULT_RSYNC_OPTS
|
||||
# Base rsync flags for unproiled shares. Note: --delete is intentionally absent —
|
||||
# media shares spread files only, arr cleanup scripts own deletions. Profile-specific
|
||||
# opts set --delete explicitly where needed.
|
||||
#
|
||||
# BW_LIMIT
|
||||
# Default bandwidth cap in KB/s when no PROFILE_BW_LIMIT is set. (default: 0 = unlimited)
|
||||
#
|
||||
# RETRY_COUNT
|
||||
# Default retry attempts on rsync failure. (default: 3)
|
||||
#
|
||||
# SLEEP
|
||||
# Default seconds between retry attempts. (default: 60)
|
||||
#
|
||||
# ROOTFS_WARN_PCT
|
||||
# Abort threshold for remote rootfs percentage full. (default: 75)
|
||||
#
|
||||
# PROFILES["profile_KEY"]
|
||||
# Profile definitions — one entry per PROFILE_* key per profile name.
|
||||
# See OPERATIONAL MODEL above for all supported keys.
|
||||
#
|
||||
# BANDWIDTH_LOG / BANDWIDTH_WARN_GB
|
||||
# Shared with bandwidth_monitor.sh — set once, used by both.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# rsync.sh /path/to/share
|
||||
# Sync the given path to the remote. Profile inferred from directory basename.
|
||||
#
|
||||
# rsync.sh /path/to/share --profile=name
|
||||
# Sync with explicit profile override — bypasses basename inference.
|
||||
#
|
||||
# rsync.sh /path/to/share --dry-run
|
||||
# Run all pre-flight checks and show what rsync would transfer. No transfer,
|
||||
# no container stops.
|
||||
#
|
||||
# rsync.sh /path/to/share --status
|
||||
# Show resolved profile, remote identity, and configuration. Then exit.
|
||||
#
|
||||
# rsync.sh /path/to/share --log
|
||||
# Verbose output throughout — every decision logged.
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
# ── Separate positional directory arg from flags ───────────────────────────────────────────────
|
||||
DIRECTORY=""
|
||||
PROFILE_OVERRIDE=""
|
||||
RAW_ARGS=()
|
||||
|
||||
for ARG in "$@"; do
|
||||
case "$ARG" in
|
||||
--profile=*) PROFILE_OVERRIDE="${ARG#--profile=}" ;;
|
||||
--*|*=*) RAW_ARGS+=("$ARG") ;;
|
||||
*) [[ -z "$DIRECTORY" ]] && DIRECTORY="$ARG" || RAW_ARGS+=("$ARG") ;;
|
||||
esac
|
||||
done
|
||||
|
||||
parse_args "${RAW_ARGS[@]}"
|
||||
|
||||
[[ -z "$DIRECTORY" ]] && {
|
||||
error "No directory specified"
|
||||
error "Usage: rsync.sh <dir> [--dry-run] [--log] [--profile=name]"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
acquire_lock
|
||||
|
||||
if ! command -v docker &>/dev/null; then
|
||||
error "Docker command not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
platform_require_cmd \
|
||||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||||
"" "" \
|
||||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||
|
||||
detect_hosts
|
||||
|
||||
# Tier 1 global gate — Tier 2 (per-orchestrator) checked by caller
|
||||
if ! check_rsync_enabled; then
|
||||
warn "RSYNC_ENABLED=false — exiting cleanly"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Blocklist gate — refuse to sync with a partner blocked after offboard
|
||||
BLOCKLIST_FILE="${PARTNERSHIP_BLOCKLIST_FILE:-${STATE_DIR:-/boot/config}/partnership_blocklist.db}"
|
||||
if [[ -f "$BLOCKLIST_FILE" ]] && grep -q "^${REMOTE_SERVER_NAME}|" "$BLOCKLIST_FILE" 2>/dev/null; then
|
||||
error "Rsync blocked — $REMOTE_SERVER_NAME is on the partnership blocklist"
|
||||
error "Re-onboard the partnership to restore access: partnership_manager.sh --onboard"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
resolve_remote_ip
|
||||
|
||||
# ── Profile inference ─────────────────────────────────────────────────────────────────────────
|
||||
if [[ -n "$PROFILE_OVERRIDE" ]]; then
|
||||
PROFILE_NAME="$PROFILE_OVERRIDE"
|
||||
log "Profile override: $PROFILE_NAME"
|
||||
else
|
||||
PROFILE_NAME=$(basename "$DIRECTORY" | tr '[:upper:]' '[:lower:]')
|
||||
log "Profile inferred: $PROFILE_NAME"
|
||||
fi
|
||||
|
||||
# Acquire per-profile lock and check global concurrent limit
|
||||
acquire_rsync_lock "$PROFILE_NAME"
|
||||
|
||||
# ── Load profile settings ─────────────────────────────────────────────────────────────────────
|
||||
BW_LIMIT=${PROFILE_BW_LIMIT[$PROFILE_NAME]:-$BW_LIMIT}
|
||||
RETRY_COUNT=${PROFILE_RETRY_COUNT[$PROFILE_NAME]:-$RETRY_COUNT}
|
||||
SLEEP=${PROFILE_SLEEP[$PROFILE_NAME]:-$SLEEP}
|
||||
CONTAINER_DELAY=${PROFILE_CONTAINER_DELAY[$PROFILE_NAME]:-$CONTAINER_DELAY}
|
||||
|
||||
read -r -a CRITICAL_CONTAINER_NAMES <<< "${PROFILE_CRITICAL_CONTAINER_NAMES[$PROFILE_NAME]:-}"
|
||||
read -r -a DELAYED_CONTAINERS <<< "${PROFILE_DELAYED_CONTAINERS[$PROFILE_NAME]:-}"
|
||||
read -r -a EXCLUDE_DIRS <<< "${PROFILE_EXCLUDE_DIRS[$PROFILE_NAME]:-}"
|
||||
read -r -a REMOTE_RESTART_CONTAINERS <<< "${PROFILE_REMOTE_RESTART_CONTAINERS[$PROFILE_NAME]:-}"
|
||||
|
||||
# Local containers use same names as remote (mirrored naming scheme)
|
||||
LOCAL_CRITICAL_CONTAINER_NAMES=("${CRITICAL_CONTAINER_NAMES[@]}")
|
||||
|
||||
log "$ICON_GEAR Config: profile=${PROFILE_NAME} bw-limit=${BW_LIMIT}KB/s retry=${RETRY_COUNT} sleep=${SLEEP}s container-delay=${CONTAINER_DELAY}s"
|
||||
log "$ICON_GEAR Containers: critical=${CRITICAL_CONTAINER_NAMES[*]:-none} delayed=${DELAYED_CONTAINERS[*]:-none} remote-restart=${REMOTE_RESTART_CONTAINERS[*]:-none}"
|
||||
|
||||
[[ "$SHOW_STATUS" == true ]] && show_status && exit 0
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Pre-flight Checks ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_SHIELD Pre-flight ━━━"
|
||||
|
||||
# Disk temp — before touching remote or moving data
|
||||
# Exit 1 = skip this profile | Exit 2 = abort all remaining profiles
|
||||
check_local_disk_temps
|
||||
TEMP_RESULT=$?
|
||||
if [[ "$TEMP_RESULT" -eq 2 ]]; then
|
||||
error "Drive temps CRITICAL — aborting all remaining syncs"
|
||||
exit 2
|
||||
elif [[ "$TEMP_RESULT" -eq 1 ]]; then
|
||||
warn "Drive temps high — skipping profile [$PROFILE_NAME]"
|
||||
exit 1
|
||||
else
|
||||
log "Drive temps OK — $TEMP_CHECK_RESULT"
|
||||
fi
|
||||
|
||||
# Version parity — refuse if servers on incompatible unRAID versions
|
||||
check_unraid_version_parity || exit 1
|
||||
|
||||
check_connectivity
|
||||
check_remote_rootfs
|
||||
check_remote_share "$DIRECTORY"
|
||||
check_remote_disks "$DIRECTORY"
|
||||
|
||||
# Remote Docker daemon — check before attempting container operations
|
||||
if [[ ${#CRITICAL_CONTAINER_NAMES[@]} -gt 0 ]] || [[ ${#REMOTE_RESTART_CONTAINERS[@]} -gt 0 ]]; then
|
||||
check_remote_docker_daemon || {
|
||||
warn "Remote Docker daemon unresponsive — skipping container operations"
|
||||
warn "Proceeding with rsync only — containers will not be stopped or restarted"
|
||||
CRITICAL_CONTAINER_NAMES=()
|
||||
LOCAL_CRITICAL_CONTAINER_NAMES=()
|
||||
REMOTE_RESTART_CONTAINERS=()
|
||||
}
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Stop Containers ━━━
|
||||
# ==============================================================================================
|
||||
if [[ ${#CRITICAL_CONTAINER_NAMES[@]} -gt 0 || ${#LOCAL_CRITICAL_CONTAINER_NAMES[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_STOP $ICON_CONTAINERS Stop Containers ━━━"
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
for c in "${LOCAL_CRITICAL_CONTAINER_NAMES[@]:-}"; do
|
||||
[[ -n "$c" ]] && warn "DRY RUN — would stop local: $c"
|
||||
done
|
||||
for c in "${CRITICAL_CONTAINER_NAMES[@]:-}"; do
|
||||
[[ -n "$c" ]] && warn "DRY RUN — would stop remote: $c"
|
||||
done
|
||||
else
|
||||
# Local first — flush local databases before pushing
|
||||
stop_local_containers
|
||||
# Remote next — prevent writes while receiving
|
||||
stop_containers
|
||||
fi
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Transfer ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Transfer ━━━"
|
||||
echo "$ICON_RUN Source: $DIRECTORY"
|
||||
echo "$ICON_NET Remote: $REMOTE_SERVER:$DIRECTORY"
|
||||
echo "$ICON_GEAR Profile: $PROFILE_NAME"
|
||||
echo "$ICON_HOST Identity: $MY_ID → $REMOTE_ID"
|
||||
echo ""
|
||||
|
||||
get_rsync_opts
|
||||
|
||||
# Append profile excludes
|
||||
for ex in "${EXCLUDE_DIRS[@]:-}"; do
|
||||
[[ -n "$ex" ]] && RSYNC_OPTS+=(--exclude="$ex")
|
||||
done
|
||||
|
||||
# Add --stats to capture bytes transferred for bandwidth logging
|
||||
RSYNC_OPTS+=(--stats)
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && RSYNC_OPTS+=("--dry-run")
|
||||
|
||||
START=$(date +%s)
|
||||
RSYNC_SUCCESS=false
|
||||
BYTES_TRANSFERRED=0
|
||||
ATTEMPT=0
|
||||
|
||||
for (( ATTEMPT=1; ATTEMPT<=RETRY_COUNT; ATTEMPT++ )); do
|
||||
log "$ICON_RETRY Attempt $ATTEMPT of $RETRY_COUNT..."
|
||||
echo "$ICON_SYNC Rsync running — this may take a while..."
|
||||
|
||||
RSYNC_OUTPUT=$(rsync "${RSYNC_OPTS[@]}" \
|
||||
-e "ssh -i $SSH_KEY -T -o Compression=no -o IPQoS=throughput" \
|
||||
"$DIRECTORY" "root@${REMOTE_SERVER}:$(dirname "$DIRECTORY")/" 2>&1)
|
||||
|
||||
RSYNC_EXIT=$?
|
||||
|
||||
if [[ "$RSYNC_EXIT" -eq 0 ]]; then
|
||||
# Parse bytes transferred from --stats output
|
||||
BYTES_TRANSFERRED=$(echo "$RSYNC_OUTPUT" | \
|
||||
awk '/Total transferred file size:/{gsub(/,/,"",$NF); gsub(/[^0-9]/,"",$NF); print $NF+0}')
|
||||
BYTES_TRANSFERRED="${BYTES_TRANSFERRED:-0}"
|
||||
|
||||
log "$ICON_DONE Rsync complete — $BYTES_TRANSFERRED bytes transferred"
|
||||
RSYNC_SUCCESS=true
|
||||
break
|
||||
else
|
||||
warn "$ICON_RETRY Rsync failed (attempt $ATTEMPT/$RETRY_COUNT)"
|
||||
log "Exit code: $RSYNC_EXIT"
|
||||
if [[ "$ATTEMPT" -lt "$RETRY_COUNT" ]]; then
|
||||
log "Retrying in ${SLEEP}s..."
|
||||
sleep "$SLEEP"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Start Containers ━━━
|
||||
# ==============================================================================================
|
||||
if [[ ${#CRITICAL_CONTAINER_NAMES[@]} -gt 0 || ${#LOCAL_CRITICAL_CONTAINER_NAMES[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_START $ICON_CONTAINERS Start Containers ━━━"
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
for c in "${CRITICAL_CONTAINER_NAMES[@]:-}"; do
|
||||
[[ -n "$c" ]] && warn "DRY RUN — would start remote: $c"
|
||||
done
|
||||
for c in "${LOCAL_CRITICAL_CONTAINER_NAMES[@]:-}"; do
|
||||
[[ -n "$c" ]] && warn "DRY RUN — would start local: $c"
|
||||
done
|
||||
else
|
||||
# Remote first — can be coming up while local restarts
|
||||
start_containers
|
||||
# Local next
|
||||
start_local_containers
|
||||
fi
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Remote Restart (dirty sync profiles) ━━━
|
||||
# ==============================================================================================
|
||||
# For dirty sync profiles (critical-fallback, emby-fallback) — restart containers on remote
|
||||
# that were running before sync so they pick up config changes from the dirty sync window.
|
||||
# Was running → restart. Was stopped → leave stopped.
|
||||
if [[ ${#REMOTE_RESTART_CONTAINERS[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_START $ICON_CONTAINERS Remote Restart (post dirty sync) ━━━"
|
||||
log "Restarting configured containers on $REMOTE_SERVER_NAME..."
|
||||
|
||||
for container in "${REMOTE_RESTART_CONTAINERS[@]}"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
|
||||
# Check if container was running before sync (still tracked via RUNNING_CONTAINERS)
|
||||
WAS_RUNNING=false
|
||||
for prev in "${RUNNING_CONTAINERS[@]:-}"; do
|
||||
[[ "$prev" == "$container" ]] && WAS_RUNNING=true && break
|
||||
done
|
||||
|
||||
if [[ "$WAS_RUNNING" == false ]]; then
|
||||
# Not in stop list — check current remote state
|
||||
REMOTE_STATUS=$(timeout 15 ssh -i "$SSH_KEY" \
|
||||
-o ConnectTimeout=10 root@"$REMOTE_SERVER" \
|
||||
"docker inspect -f '{{.State.Running}}' $container 2>/dev/null" 2>/dev/null)
|
||||
[[ "$REMOTE_STATUS" != "true" ]] && \
|
||||
log "$container not running on $REMOTE_SERVER_NAME — skipping remote restart" && \
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would restart $container on $REMOTE_SERVER_NAME"
|
||||
continue
|
||||
fi
|
||||
|
||||
timeout 15 ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \
|
||||
"docker restart $container" >/dev/null 2>&1 && \
|
||||
log "$ICON_STARTED $container restarted on $REMOTE_SERVER_NAME ✅" || \
|
||||
warn "Failed to restart $container on $REMOTE_SERVER_NAME"
|
||||
done
|
||||
fi
|
||||
|
||||
END=$(date +%s)
|
||||
DURATION=$(( END - START ))
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Bandwidth Logging ━━━
|
||||
# ==============================================================================================
|
||||
# Logs to bandwidth_monitor.sh — new format includes bytes transferred and warn flag.
|
||||
# Only logs on actual runs (not dry-run) and only when bandwidth_monitor.sh exists.
|
||||
BANDWIDTH_MONITOR="$SCRIPT_DIR/../Monitors/bandwidth_monitor.sh"
|
||||
|
||||
if [[ "$DRY_RUN" == false ]] && [[ -f "$BANDWIDTH_MONITOR" ]]; then
|
||||
STATUS="success"
|
||||
[[ "$RSYNC_SUCCESS" == false ]] && STATUS="failed"
|
||||
bash "$BANDWIDTH_MONITOR" --log-transfer \
|
||||
"$PROFILE_NAME" "$DURATION" "$STATUS" "$BYTES_TRANSFERRED"
|
||||
log "$ICON_BANDWIDTH Transfer logged to bandwidth monitor ($BYTES_TRANSFERRED bytes)"
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY RSYNC SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_RUN Directory: $DIRECTORY"
|
||||
echo "$ICON_GEAR Profile: $PROFILE_NAME"
|
||||
echo "$ICON_TIME Duration: $(format_duration $DURATION)"
|
||||
[[ "$BYTES_TRANSFERRED" -gt 0 ]] && \
|
||||
echo "$ICON_BANDWIDTH Transferred: $(awk "BEGIN {printf \"%.2fGB\", $BYTES_TRANSFERRED / 1073741824}")"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no changes made"
|
||||
elif [[ "$RSYNC_SUCCESS" == true ]]; then
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS DONE"
|
||||
else
|
||||
echo "$ICON_ERROR Status: FAILED after $RETRY_COUNT attempts"
|
||||
notify "Rsync FAILED — $DIRECTORY ($PROFILE_NAME) after $RETRY_COUNT attempts on $(hostname)" \
|
||||
"Rsync" "warning"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ "$RSYNC_SUCCESS" == false ]] && [[ "$DRY_RUN" == false ]] && exit 1
|
||||
exit 0
|
||||
+461
@@ -0,0 +1,461 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ============================= Rsync Core Script ==============================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Core rsync engine for the two-server ecosystem. Called per share or per
|
||||
# appdata profile by orchestrators (daily_sync_maintenance, weekly_sync_maintenance,
|
||||
# critical_sync_maintenance) and directly for manual or scheduled dirty syncs.
|
||||
#
|
||||
# Profile is inferred from the directory basename (lowercased). Override with
|
||||
# --profile=name for explicit selection. If no profile matches, global defaults
|
||||
# from master.conf apply and no containers are stopped.
|
||||
#
|
||||
# After each sync, logs transfer data to bandwidth_monitor.sh for the weekly
|
||||
# bandwidth report. Silent on success — only failures produce visible output.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Profiles define per-share behavior:
|
||||
# PROFILE_CRITICAL_CONTAINER_NAMES — containers stopped on both servers before sync
|
||||
# PROFILE_DELAYED_CONTAINERS — containers with a delay before restart after sync
|
||||
# PROFILE_CONTAINER_DELAY — seconds before delayed containers start
|
||||
# PROFILE_RSYNC_OPTS — rsync flags (does not inherit DEFAULT_RSYNC_OPTS)
|
||||
# PROFILE_BW_LIMIT — bandwidth limit in KB/s
|
||||
# PROFILE_RETRY_COUNT — retry attempts on failure
|
||||
# PROFILE_SLEEP — seconds between retry attempts
|
||||
# PROFILE_EXCLUDE_DIRS — paths excluded from transfer
|
||||
# PROFILE_REMOTE_RESTART_CONTAINERS — containers restarted on remote after dirty sync
|
||||
# Was running → restart. Was stopped → leave stopped.
|
||||
#
|
||||
# Two-tier rsync enable/disable:
|
||||
# Tier 1: RSYNC_ENABLED=false → all rsync stops immediately (checked by this script)
|
||||
# Tier 2: per-orchestrator flag (DAILY_RSYNC_ENABLED etc.) → checked by caller
|
||||
#
|
||||
# Bandwidth logging: after each sync, logs profile/duration/status/bytes to
|
||||
# bandwidth_monitor.sh --log-transfer. Bytes captured from rsync --stats via awk
|
||||
# using version-stable field names.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Global Rsync Gate
|
||||
# check_rsync_enabled() — RSYNC_ENABLED=false exits cleanly before any operation.
|
||||
#
|
||||
# Partnership Blocklist
|
||||
# Refuses to sync if REMOTE_SERVER_NAME appears in the partnership blocklist.
|
||||
# Written at offboard — prevents stale access after a partnership ends.
|
||||
#
|
||||
# Version Parity
|
||||
# check_unraid_version_parity — refuses sync if servers on incompatible unRAID versions.
|
||||
#
|
||||
# Remote Health Pre-flights
|
||||
# check_connectivity() — Tailscale IP reachable before any SSH
|
||||
# check_remote_rootfs() — aborts if remote rootfs exceeds ROOTFS_WARN_PCT
|
||||
# check_remote_share() — aborts if target directory missing or empty on remote
|
||||
# check_remote_disks() — verifies all backing disks online on remote
|
||||
#
|
||||
# Drive Temperature Check
|
||||
# check_local_disk_temps() — runs before any transfer. Exit 1 = skip this profile,
|
||||
# exit 2 = abort all remaining syncs (CRITICAL temperature).
|
||||
#
|
||||
# Remote Docker Daemon Check
|
||||
# check_remote_docker_daemon — verified before any container stop/start operations.
|
||||
# If daemon unresponsive: container operations skipped, rsync proceeds without stopping.
|
||||
#
|
||||
# Per-Profile Concurrency Lock
|
||||
# acquire_rsync_lock() — per-profile lock prevents parallel runs of the same profile.
|
||||
# Global concurrent limit prevents too many simultaneous rsync processes.
|
||||
#
|
||||
# Notification Validated
|
||||
# platform_require_cmd confirms the notify script is present before use.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# master.conf
|
||||
#
|
||||
# RSYNC_ENABLED
|
||||
# Global on/off toggle for all rsync operations. (default: true)
|
||||
#
|
||||
# DEFAULT_RSYNC_OPTS
|
||||
# Base rsync flags for unproiled shares. Note: --delete is intentionally absent —
|
||||
# media shares spread files only, arr cleanup scripts own deletions. Profile-specific
|
||||
# opts set --delete explicitly where needed.
|
||||
#
|
||||
# BW_LIMIT
|
||||
# Default bandwidth cap in KB/s when no PROFILE_BW_LIMIT is set. (default: 0 = unlimited)
|
||||
#
|
||||
# RETRY_COUNT
|
||||
# Default retry attempts on rsync failure. (default: 3)
|
||||
#
|
||||
# SLEEP
|
||||
# Default seconds between retry attempts. (default: 60)
|
||||
#
|
||||
# ROOTFS_WARN_PCT
|
||||
# Abort threshold for remote rootfs percentage full. (default: 75)
|
||||
#
|
||||
# PROFILES["profile_KEY"]
|
||||
# Profile definitions — one entry per PROFILE_* key per profile name.
|
||||
# See OPERATIONAL MODEL above for all supported keys.
|
||||
#
|
||||
# BANDWIDTH_LOG / BANDWIDTH_WARN_GB
|
||||
# Shared with bandwidth_monitor.sh — set once, used by both.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# rsync.sh /path/to/share
|
||||
# Sync the given path to the remote. Profile inferred from directory basename.
|
||||
#
|
||||
# rsync.sh /path/to/share --profile=name
|
||||
# Sync with explicit profile override — bypasses basename inference.
|
||||
#
|
||||
# rsync.sh /path/to/share --dry-run
|
||||
# Run all pre-flight checks and show what rsync would transfer. No transfer,
|
||||
# no container stops.
|
||||
#
|
||||
# rsync.sh /path/to/share --status
|
||||
# Show resolved profile, remote identity, and configuration. Then exit.
|
||||
#
|
||||
# rsync.sh /path/to/share --log
|
||||
# Verbose output throughout — every decision logged.
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
# ── Separate positional directory arg from flags ───────────────────────────────────────────────
|
||||
DIRECTORY=""
|
||||
PROFILE_OVERRIDE=""
|
||||
RAW_ARGS=()
|
||||
|
||||
for ARG in "$@"; do
|
||||
case "$ARG" in
|
||||
--profile=*) PROFILE_OVERRIDE="${ARG#--profile=}" ;;
|
||||
--*|*=*) RAW_ARGS+=("$ARG") ;;
|
||||
*) [[ -z "$DIRECTORY" ]] && DIRECTORY="$ARG" || RAW_ARGS+=("$ARG") ;;
|
||||
esac
|
||||
done
|
||||
|
||||
parse_args "${RAW_ARGS[@]}"
|
||||
|
||||
[[ -z "$DIRECTORY" ]] && {
|
||||
error "No directory specified"
|
||||
error "Usage: rsync.sh <dir> [--dry-run] [--log] [--profile=name]"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
acquire_lock
|
||||
|
||||
if ! command -v docker &>/dev/null; then
|
||||
error "Docker command not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
platform_require_cmd \
|
||||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||||
"" "" \
|
||||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||
|
||||
detect_hosts
|
||||
|
||||
# Tier 1 global gate — Tier 2 (per-orchestrator) checked by caller
|
||||
if ! check_rsync_enabled; then
|
||||
warn "RSYNC_ENABLED=false — exiting cleanly"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Blocklist gate — refuse to sync with a partner blocked after offboard
|
||||
BLOCKLIST_FILE="${PARTNERSHIP_BLOCKLIST_FILE:-${STATE_DIR:-/boot/config}/partnership_blocklist.db}"
|
||||
if [[ -f "$BLOCKLIST_FILE" ]] && grep -q "^${REMOTE_SERVER_NAME}|" "$BLOCKLIST_FILE" 2>/dev/null; then
|
||||
error "Rsync blocked — $REMOTE_SERVER_NAME is on the partnership blocklist"
|
||||
error "Re-onboard the partnership to restore access: partnership_manager.sh --onboard"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
resolve_remote_ip
|
||||
|
||||
# ── Profile inference ─────────────────────────────────────────────────────────────────────────
|
||||
if [[ -n "$PROFILE_OVERRIDE" ]]; then
|
||||
PROFILE_NAME="$PROFILE_OVERRIDE"
|
||||
log "Profile override: $PROFILE_NAME"
|
||||
else
|
||||
PROFILE_NAME=$(basename "$DIRECTORY" | tr '[:upper:]' '[:lower:]')
|
||||
log "Profile inferred: $PROFILE_NAME"
|
||||
fi
|
||||
|
||||
# Acquire per-profile lock and check global concurrent limit
|
||||
acquire_rsync_lock "$PROFILE_NAME"
|
||||
|
||||
# Tee all output to a live log file for the Varaverk UI
|
||||
VV_LIVE_LOG="/tmp/unraid_locks/rsync_${PROFILE_NAME}.log"
|
||||
VV_LAST_LOG="/tmp/unraid_locks/rsync_${PROFILE_NAME}.last.log"
|
||||
: > "$VV_LIVE_LOG"
|
||||
exec 1> >(tee -a "$VV_LIVE_LOG") 2>&1
|
||||
|
||||
# ── Load profile settings ─────────────────────────────────────────────────────────────────────
|
||||
BW_LIMIT=${PROFILE_BW_LIMIT[$PROFILE_NAME]:-$BW_LIMIT}
|
||||
RETRY_COUNT=${PROFILE_RETRY_COUNT[$PROFILE_NAME]:-$RETRY_COUNT}
|
||||
SLEEP=${PROFILE_SLEEP[$PROFILE_NAME]:-$SLEEP}
|
||||
CONTAINER_DELAY=${PROFILE_CONTAINER_DELAY[$PROFILE_NAME]:-$CONTAINER_DELAY}
|
||||
|
||||
read -r -a CRITICAL_CONTAINER_NAMES <<< "${PROFILE_CRITICAL_CONTAINER_NAMES[$PROFILE_NAME]:-}"
|
||||
read -r -a DELAYED_CONTAINERS <<< "${PROFILE_DELAYED_CONTAINERS[$PROFILE_NAME]:-}"
|
||||
read -r -a EXCLUDE_DIRS <<< "${PROFILE_EXCLUDE_DIRS[$PROFILE_NAME]:-}"
|
||||
read -r -a REMOTE_RESTART_CONTAINERS <<< "${PROFILE_REMOTE_RESTART_CONTAINERS[$PROFILE_NAME]:-}"
|
||||
|
||||
# Local containers use same names as remote (mirrored naming scheme)
|
||||
LOCAL_CRITICAL_CONTAINER_NAMES=("${CRITICAL_CONTAINER_NAMES[@]}")
|
||||
|
||||
log "$ICON_GEAR Config: profile=${PROFILE_NAME} bw-limit=${BW_LIMIT}KB/s retry=${RETRY_COUNT} sleep=${SLEEP}s container-delay=${CONTAINER_DELAY}s"
|
||||
log "$ICON_GEAR Containers: critical=${CRITICAL_CONTAINER_NAMES[*]:-none} delayed=${DELAYED_CONTAINERS[*]:-none} remote-restart=${REMOTE_RESTART_CONTAINERS[*]:-none}"
|
||||
|
||||
[[ "$SHOW_STATUS" == true ]] && show_status && exit 0
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Pre-flight Checks ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_SHIELD Pre-flight ━━━"
|
||||
|
||||
# Disk temp — before touching remote or moving data
|
||||
# Exit 1 = skip this profile | Exit 2 = abort all remaining profiles
|
||||
check_local_disk_temps
|
||||
TEMP_RESULT=$?
|
||||
if [[ "$TEMP_RESULT" -eq 2 ]]; then
|
||||
error "Drive temps CRITICAL — aborting all remaining syncs"
|
||||
exit 2
|
||||
elif [[ "$TEMP_RESULT" -eq 1 ]]; then
|
||||
warn "Drive temps high — skipping profile [$PROFILE_NAME]"
|
||||
exit 1
|
||||
else
|
||||
log "Drive temps OK — $TEMP_CHECK_RESULT"
|
||||
fi
|
||||
|
||||
# Version parity — refuse if servers on incompatible unRAID versions
|
||||
check_unraid_version_parity || exit 1
|
||||
|
||||
check_connectivity
|
||||
check_remote_rootfs
|
||||
check_remote_share "$DIRECTORY"
|
||||
check_remote_disks "$DIRECTORY"
|
||||
|
||||
# Remote Docker daemon — check before attempting container operations
|
||||
if [[ ${#CRITICAL_CONTAINER_NAMES[@]} -gt 0 ]] || [[ ${#REMOTE_RESTART_CONTAINERS[@]} -gt 0 ]]; then
|
||||
check_remote_docker_daemon || {
|
||||
warn "Remote Docker daemon unresponsive — skipping container operations"
|
||||
warn "Proceeding with rsync only — containers will not be stopped or restarted"
|
||||
CRITICAL_CONTAINER_NAMES=()
|
||||
LOCAL_CRITICAL_CONTAINER_NAMES=()
|
||||
REMOTE_RESTART_CONTAINERS=()
|
||||
}
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Stop Containers ━━━
|
||||
# ==============================================================================================
|
||||
if [[ ${#CRITICAL_CONTAINER_NAMES[@]} -gt 0 || ${#LOCAL_CRITICAL_CONTAINER_NAMES[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_STOP $ICON_CONTAINERS Stop Containers ━━━"
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
for c in "${LOCAL_CRITICAL_CONTAINER_NAMES[@]:-}"; do
|
||||
[[ -n "$c" ]] && warn "DRY RUN — would stop local: $c"
|
||||
done
|
||||
for c in "${CRITICAL_CONTAINER_NAMES[@]:-}"; do
|
||||
[[ -n "$c" ]] && warn "DRY RUN — would stop remote: $c"
|
||||
done
|
||||
else
|
||||
# Local first — flush local databases before pushing
|
||||
stop_local_containers
|
||||
# Remote next — prevent writes while receiving
|
||||
stop_containers
|
||||
fi
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Transfer ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Transfer ━━━"
|
||||
echo "$ICON_RUN Source: $DIRECTORY"
|
||||
echo "$ICON_NET Remote: $REMOTE_SERVER:$DIRECTORY"
|
||||
echo "$ICON_GEAR Profile: $PROFILE_NAME"
|
||||
echo "$ICON_HOST Identity: $MY_ID → $REMOTE_ID"
|
||||
echo ""
|
||||
|
||||
get_rsync_opts
|
||||
|
||||
# Append profile excludes
|
||||
for ex in "${EXCLUDE_DIRS[@]:-}"; do
|
||||
[[ -n "$ex" ]] && RSYNC_OPTS+=(--exclude="$ex")
|
||||
done
|
||||
|
||||
# Add --stats to capture bytes transferred for bandwidth logging
|
||||
RSYNC_OPTS+=(--stats)
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && RSYNC_OPTS+=("--dry-run")
|
||||
|
||||
START=$(date +%s)
|
||||
RSYNC_SUCCESS=false
|
||||
BYTES_TRANSFERRED=0
|
||||
ATTEMPT=0
|
||||
|
||||
for (( ATTEMPT=1; ATTEMPT<=RETRY_COUNT; ATTEMPT++ )); do
|
||||
log "$ICON_RETRY Attempt $ATTEMPT of $RETRY_COUNT..."
|
||||
echo "$ICON_SYNC Rsync running — this may take a while..."
|
||||
|
||||
RSYNC_OUTPUT=$(rsync "${RSYNC_OPTS[@]}" \
|
||||
-e "ssh -i $SSH_KEY -T -o Compression=no -o IPQoS=throughput" \
|
||||
"$DIRECTORY" "root@${REMOTE_SERVER}:$(dirname "$DIRECTORY")/" 2>&1)
|
||||
|
||||
RSYNC_EXIT=$?
|
||||
|
||||
if [[ "$RSYNC_EXIT" -eq 0 ]]; then
|
||||
# Parse bytes transferred from --stats output
|
||||
BYTES_TRANSFERRED=$(echo "$RSYNC_OUTPUT" | \
|
||||
awk '/Total transferred file size:/{gsub(/,/,"",$NF); gsub(/[^0-9]/,"",$NF); print $NF+0}')
|
||||
BYTES_TRANSFERRED="${BYTES_TRANSFERRED:-0}"
|
||||
|
||||
log "$ICON_DONE Rsync complete — $BYTES_TRANSFERRED bytes transferred"
|
||||
RSYNC_SUCCESS=true
|
||||
break
|
||||
else
|
||||
warn "$ICON_RETRY Rsync failed (attempt $ATTEMPT/$RETRY_COUNT)"
|
||||
log "Exit code: $RSYNC_EXIT"
|
||||
if [[ "$ATTEMPT" -lt "$RETRY_COUNT" ]]; then
|
||||
log "Retrying in ${SLEEP}s..."
|
||||
sleep "$SLEEP"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Start Containers ━━━
|
||||
# ==============================================================================================
|
||||
if [[ ${#CRITICAL_CONTAINER_NAMES[@]} -gt 0 || ${#LOCAL_CRITICAL_CONTAINER_NAMES[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_START $ICON_CONTAINERS Start Containers ━━━"
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
for c in "${CRITICAL_CONTAINER_NAMES[@]:-}"; do
|
||||
[[ -n "$c" ]] && warn "DRY RUN — would start remote: $c"
|
||||
done
|
||||
for c in "${LOCAL_CRITICAL_CONTAINER_NAMES[@]:-}"; do
|
||||
[[ -n "$c" ]] && warn "DRY RUN — would start local: $c"
|
||||
done
|
||||
else
|
||||
# Remote first — can be coming up while local restarts
|
||||
start_containers
|
||||
# Local next
|
||||
start_local_containers
|
||||
fi
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Remote Restart (dirty sync profiles) ━━━
|
||||
# ==============================================================================================
|
||||
# For dirty sync profiles (critical-fallback, emby-fallback) — restart containers on remote
|
||||
# that were running before sync so they pick up config changes from the dirty sync window.
|
||||
# Was running → restart. Was stopped → leave stopped.
|
||||
if [[ ${#REMOTE_RESTART_CONTAINERS[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_START $ICON_CONTAINERS Remote Restart (post dirty sync) ━━━"
|
||||
log "Restarting configured containers on $REMOTE_SERVER_NAME..."
|
||||
|
||||
for container in "${REMOTE_RESTART_CONTAINERS[@]}"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
|
||||
# Check if container was running before sync (still tracked via RUNNING_CONTAINERS)
|
||||
WAS_RUNNING=false
|
||||
for prev in "${RUNNING_CONTAINERS[@]:-}"; do
|
||||
[[ "$prev" == "$container" ]] && WAS_RUNNING=true && break
|
||||
done
|
||||
|
||||
if [[ "$WAS_RUNNING" == false ]]; then
|
||||
# Not in stop list — check current remote state
|
||||
REMOTE_STATUS=$(timeout 15 ssh -i "$SSH_KEY" \
|
||||
-o ConnectTimeout=10 root@"$REMOTE_SERVER" \
|
||||
"docker inspect -f '{{.State.Running}}' $container 2>/dev/null" 2>/dev/null)
|
||||
[[ "$REMOTE_STATUS" != "true" ]] && \
|
||||
log "$container not running on $REMOTE_SERVER_NAME — skipping remote restart" && \
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would restart $container on $REMOTE_SERVER_NAME"
|
||||
continue
|
||||
fi
|
||||
|
||||
timeout 15 ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \
|
||||
"docker restart $container" >/dev/null 2>&1 && \
|
||||
log "$ICON_STARTED $container restarted on $REMOTE_SERVER_NAME ✅" || \
|
||||
warn "Failed to restart $container on $REMOTE_SERVER_NAME"
|
||||
done
|
||||
fi
|
||||
|
||||
END=$(date +%s)
|
||||
DURATION=$(( END - START ))
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Bandwidth Logging ━━━
|
||||
# ==============================================================================================
|
||||
# Logs to bandwidth_monitor.sh — new format includes bytes transferred and warn flag.
|
||||
# Only logs on actual runs (not dry-run) and only when bandwidth_monitor.sh exists.
|
||||
BANDWIDTH_MONITOR="$SCRIPT_DIR/../Monitors/bandwidth_monitor.sh"
|
||||
|
||||
if [[ "$DRY_RUN" == false ]] && [[ -f "$BANDWIDTH_MONITOR" ]]; then
|
||||
STATUS="success"
|
||||
[[ "$RSYNC_SUCCESS" == false ]] && STATUS="failed"
|
||||
bash "$BANDWIDTH_MONITOR" --log-transfer \
|
||||
"$PROFILE_NAME" "$DURATION" "$STATUS" "$BYTES_TRANSFERRED"
|
||||
log "$ICON_BANDWIDTH Transfer logged to bandwidth monitor ($BYTES_TRANSFERRED bytes)"
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY RSYNC SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_RUN Directory: $DIRECTORY"
|
||||
echo "$ICON_GEAR Profile: $PROFILE_NAME"
|
||||
echo "$ICON_TIME Duration: $(format_duration $DURATION)"
|
||||
[[ "$BYTES_TRANSFERRED" -gt 0 ]] && \
|
||||
echo "$ICON_BANDWIDTH Transferred: $(awk "BEGIN {printf \"%.2fGB\", $BYTES_TRANSFERRED / 1073741824}")"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no changes made"
|
||||
elif [[ "$RSYNC_SUCCESS" == true ]]; then
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS DONE"
|
||||
else
|
||||
echo "$ICON_ERROR Status: FAILED after $RETRY_COUNT attempts"
|
||||
notify "Rsync FAILED — $DIRECTORY ($PROFILE_NAME) after $RETRY_COUNT attempts on $(hostname)" \
|
||||
"Rsync" "warning"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
# Flush tee and preserve log for UI
|
||||
exec 1>&-; wait
|
||||
cp "$VV_LIVE_LOG" "$VV_LAST_LOG" 2>/dev/null
|
||||
rm -f "$VV_LIVE_LOG"
|
||||
|
||||
[[ "$RSYNC_SUCCESS" == false ]] && [[ "$DRY_RUN" == false ]] && exit 1
|
||||
exit 0
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,91 @@
|
||||
<?php
|
||||
header('Content-Type: application/json');
|
||||
require_once dirname(__DIR__) . '/include/config.php';
|
||||
|
||||
$action = ($_SERVER['REQUEST_METHOD'] === 'POST')
|
||||
? trim($_POST['action'] ?? '')
|
||||
: trim($_GET['action'] ?? '');
|
||||
|
||||
$cacheFile = STATE_DIR . '/cert_status.json';
|
||||
|
||||
// ── Read configured domains (without running checks) ─────────────────────────
|
||||
if ($action === 'domains') {
|
||||
$hostId = vv_detect_host();
|
||||
$hostIdUp = strtoupper($hostId);
|
||||
$confRaw = ($hostId !== 'unknown') ? vv_read_conf_raw($hostId . '.conf') : '';
|
||||
$master = vv_read_conf_raw('master.conf');
|
||||
|
||||
// Extract CERT_WARN_DAYS / CERT_CRIT_DAYS from master
|
||||
preg_match('/^\s*CERT_WARN_DAYS\s*=\s*(\d+)/m', $master, $w);
|
||||
preg_match('/^\s*CERT_CRIT_DAYS\s*=\s*(\d+)/m', $master, $c);
|
||||
|
||||
// Extract domains array from host conf
|
||||
$domains = [];
|
||||
if (preg_match('/' . $hostIdUp . '_CERT_MONITOR_DOMAINS\s*=\s*\(([^)]*)\)/s', $confRaw, $dm)) {
|
||||
preg_match_all('/"([^"]+)"/', $dm[1], $dd);
|
||||
$domains = $dd[1] ?? [];
|
||||
}
|
||||
|
||||
echo json_encode([
|
||||
'ok' => true,
|
||||
'host_id' => $hostId,
|
||||
'domains' => $domains,
|
||||
'warn_days' => (int)($w[1] ?? 30),
|
||||
'crit_days' => (int)($c[1] ?? 7),
|
||||
]);
|
||||
exit;
|
||||
}
|
||||
|
||||
// ── Run cert_monitor.sh now ───────────────────────────────────────────────────
|
||||
if ($action === 'run') {
|
||||
$script = SCRIPTS_DIR . '/Monitors/cert_monitor.sh';
|
||||
if (!file_exists($script)) {
|
||||
echo json_encode(['ok' => false, 'error' => 'cert_monitor.sh not found']);
|
||||
exit;
|
||||
}
|
||||
set_time_limit(180);
|
||||
exec('bash ' . escapeshellarg($script) . ' 2>&1', $out, $rc);
|
||||
// Read freshly written cache
|
||||
$data = file_exists($cacheFile)
|
||||
? (json_decode(file_get_contents($cacheFile), true) ?: null)
|
||||
: null;
|
||||
echo json_encode([
|
||||
'ok' => true,
|
||||
'data' => $data,
|
||||
'output' => array_slice(array_filter(array_map('trim', $out)), 0, 30),
|
||||
'rc' => $rc,
|
||||
]);
|
||||
exit;
|
||||
}
|
||||
|
||||
// ── Default: return cached status ─────────────────────────────────────────────
|
||||
if (!file_exists($cacheFile)) {
|
||||
// No cache yet — return configured domains so UI can show them unchecked
|
||||
$hostId = vv_detect_host();
|
||||
$hostIdUp = strtoupper($hostId);
|
||||
$confRaw = ($hostId !== 'unknown') ? vv_read_conf_raw($hostId . '.conf') : '';
|
||||
$master = vv_read_conf_raw('master.conf');
|
||||
|
||||
preg_match('/^\s*CERT_WARN_DAYS\s*=\s*(\d+)/m', $master, $w);
|
||||
preg_match('/^\s*CERT_CRIT_DAYS\s*=\s*(\d+)/m', $master, $c);
|
||||
$domains = [];
|
||||
if (preg_match('/' . $hostIdUp . '_CERT_MONITOR_DOMAINS\s*=\s*\(([^)]*)\)/s', $confRaw, $dm)) {
|
||||
preg_match_all('/"([^"]+)"/', $dm[1], $dd);
|
||||
foreach ($dd[1] ?? [] as $d) {
|
||||
$domains[] = ['domain' => $d, 'status' => 'UNKN', 'days' => null, 'expires' => ''];
|
||||
}
|
||||
}
|
||||
|
||||
echo json_encode([
|
||||
'ok' => true,
|
||||
'checked_at' => null,
|
||||
'host' => $hostId !== 'unknown' ? strtoupper($hostId) : null,
|
||||
'warn_days' => (int)($w[1] ?? 30),
|
||||
'crit_days' => (int)($c[1] ?? 7),
|
||||
'domains' => $domains,
|
||||
]);
|
||||
exit;
|
||||
}
|
||||
|
||||
$data = json_decode(file_get_contents($cacheFile), true) ?: [];
|
||||
echo json_encode(array_merge(['ok' => true], $data));
|
||||
@@ -0,0 +1,129 @@
|
||||
<?php
|
||||
header('Content-Type: application/json');
|
||||
require_once dirname(__DIR__) . '/include/config.php';
|
||||
require_once dirname(__DIR__) . '/include/auth.php';
|
||||
|
||||
$action = ($_SERVER['REQUEST_METHOD'] === 'POST')
|
||||
? trim($_POST['action'] ?? '')
|
||||
: trim($_GET['action'] ?? '');
|
||||
|
||||
$cacheFile = STATE_DIR . '/cert_status.json';
|
||||
|
||||
// ── Live NPM cert list ────────────────────────────────────────────────────────
|
||||
if ($action === 'npm') {
|
||||
$raw = vv_npm_req('GET', '/api/nginx/certificates');
|
||||
if (!is_array($raw) || isset($raw['_err']))
|
||||
die(json_encode(['ok' => false, 'error' => $raw['_err'] ?? 'NPM request failed']));
|
||||
|
||||
$master = vv_read_conf_raw('master.conf');
|
||||
preg_match('/^\s*CERT_WARN_DAYS\s*=\s*(\d+)/m', $master, $w);
|
||||
preg_match('/^\s*CERT_CRIT_DAYS\s*=\s*(\d+)/m', $master, $c);
|
||||
$warn = (int)($w[1] ?? 30);
|
||||
$crit = (int)($c[1] ?? 7);
|
||||
$now = time();
|
||||
|
||||
$certs = [];
|
||||
foreach ($raw as $cert) {
|
||||
$exp = !empty($cert['expires_on']) ? strtotime($cert['expires_on']) : false;
|
||||
$days = $exp !== false ? (int)(($exp - $now) / 86400) : null;
|
||||
$status = $days === null ? 'UNKN'
|
||||
: ($days < 0 ? 'CRIT'
|
||||
: ($days <= $crit ? 'CRIT'
|
||||
: ($days <= $warn ? 'WARN' : 'OK')));
|
||||
$certs[] = [
|
||||
'id' => $cert['id'],
|
||||
'nice_name' => $cert['nice_name'] ?? implode(', ', $cert['domain_names'] ?? []),
|
||||
'domain_names'=> $cert['domain_names'] ?? [],
|
||||
'provider' => $cert['provider'] ?? 'unknown',
|
||||
'expires' => !empty($cert['expires_on']) ? substr($cert['expires_on'], 0, 10) : '',
|
||||
'days' => $days,
|
||||
'status' => $status,
|
||||
];
|
||||
}
|
||||
usort($certs, fn($a, $b) => ($a['days'] ?? PHP_INT_MAX) <=> ($b['days'] ?? PHP_INT_MAX));
|
||||
|
||||
echo json_encode(['ok' => true, 'certs' => $certs, 'warn_days' => $warn, 'crit_days' => $crit]);
|
||||
exit;
|
||||
}
|
||||
|
||||
// ── Read configured domains (without running checks) ─────────────────────────
|
||||
if ($action === 'domains') {
|
||||
$hostId = vv_detect_host();
|
||||
$hostIdUp = strtoupper($hostId);
|
||||
$confRaw = ($hostId !== 'unknown') ? vv_read_conf_raw($hostId . '.conf') : '';
|
||||
$master = vv_read_conf_raw('master.conf');
|
||||
|
||||
// Extract CERT_WARN_DAYS / CERT_CRIT_DAYS from master
|
||||
preg_match('/^\s*CERT_WARN_DAYS\s*=\s*(\d+)/m', $master, $w);
|
||||
preg_match('/^\s*CERT_CRIT_DAYS\s*=\s*(\d+)/m', $master, $c);
|
||||
|
||||
// Extract domains array from host conf
|
||||
$domains = [];
|
||||
if (preg_match('/' . $hostIdUp . '_CERT_MONITOR_DOMAINS\s*=\s*\(([^)]*)\)/s', $confRaw, $dm)) {
|
||||
preg_match_all('/"([^"]+)"/', $dm[1], $dd);
|
||||
$domains = $dd[1] ?? [];
|
||||
}
|
||||
|
||||
echo json_encode([
|
||||
'ok' => true,
|
||||
'host_id' => $hostId,
|
||||
'domains' => $domains,
|
||||
'warn_days' => (int)($w[1] ?? 30),
|
||||
'crit_days' => (int)($c[1] ?? 7),
|
||||
]);
|
||||
exit;
|
||||
}
|
||||
|
||||
// ── Run cert_monitor.sh now ───────────────────────────────────────────────────
|
||||
if ($action === 'run') {
|
||||
$script = SCRIPTS_DIR . '/Monitors/cert_monitor.sh';
|
||||
if (!file_exists($script)) {
|
||||
echo json_encode(['ok' => false, 'error' => 'cert_monitor.sh not found']);
|
||||
exit;
|
||||
}
|
||||
set_time_limit(180);
|
||||
exec('bash ' . escapeshellarg($script) . ' 2>&1', $out, $rc);
|
||||
// Read freshly written cache
|
||||
$data = file_exists($cacheFile)
|
||||
? (json_decode(file_get_contents($cacheFile), true) ?: null)
|
||||
: null;
|
||||
echo json_encode([
|
||||
'ok' => true,
|
||||
'data' => $data,
|
||||
'output' => array_slice(array_filter(array_map('trim', $out)), 0, 30),
|
||||
'rc' => $rc,
|
||||
]);
|
||||
exit;
|
||||
}
|
||||
|
||||
// ── Default: return cached status ─────────────────────────────────────────────
|
||||
if (!file_exists($cacheFile)) {
|
||||
// No cache yet — return configured domains so UI can show them unchecked
|
||||
$hostId = vv_detect_host();
|
||||
$hostIdUp = strtoupper($hostId);
|
||||
$confRaw = ($hostId !== 'unknown') ? vv_read_conf_raw($hostId . '.conf') : '';
|
||||
$master = vv_read_conf_raw('master.conf');
|
||||
|
||||
preg_match('/^\s*CERT_WARN_DAYS\s*=\s*(\d+)/m', $master, $w);
|
||||
preg_match('/^\s*CERT_CRIT_DAYS\s*=\s*(\d+)/m', $master, $c);
|
||||
$domains = [];
|
||||
if (preg_match('/' . $hostIdUp . '_CERT_MONITOR_DOMAINS\s*=\s*\(([^)]*)\)/s', $confRaw, $dm)) {
|
||||
preg_match_all('/"([^"]+)"/', $dm[1], $dd);
|
||||
foreach ($dd[1] ?? [] as $d) {
|
||||
$domains[] = ['domain' => $d, 'status' => 'UNKN', 'days' => null, 'expires' => ''];
|
||||
}
|
||||
}
|
||||
|
||||
echo json_encode([
|
||||
'ok' => true,
|
||||
'checked_at' => null,
|
||||
'host' => $hostId !== 'unknown' ? strtoupper($hostId) : null,
|
||||
'warn_days' => (int)($w[1] ?? 30),
|
||||
'crit_days' => (int)($c[1] ?? 7),
|
||||
'domains' => $domains,
|
||||
]);
|
||||
exit;
|
||||
}
|
||||
|
||||
$data = json_decode(file_get_contents($cacheFile), true) ?: [];
|
||||
echo json_encode(array_merge(['ok' => true], $data));
|
||||
+532
@@ -0,0 +1,532 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ================================= Play State Sync ============================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Syncs watched/played state and resume positions across all configured Emby
|
||||
# and Jellyfin servers. Newest timestamp wins — no data is ever lost.
|
||||
#
|
||||
# Users are matched by name (case-insensitive). If a user exists on some servers
|
||||
# but not others, those servers are skipped for that user — no errors, no partial
|
||||
# syncs from unrelated accounts.
|
||||
#
|
||||
# Items are matched by external provider IDs:
|
||||
# Movies → IMDb ID, then TMDB ID
|
||||
# Episodes → TVDB ID + season + episode number
|
||||
# Audio → MusicBrainz Track ID
|
||||
#
|
||||
# ==============================================================================================
|
||||
# SYNC LOGIC
|
||||
# ==============================================================================================
|
||||
#
|
||||
# For each matched item across ≥2 servers:
|
||||
# 1. Compare LastPlayedDate across all servers that have a play record.
|
||||
# 2. The server with the newest LastPlayedDate is authoritative.
|
||||
# 3. Push that server's state (Played, PlayCount, LastPlayedDate,
|
||||
# PlaybackPositionTicks) to every other server.
|
||||
# 4. Servers with no record for that item also receive the state.
|
||||
#
|
||||
# Resume positions (partial plays, not marked Played):
|
||||
# Synced by comparing PlaybackPositionTicks when LastPlayedDate is absent.
|
||||
# The higher tick count wins.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION (host*.conf, aliased by detect_hosts)
|
||||
# ==============================================================================================
|
||||
#
|
||||
# HOST*_TRANSCODE_SERVERS "Name|URL|APIKey|type" entries per host (emby/jellyfin)
|
||||
# All hosts are discovered automatically — no extra config needed.
|
||||
# Remote host URLs have localhost rewritten to their Tailscale IP.
|
||||
#
|
||||
# master.conf
|
||||
#
|
||||
# PLAY_SYNC_ENABLED Master toggle (default: true)
|
||||
# PLAY_SYNC_REMOTE Sync across all hosts via Tailscale (default: true)
|
||||
# false = local servers only (this host's Emby + Jellyfin)
|
||||
# PLAY_SYNC_DAYS How many days back to check for played items (default: 90)
|
||||
# Use 0 to sync all played items (slow on large libraries).
|
||||
# PLAY_SYNC_TYPES Comma-separated item types to sync (default: Movie,Episode,Audio)
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# play_state_sync.sh
|
||||
# Sync all matched users across all configured servers.
|
||||
#
|
||||
# play_state_sync.sh --dry-run
|
||||
# Show what would be synced without writing any state.
|
||||
#
|
||||
# play_state_sync.sh --status
|
||||
# Show configured servers, reachability, and user counts.
|
||||
#
|
||||
# play_state_sync.sh --full
|
||||
# Ignore PLAY_SYNC_DAYS — sync all played items (may be slow).
|
||||
#
|
||||
# play_state_sync.sh --log
|
||||
# Verbose output — show each item comparison.
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
# ── Handle --full before parse_args ───────────────────────────────────────────
|
||||
FULL_SYNC=false
|
||||
_FILTERED=()
|
||||
for _a in "$@"; do
|
||||
[[ "$_a" == "--full" ]] && FULL_SYNC=true || _FILTERED+=("$_a")
|
||||
done
|
||||
parse_args "${_FILTERED[@]}"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
[[ "${PLAY_SYNC_ENABLED:-true}" != "true" ]] && echo "Play state sync disabled" && exit 0
|
||||
|
||||
SYNC_DAYS="${PLAY_SYNC_DAYS:-90}"
|
||||
SYNC_TYPES="${PLAY_SYNC_TYPES:-Movie,Episode,Audio}"
|
||||
[[ "$FULL_SYNC" == true ]] && SYNC_DAYS=0
|
||||
log "$ICON_GEAR Config: days=${SYNC_DAYS} types=${SYNC_TYPES} remote=${PLAY_SYNC_REMOTE:-true}"
|
||||
|
||||
command -v jq >/dev/null 2>&1 || { error "jq is required but not installed"; exit 1; }
|
||||
|
||||
acquire_lock
|
||||
|
||||
# ── Build server list across ALL hosts ────────────────────────────────────────
|
||||
# All HOST*_TRANSCODE_SERVERS arrays are loaded into env by load_config.sh.
|
||||
# For remote hosts, localhost in the URL is rewritten to their Tailscale IP.
|
||||
declare -a SRV_NAME SRV_URL SRV_KEY SRV_TYPE
|
||||
_srv_count=0
|
||||
_my_hostname=$(hostname -s)
|
||||
|
||||
_add_server() {
|
||||
local name="$1" url="$2" key="$3" type="$4"
|
||||
[[ -z "$url" || -z "$key" ]] && return
|
||||
[[ "$key" == "YOUR_API_KEY"* || "$key" == "placeholder"* ]] && return
|
||||
SRV_NAME[$_srv_count]="$name"
|
||||
SRV_URL[$_srv_count]="$url"
|
||||
SRV_KEY[$_srv_count]="$key"
|
||||
SRV_TYPE[$_srv_count]="$type"
|
||||
(( _srv_count++ ))
|
||||
}
|
||||
|
||||
for _varname in $(compgen -v | grep -E '^HOST[0-9]+$' | sort); do
|
||||
_num="${_varname//[^0-9]/}"
|
||||
_host_hostname="${!_varname}"
|
||||
[[ -z "$_host_hostname" ]] && continue
|
||||
|
||||
_is_me=false
|
||||
[[ "${_host_hostname,,}" == "${_my_hostname,,}" ]] && _is_me=true
|
||||
|
||||
# Skip remote hosts when PLAY_SYNC_REMOTE=false
|
||||
if [[ "$_is_me" == false && "${PLAY_SYNC_REMOTE:-true}" != "true" ]]; then
|
||||
log "$_host_hostname — remote sync disabled (PLAY_SYNC_REMOTE=false), skipping"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Resolve Tailscale IP for remote hosts
|
||||
_ts_ip=""
|
||||
if [[ "$_is_me" == false ]]; then
|
||||
_ts_ip=$(resolve_tailscale_ip "$_host_hostname")
|
||||
if [[ -z "$_ts_ip" ]]; then
|
||||
log "$_host_hostname — Tailscale IP not found, skipping"
|
||||
continue
|
||||
fi
|
||||
fi
|
||||
|
||||
# Load this host's TRANSCODE_SERVERS array
|
||||
_srv_arr_name="HOST${_num}_TRANSCODE_SERVERS"
|
||||
eval "_host_entries=(\"\${${_srv_arr_name}[@]}\")"
|
||||
[[ "${#_host_entries[@]}" -eq 0 ]] && continue
|
||||
|
||||
for _entry in "${_host_entries[@]}"; do
|
||||
IFS='|' read -r _name _url _key _type <<< "$_entry"
|
||||
[[ "$_type" == "emby" || "$_type" == "jellyfin" ]] || continue
|
||||
# For remote hosts rewrite localhost/127.0.0.1 → Tailscale IP
|
||||
if [[ "$_is_me" == false ]]; then
|
||||
_url="${_url//localhost/$_ts_ip}"
|
||||
_url="${_url//127.0.0.1/$_ts_ip}"
|
||||
fi
|
||||
_add_server "${_name} (${_host_hostname})" "$_url" "$_key" "$_type"
|
||||
done
|
||||
done
|
||||
|
||||
if [[ "$_srv_count" -lt 2 ]]; then
|
||||
error "Need at least 2 media servers configured — found $_srv_count"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ API Helpers ━━━
|
||||
# ==============================================================================================
|
||||
_api_get() {
|
||||
local url="$1" key="$2" endpoint="$3"
|
||||
curl -sf --max-time 30 \
|
||||
-H "X-Emby-Token: $key" \
|
||||
"${url%/}/${endpoint}" 2>/dev/null
|
||||
}
|
||||
|
||||
_api_post() {
|
||||
local url="$1" key="$2" endpoint="$3" data="${4:-}"
|
||||
if [[ -n "$data" ]]; then
|
||||
curl -sf --max-time 30 -s -o /dev/null -w "%{http_code}" -X POST \
|
||||
-H "X-Emby-Token: $key" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "$data" \
|
||||
"${url%/}/${endpoint}" 2>/dev/null
|
||||
else
|
||||
curl -sf --max-time 30 -s -o /dev/null -w "%{http_code}" -X POST \
|
||||
-H "X-Emby-Token: $key" \
|
||||
"${url%/}/${endpoint}" 2>/dev/null
|
||||
fi
|
||||
}
|
||||
|
||||
# ISO 8601 date → unix seconds (portable, no date -d on BusyBox)
|
||||
_iso_to_epoch() {
|
||||
local dt="$1"
|
||||
[[ -z "$dt" || "$dt" == "null" ]] && echo 0 && return
|
||||
# Strip fractional seconds and Z, convert to seconds
|
||||
dt="${dt%.*}" # remove .NNNNNNN
|
||||
dt="${dt%Z}" # remove trailing Z
|
||||
dt="${dt/T/ }" # T → space
|
||||
date -u -d "$dt UTC" +%s 2>/dev/null || echo 0
|
||||
}
|
||||
|
||||
# Ticks → seconds (1 tick = 100ns, 10_000_000 ticks = 1s)
|
||||
_ticks_to_sec() {
|
||||
echo $(( ${1:-0} / 10000000 ))
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY PLAY STATE SYNC STATUS ━━━━━"
|
||||
echo "$ICON_GEAR Remote sync: ${PLAY_SYNC_REMOTE:-true}"
|
||||
echo "$ICON_GEAR Sync days: ${SYNC_DAYS:-all}"
|
||||
echo "$ICON_GEAR Item types: $SYNC_TYPES"
|
||||
echo ""
|
||||
for i in $(seq 0 $(( _srv_count - 1 ))); do
|
||||
echo "$ICON_HOST [${SRV_TYPE[$i]}] ${SRV_NAME[$i]} (${SRV_URL[$i]})"
|
||||
_users=$(_api_get "${SRV_URL[$i]}" "${SRV_KEY[$i]}" "Users" 2>/dev/null | jq -r '.[].Name' 2>/dev/null | wc -l)
|
||||
if [[ "$_users" -gt 0 ]]; then
|
||||
echo " $ICON_DONE Reachable — $_users user(s)"
|
||||
_api_get "${SRV_URL[$i]}" "${SRV_KEY[$i]}" "Users" 2>/dev/null \
|
||||
| jq -r '.[].Name' 2>/dev/null | while read -r n; do echo " · $n"; done
|
||||
else
|
||||
echo " $ICON_ERROR Unreachable or no users"
|
||||
fi
|
||||
done
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Main Sync ━━━
|
||||
# ==============================================================================================
|
||||
echo "━━━ $ICON_SYNC Play State Sync — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no state will be written"
|
||||
[[ "$FULL_SYNC" == true ]] && log "Full sync mode — ignoring PLAY_SYNC_DAYS"
|
||||
|
||||
START=$(date +%s)
|
||||
TOTAL_SYNCED=0
|
||||
TOTAL_SKIPPED=0
|
||||
TOTAL_ERRORS=0
|
||||
|
||||
# ── Step 1: Fetch users from each server ─────────────────────────────────────
|
||||
declare -A SRV_USERS # idx → JSON array string of users
|
||||
|
||||
log "Fetching users..."
|
||||
for i in $(seq 0 $(( _srv_count - 1 ))); do
|
||||
_resp=$(_api_get "${SRV_URL[$i]}" "${SRV_KEY[$i]}" "Users")
|
||||
if [[ -z "$_resp" ]]; then
|
||||
warn "${SRV_NAME[$i]} — unreachable, skipping"
|
||||
SRV_USERS[$i]=""
|
||||
continue
|
||||
fi
|
||||
SRV_USERS[$i]="$_resp"
|
||||
_count=$(echo "$_resp" | jq 'length' 2>/dev/null || echo 0)
|
||||
log "${SRV_NAME[$i]} — $_count user(s)"
|
||||
done
|
||||
|
||||
# ── Step 2: Build cross-server user map ──────────────────────────────────────
|
||||
# lowercase_name → "server_idx:user_id server_idx:user_id ..."
|
||||
declare -A USER_MAP
|
||||
|
||||
for i in $(seq 0 $(( _srv_count - 1 ))); do
|
||||
[[ -z "${SRV_USERS[$i]}" ]] && continue
|
||||
while IFS=$'\t' read -r uid uname; do
|
||||
[[ -z "$uid" || -z "$uname" ]] && continue
|
||||
lname="${uname,,}"
|
||||
if [[ -n "${USER_MAP[$lname]}" ]]; then
|
||||
USER_MAP[$lname]+=" ${i}:${uid}"
|
||||
else
|
||||
USER_MAP[$lname]="${i}:${uid}"
|
||||
fi
|
||||
done < <(echo "${SRV_USERS[$i]}" | jq -r '.[] | [.Id, .Name] | @tsv' 2>/dev/null)
|
||||
done
|
||||
|
||||
# ── Step 3: Sync per matched user ────────────────────────────────────────────
|
||||
_date_filter=""
|
||||
if [[ "$SYNC_DAYS" -gt 0 ]]; then
|
||||
_cutoff=$(date -u -d "$SYNC_DAYS days ago" '+%Y-%m-%dT%H:%M:%SZ' 2>/dev/null || \
|
||||
date -u -v "-${SYNC_DAYS}d" '+%Y-%m-%dT%H:%M:%SZ' 2>/dev/null)
|
||||
[[ -n "$_cutoff" ]] && _date_filter="&MinDateLastSaved=$_cutoff"
|
||||
fi
|
||||
|
||||
for lname in "${!USER_MAP[@]}"; do
|
||||
read -ra _pairs <<< "${USER_MAP[$lname]}"
|
||||
|
||||
# Skip users only on one server
|
||||
[[ "${#_pairs[@]}" -lt 2 ]] && log " $lname — only on 1 server, skipping" && continue
|
||||
|
||||
echo ""
|
||||
echo "── User: $lname (${#_pairs[@]} server(s)) ──"
|
||||
|
||||
# Build per-server user context
|
||||
declare -A U_IDX U_UID
|
||||
for _pair in "${_pairs[@]}"; do
|
||||
IFS=':' read -r _si _ui <<< "$_pair"
|
||||
U_IDX["$_si"]="$_si"
|
||||
U_UID["$_si"]="$_ui"
|
||||
done
|
||||
|
||||
# ── Fetch played items from each server for this user ────────────────────
|
||||
# Key: provider_id_string → sorted list of (epoch, srv_idx, item_id, play_count, ticks, played)
|
||||
declare -A ITEM_MAP # provider_key → JSON per-server data
|
||||
|
||||
for _si in "${!U_IDX[@]}"; do
|
||||
_uid="${U_UID[$_si]}"
|
||||
_endpoint="Users/${_uid}/Items?Recursive=true&Fields=ProviderIds,UserData,Type,ParentIndexNumber,IndexNumber,SeriesName&IncludeItemTypes=${SYNC_TYPES}&Filters=IsPlayed${_date_filter}&Limit=5000"
|
||||
_resp=$(_api_get "${SRV_URL[$_si]}" "${SRV_KEY[$_si]}" "$_endpoint")
|
||||
if [[ -z "$_resp" ]]; then
|
||||
warn " ${SRV_NAME[$_si]} — failed to fetch items for $lname"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Also fetch items with resume position (not yet marked played)
|
||||
_endpoint2="Users/${_uid}/Items?Recursive=true&Fields=ProviderIds,UserData,Type,ParentIndexNumber,IndexNumber,SeriesName&IncludeItemTypes=${SYNC_TYPES}&SortBy=DatePlayed&SortOrder=Descending&Filters=IsResumable${_date_filter}&Limit=500"
|
||||
_resp2=$(_api_get "${SRV_URL[$_si]}" "${SRV_KEY[$_si]}" "$_endpoint2")
|
||||
|
||||
# Combine and deduplicate by Id
|
||||
if [[ -n "$_resp2" ]]; then
|
||||
_combined=$(printf '%s\n%s' "$_resp" "$_resp2" | jq -s \
|
||||
'[.[0].Items // [], .[1].Items // []] | add // [] | unique_by(.Id)' 2>/dev/null)
|
||||
else
|
||||
_combined=$(echo "$_resp" | jq '.Items // []' 2>/dev/null)
|
||||
fi
|
||||
|
||||
_count=$(echo "$_combined" | jq 'length' 2>/dev/null || echo 0)
|
||||
log " ${SRV_NAME[$_si]} — $_count item(s) with state for $lname"
|
||||
|
||||
# Build item lookup by provider key
|
||||
while IFS=$'\t' read -r iid itype season ep imdb tmdb tvdb mbtrack played ticks lplayed pcount; do
|
||||
# Build canonical provider key
|
||||
_pkey=""
|
||||
case "$itype" in
|
||||
Movie)
|
||||
[[ "$imdb" != "null" && -n "$imdb" ]] && _pkey="imdb:${imdb}"
|
||||
[[ -z "$_pkey" && "$tmdb" != "null" && -n "$tmdb" ]] && _pkey="tmdb:movie:${tmdb}"
|
||||
;;
|
||||
Episode)
|
||||
[[ "$tvdb" != "null" && -n "$tvdb" && "$season" != "null" && "$ep" != "null" ]] && \
|
||||
_pkey="tvdb:ep:${tvdb}:s${season}e${ep}"
|
||||
;;
|
||||
Audio)
|
||||
[[ "$mbtrack" != "null" && -n "$mbtrack" ]] && _pkey="mb:track:${mbtrack}"
|
||||
;;
|
||||
esac
|
||||
[[ -z "$_pkey" ]] && continue
|
||||
|
||||
_epoch=$(_iso_to_epoch "$lplayed")
|
||||
_entry="${_si}|${iid}|${played}|${pcount}|${ticks}|${_epoch}|${lplayed}"
|
||||
|
||||
if [[ -n "${ITEM_MAP[$_pkey]}" ]]; then
|
||||
ITEM_MAP[$_pkey]+=$'\n'"$_entry"
|
||||
else
|
||||
ITEM_MAP[$_pkey]="$_entry"
|
||||
fi
|
||||
|
||||
done < <(echo "$_combined" | jq -r '.[] | [
|
||||
.Id,
|
||||
.Type,
|
||||
(.ParentIndexNumber // "null" | tostring),
|
||||
(.IndexNumber // "null" | tostring),
|
||||
(.ProviderIds.Imdb // "null"),
|
||||
(.ProviderIds.Tmdb // "null"),
|
||||
(.ProviderIds.Tvdb // "null"),
|
||||
(.ProviderIds.MusicBrainzTrackId // "null"),
|
||||
(.UserData.Played // false | tostring),
|
||||
(.UserData.PlaybackPositionTicks // 0 | tostring),
|
||||
(.UserData.LastPlayedDate // "null"),
|
||||
(.UserData.PlayCount // 0 | tostring)
|
||||
] | @tsv' 2>/dev/null)
|
||||
done
|
||||
|
||||
# ── Compare and sync ─────────────────────────────────────────────────────
|
||||
for _pkey in "${!ITEM_MAP[@]}"; do
|
||||
# Collect all server entries for this item
|
||||
declare -A E_EPOCH E_PLAYED E_PCOUNT E_TICKS E_IID E_LPLAYED E_SIDX
|
||||
_has_entries=false
|
||||
|
||||
while IFS='|' read -r _si _iid _played _pcount _ticks _epoch _lplayed; do
|
||||
[[ -z "$_si" ]] && continue
|
||||
E_SIDX[$_si]="$_si"
|
||||
E_IID[$_si]="$_iid"
|
||||
E_PLAYED[$_si]="$_played"
|
||||
E_PCOUNT[$_si]="$_pcount"
|
||||
E_TICKS[$_si]="$_ticks"
|
||||
E_EPOCH[$_si]="$_epoch"
|
||||
E_LPLAYED[$_si]="$_lplayed"
|
||||
_has_entries=true
|
||||
done <<< "${ITEM_MAP[$_pkey]}"
|
||||
|
||||
[[ "$_has_entries" == false ]] && continue
|
||||
|
||||
# Find the authoritative server: newest LastPlayedDate epoch
|
||||
# Tie-break: higher PlayCount, then higher Ticks
|
||||
_auth_si=""
|
||||
_auth_epoch=0
|
||||
_auth_pcount=0
|
||||
_auth_ticks=0
|
||||
|
||||
for _si in "${!E_SIDX[@]}"; do
|
||||
_e="${E_EPOCH[$_si]:-0}"
|
||||
_pc="${E_PCOUNT[$_si]:-0}"
|
||||
_tk="${E_TICKS[$_si]:-0}"
|
||||
if [[ "$_e" -gt "$_auth_epoch" ]] || \
|
||||
[[ "$_e" -eq "$_auth_epoch" && "$_pc" -gt "$_auth_pcount" ]] || \
|
||||
[[ "$_e" -eq "$_auth_epoch" && "$_pc" -eq "$_auth_pcount" && "$_tk" -gt "$_auth_ticks" ]]; then
|
||||
_auth_si="$_si"
|
||||
_auth_epoch="$_e"
|
||||
_auth_pcount="$_pc"
|
||||
_auth_ticks="$_tk"
|
||||
fi
|
||||
done
|
||||
|
||||
[[ -z "$_auth_si" ]] && continue
|
||||
|
||||
_auth_played="${E_PLAYED[$_auth_si]}"
|
||||
_auth_lplayed="${E_LPLAYED[$_auth_si]}"
|
||||
_auth_pcount="${E_PCOUNT[$_auth_si]}"
|
||||
_auth_ticks="${E_TICKS[$_auth_si]}"
|
||||
|
||||
# Push to servers with older state OR no state at all
|
||||
for _si in "${!U_IDX[@]}"; do
|
||||
[[ "$_si" == "$_auth_si" ]] && continue
|
||||
_their_epoch="${E_EPOCH[$_si]:-0}"
|
||||
_their_played="${E_PLAYED[$_si]:-false}"
|
||||
|
||||
# Skip if they already have the same/newer state
|
||||
if [[ "$_their_epoch" -ge "$_auth_epoch" ]] && \
|
||||
[[ "$_their_played" == "$_auth_played" ]]; then
|
||||
log " SKIP $_pkey → ${SRV_NAME[$_si]} already up to date"
|
||||
(( TOTAL_SKIPPED++ ))
|
||||
continue
|
||||
fi
|
||||
|
||||
_uid="${U_UID[$_si]}"
|
||||
_iid="${E_IID[$_si]:-}" # might not exist on this server yet
|
||||
|
||||
# Find item ID on target server by provider key if not in our map
|
||||
if [[ -z "$_iid" ]]; then
|
||||
_ptype="${_pkey%%:*}"
|
||||
_pval="${_pkey##*:}"
|
||||
case "$_ptype" in
|
||||
imdb) _search_field="imdb.${_pval}" ;;
|
||||
tmdb) _search_field="tmdb.${_pval##movie:}" ;;
|
||||
tvdb)
|
||||
# _pkey format: tvdb:ep:{tvdb_id}:s{season}e{ep}
|
||||
# ##*: gives "s7e2" (wrong); strip prefix then first :
|
||||
_tvdb_num="${_pkey#tvdb:ep:}"; _tvdb_num="${_tvdb_num%%:*}"
|
||||
_search_field="tvdb.${_tvdb_num}"
|
||||
;;
|
||||
mb) _search_field="" ;; # skip music if not found
|
||||
esac
|
||||
|
||||
if [[ -n "$_search_field" ]]; then
|
||||
_iid=$(_api_get "${SRV_URL[$_si]}" "${SRV_KEY[$_si]}" \
|
||||
"Items?AnyProviderIdEquals=${_search_field}&Recursive=true&Fields=ProviderIds&Limit=1" 2>/dev/null \
|
||||
| jq -r '.Items[0].Id // empty' 2>/dev/null)
|
||||
fi
|
||||
[[ -z "$_iid" ]] && log " SKIP $_pkey → ${SRV_NAME[$_si]} item not found on server" && continue
|
||||
fi
|
||||
|
||||
log " SYNC $_pkey → ${SRV_NAME[$_si]} (auth: ${SRV_NAME[$_auth_si]}, epoch: $_auth_epoch)"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo " DRY RUN: would sync $_pkey → ${SRV_NAME[$_si]} user=$lname played=$_auth_played date=$_auth_lplayed"
|
||||
(( TOTAL_SYNCED++ ))
|
||||
continue
|
||||
fi
|
||||
|
||||
# Write state to target server
|
||||
if [[ "$_auth_played" == "true" ]]; then
|
||||
# Mark as played with date
|
||||
_date_param=""
|
||||
if [[ "$_auth_lplayed" != "null" && -n "$_auth_lplayed" ]]; then
|
||||
# Emby's PlayedItems endpoint rejects 7-digit fractional seconds (.0000000)
|
||||
# with HTTP 500; strip to whole seconds before encoding
|
||||
if [[ "$_auth_lplayed" == *.* ]]; then
|
||||
_lp="${_auth_lplayed%.*}"
|
||||
[[ "$_auth_lplayed" == *Z ]] && _lp+="Z"
|
||||
else
|
||||
_lp="$_auth_lplayed"
|
||||
fi
|
||||
_date_param="?DatePlayed=${_lp//[: ]/%3A}"
|
||||
fi
|
||||
_http=$(_api_post "${SRV_URL[$_si]}" "${SRV_KEY[$_si]}" \
|
||||
"Users/${_uid}/PlayedItems/${_iid}${_date_param}")
|
||||
if [[ "$_http" == "200" || "$_http" == "201" ]]; then
|
||||
success " ✓ $_pkey → ${SRV_NAME[$_si]} marked played"
|
||||
(( TOTAL_SYNCED++ ))
|
||||
else
|
||||
warn " ✗ $_pkey → ${SRV_NAME[$_si]} failed (HTTP ${_http:-err})"
|
||||
(( TOTAL_ERRORS++ ))
|
||||
fi
|
||||
else
|
||||
# Sync resume position only
|
||||
_ticks_int=$(( ${_auth_ticks:-0} ))
|
||||
if [[ "$_ticks_int" -gt 0 ]]; then
|
||||
_payload="{\"PlaybackPositionTicks\":${_ticks_int}}"
|
||||
_http=$(_api_post "${SRV_URL[$_si]}" "${SRV_KEY[$_si]}" \
|
||||
"Users/${_uid}/Items/${_iid}/UserData" "$_payload")
|
||||
if [[ "$_http" == "200" || "$_http" == "204" ]]; then
|
||||
success " ✓ $_pkey → ${SRV_NAME[$_si]} resume synced ($(_ticks_to_sec "$_ticks_int")s)"
|
||||
(( TOTAL_SYNCED++ ))
|
||||
else
|
||||
warn " ✗ $_pkey → ${SRV_NAME[$_si]} resume sync failed (HTTP ${_http:-err})"
|
||||
(( TOTAL_ERRORS++ ))
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
unset E_EPOCH E_PLAYED E_PCOUNT E_TICKS E_IID E_LPLAYED E_SIDX
|
||||
done
|
||||
|
||||
unset ITEM_MAP U_IDX U_UID
|
||||
done
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
END=$(date +%s)
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY PLAY STATE SYNC SUMMARY ━━━━━"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo "$ICON_DONE Synced: $TOTAL_SYNCED"
|
||||
[[ "$TOTAL_SKIPPED" -gt 0 ]] && echo "$ICON_RUNNING Skipped: $TOTAL_SKIPPED (already current)"
|
||||
[[ "$TOTAL_ERRORS" -gt 0 ]] && echo "$ICON_ERROR Errors: $TOTAL_ERRORS"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no changes written"
|
||||
elif [[ "$TOTAL_ERRORS" -eq 0 ]]; then
|
||||
success "Done ✅"
|
||||
else
|
||||
warn "Done with $TOTAL_ERRORS error(s)"
|
||||
fi
|
||||
+532
@@ -0,0 +1,532 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ================================= Play State Sync ============================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Syncs watched/played state and resume positions across all configured Emby
|
||||
# and Jellyfin servers. Newest timestamp wins — no data is ever lost.
|
||||
#
|
||||
# Users are matched by name (case-insensitive). If a user exists on some servers
|
||||
# but not others, those servers are skipped for that user — no errors, no partial
|
||||
# syncs from unrelated accounts.
|
||||
#
|
||||
# Items are matched by external provider IDs:
|
||||
# Movies → IMDb ID, then TMDB ID
|
||||
# Episodes → TVDB ID + season + episode number
|
||||
# Audio → MusicBrainz Track ID
|
||||
#
|
||||
# ==============================================================================================
|
||||
# SYNC LOGIC
|
||||
# ==============================================================================================
|
||||
#
|
||||
# For each matched item across ≥2 servers:
|
||||
# 1. Compare LastPlayedDate across all servers that have a play record.
|
||||
# 2. The server with the newest LastPlayedDate is authoritative.
|
||||
# 3. Push that server's state (Played, PlayCount, LastPlayedDate,
|
||||
# PlaybackPositionTicks) to every other server.
|
||||
# 4. Servers with no record for that item also receive the state.
|
||||
#
|
||||
# Resume positions (partial plays, not marked Played):
|
||||
# Synced by comparing PlaybackPositionTicks when LastPlayedDate is absent.
|
||||
# The higher tick count wins.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION (host*.conf, aliased by detect_hosts)
|
||||
# ==============================================================================================
|
||||
#
|
||||
# HOST*_TRANSCODE_SERVERS "Name|URL|APIKey|type" entries per host (emby/jellyfin)
|
||||
# All hosts are discovered automatically — no extra config needed.
|
||||
# Remote host URLs have localhost rewritten to their Tailscale IP.
|
||||
#
|
||||
# master.conf
|
||||
#
|
||||
# PLAY_SYNC_ENABLED Master toggle (default: true)
|
||||
# PLAY_SYNC_REMOTE Sync across all hosts via Tailscale (default: true)
|
||||
# false = local servers only (this host's Emby + Jellyfin)
|
||||
# PLAY_SYNC_DAYS How many days back to check for played items (default: 90)
|
||||
# Use 0 to sync all played items (slow on large libraries).
|
||||
# PLAY_SYNC_TYPES Comma-separated item types to sync (default: Movie,Episode,Audio)
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# play_state_sync.sh
|
||||
# Sync all matched users across all configured servers.
|
||||
#
|
||||
# play_state_sync.sh --dry-run
|
||||
# Show what would be synced without writing any state.
|
||||
#
|
||||
# play_state_sync.sh --status
|
||||
# Show configured servers, reachability, and user counts.
|
||||
#
|
||||
# play_state_sync.sh --full
|
||||
# Ignore PLAY_SYNC_DAYS — sync all played items (may be slow).
|
||||
#
|
||||
# play_state_sync.sh --log
|
||||
# Verbose output — show each item comparison.
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
# ── Handle --full before parse_args ───────────────────────────────────────────
|
||||
FULL_SYNC=false
|
||||
_FILTERED=()
|
||||
for _a in "$@"; do
|
||||
[[ "$_a" == "--full" ]] && FULL_SYNC=true || _FILTERED+=("$_a")
|
||||
done
|
||||
parse_args "${_FILTERED[@]}"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
[[ "${PLAY_SYNC_ENABLED:-true}" != "true" ]] && echo "Play state sync disabled" && exit 0
|
||||
|
||||
SYNC_DAYS="${PLAY_SYNC_DAYS:-90}"
|
||||
SYNC_TYPES="${PLAY_SYNC_TYPES:-Movie,Episode,Audio}"
|
||||
[[ "$FULL_SYNC" == true ]] && SYNC_DAYS=0
|
||||
log "$ICON_GEAR Config: days=${SYNC_DAYS} types=${SYNC_TYPES} remote=${PLAY_SYNC_REMOTE:-true}"
|
||||
|
||||
command -v jq >/dev/null 2>&1 || { error "jq is required but not installed"; exit 1; }
|
||||
|
||||
acquire_lock
|
||||
|
||||
# ── Build server list across ALL hosts ────────────────────────────────────────
|
||||
# All HOST*_TRANSCODE_SERVERS arrays are loaded into env by load_config.sh.
|
||||
# For remote hosts, localhost in the URL is rewritten to their Tailscale IP.
|
||||
declare -a SRV_NAME SRV_URL SRV_KEY SRV_TYPE
|
||||
_srv_count=0
|
||||
_my_hostname=$(hostname -s)
|
||||
|
||||
_add_server() {
|
||||
local name="$1" url="$2" key="$3" type="$4"
|
||||
[[ -z "$url" || -z "$key" ]] && return
|
||||
[[ "$key" == "YOUR_API_KEY"* || "$key" == "placeholder"* ]] && return
|
||||
SRV_NAME[$_srv_count]="$name"
|
||||
SRV_URL[$_srv_count]="$url"
|
||||
SRV_KEY[$_srv_count]="$key"
|
||||
SRV_TYPE[$_srv_count]="$type"
|
||||
(( _srv_count++ ))
|
||||
}
|
||||
|
||||
for _varname in $(compgen -v | grep -E '^HOST[0-9]+$' | sort); do
|
||||
_num="${_varname//[^0-9]/}"
|
||||
_host_hostname="${!_varname}"
|
||||
[[ -z "$_host_hostname" ]] && continue
|
||||
|
||||
_is_me=false
|
||||
[[ "${_host_hostname,,}" == "${_my_hostname,,}" ]] && _is_me=true
|
||||
|
||||
# Skip remote hosts when PLAY_SYNC_REMOTE=false
|
||||
if [[ "$_is_me" == false && "${PLAY_SYNC_REMOTE:-true}" != "true" ]]; then
|
||||
log "$_host_hostname — remote sync disabled (PLAY_SYNC_REMOTE=false), skipping"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Resolve Tailscale IP for remote hosts
|
||||
_ts_ip=""
|
||||
if [[ "$_is_me" == false ]]; then
|
||||
_ts_ip=$(resolve_tailscale_ip "$_host_hostname")
|
||||
if [[ -z "$_ts_ip" ]]; then
|
||||
log "$_host_hostname — Tailscale IP not found, skipping"
|
||||
continue
|
||||
fi
|
||||
fi
|
||||
|
||||
# Load this host's TRANSCODE_SERVERS array
|
||||
_srv_arr_name="HOST${_num}_TRANSCODE_SERVERS"
|
||||
eval "_host_entries=(\"\${${_srv_arr_name}[@]}\")"
|
||||
[[ "${#_host_entries[@]}" -eq 0 ]] && continue
|
||||
|
||||
for _entry in "${_host_entries[@]}"; do
|
||||
IFS='|' read -r _name _url _key _type <<< "$_entry"
|
||||
[[ "$_type" == "emby" || "$_type" == "jellyfin" ]] || continue
|
||||
# For remote hosts rewrite localhost/127.0.0.1 → Tailscale IP
|
||||
if [[ "$_is_me" == false ]]; then
|
||||
_url="${_url//localhost/$_ts_ip}"
|
||||
_url="${_url//127.0.0.1/$_ts_ip}"
|
||||
fi
|
||||
_add_server "${_name} (${_host_hostname})" "$_url" "$_key" "$_type"
|
||||
done
|
||||
done
|
||||
|
||||
if [[ "$_srv_count" -lt 2 ]]; then
|
||||
error "Need at least 2 media servers configured — found $_srv_count"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ API Helpers ━━━
|
||||
# ==============================================================================================
|
||||
_api_get() {
|
||||
local url="$1" key="$2" endpoint="$3"
|
||||
curl -sf --max-time 30 \
|
||||
-H "X-Emby-Token: $key" \
|
||||
"${url%/}/${endpoint}" 2>/dev/null
|
||||
}
|
||||
|
||||
_api_post() {
|
||||
local url="$1" key="$2" endpoint="$3" data="${4:-}"
|
||||
if [[ -n "$data" ]]; then
|
||||
curl -sf --max-time 30 -s -o /dev/null -w "%{http_code}" -X POST \
|
||||
-H "X-Emby-Token: $key" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "$data" \
|
||||
"${url%/}/${endpoint}" 2>/dev/null
|
||||
else
|
||||
curl -sf --max-time 30 -s -o /dev/null -w "%{http_code}" -X POST \
|
||||
-H "X-Emby-Token: $key" \
|
||||
"${url%/}/${endpoint}" 2>/dev/null
|
||||
fi
|
||||
}
|
||||
|
||||
# ISO 8601 date → unix seconds (portable, no date -d on BusyBox)
|
||||
_iso_to_epoch() {
|
||||
local dt="$1"
|
||||
[[ -z "$dt" || "$dt" == "null" ]] && echo 0 && return
|
||||
# Strip fractional seconds and Z, convert to seconds
|
||||
dt="${dt%.*}" # remove .NNNNNNN
|
||||
dt="${dt%Z}" # remove trailing Z
|
||||
dt="${dt/T/ }" # T → space
|
||||
date -u -d "$dt UTC" +%s 2>/dev/null || echo 0
|
||||
}
|
||||
|
||||
# Ticks → seconds (1 tick = 100ns, 10_000_000 ticks = 1s)
|
||||
_ticks_to_sec() {
|
||||
echo $(( ${1:-0} / 10000000 ))
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY PLAY STATE SYNC STATUS ━━━━━"
|
||||
echo "$ICON_GEAR Remote sync: ${PLAY_SYNC_REMOTE:-true}"
|
||||
echo "$ICON_GEAR Sync days: ${SYNC_DAYS:-all}"
|
||||
echo "$ICON_GEAR Item types: $SYNC_TYPES"
|
||||
echo ""
|
||||
for i in $(seq 0 $(( _srv_count - 1 ))); do
|
||||
echo "$ICON_HOST [${SRV_TYPE[$i]}] ${SRV_NAME[$i]} (${SRV_URL[$i]})"
|
||||
_users=$(_api_get "${SRV_URL[$i]}" "${SRV_KEY[$i]}" "Users" 2>/dev/null | jq -r '.[].Name' 2>/dev/null | wc -l)
|
||||
if [[ "$_users" -gt 0 ]]; then
|
||||
echo " $ICON_DONE Reachable — $_users user(s)"
|
||||
_api_get "${SRV_URL[$i]}" "${SRV_KEY[$i]}" "Users" 2>/dev/null \
|
||||
| jq -r '.[].Name' 2>/dev/null | while read -r n; do echo " · $n"; done
|
||||
else
|
||||
echo " $ICON_ERROR Unreachable or no users"
|
||||
fi
|
||||
done
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Main Sync ━━━
|
||||
# ==============================================================================================
|
||||
echo "━━━ $ICON_SYNC Play State Sync — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no state will be written"
|
||||
[[ "$FULL_SYNC" == true ]] && log "Full sync mode — ignoring PLAY_SYNC_DAYS"
|
||||
|
||||
START=$(date +%s)
|
||||
TOTAL_SYNCED=0
|
||||
TOTAL_SKIPPED=0
|
||||
TOTAL_ERRORS=0
|
||||
|
||||
# ── Step 1: Fetch users from each server ─────────────────────────────────────
|
||||
declare -A SRV_USERS # idx → JSON array string of users
|
||||
|
||||
log "Fetching users..."
|
||||
for i in $(seq 0 $(( _srv_count - 1 ))); do
|
||||
_resp=$(_api_get "${SRV_URL[$i]}" "${SRV_KEY[$i]}" "Users")
|
||||
if [[ -z "$_resp" ]]; then
|
||||
warn "${SRV_NAME[$i]} — unreachable, skipping"
|
||||
SRV_USERS[$i]=""
|
||||
continue
|
||||
fi
|
||||
SRV_USERS[$i]="$_resp"
|
||||
_count=$(echo "$_resp" | jq 'length' 2>/dev/null || echo 0)
|
||||
log "${SRV_NAME[$i]} — $_count user(s)"
|
||||
done
|
||||
|
||||
# ── Step 2: Build cross-server user map ──────────────────────────────────────
|
||||
# lowercase_name → "server_idx:user_id server_idx:user_id ..."
|
||||
declare -A USER_MAP
|
||||
|
||||
for i in $(seq 0 $(( _srv_count - 1 ))); do
|
||||
[[ -z "${SRV_USERS[$i]}" ]] && continue
|
||||
while IFS=$'\t' read -r uid uname; do
|
||||
[[ -z "$uid" || -z "$uname" ]] && continue
|
||||
lname="${uname,,}"
|
||||
if [[ -n "${USER_MAP[$lname]}" ]]; then
|
||||
USER_MAP[$lname]+=" ${i}:${uid}"
|
||||
else
|
||||
USER_MAP[$lname]="${i}:${uid}"
|
||||
fi
|
||||
done < <(echo "${SRV_USERS[$i]}" | jq -r '.[] | [.Id, .Name] | @tsv' 2>/dev/null)
|
||||
done
|
||||
|
||||
# ── Step 3: Sync per matched user ────────────────────────────────────────────
|
||||
_date_filter=""
|
||||
if [[ "$SYNC_DAYS" -gt 0 ]]; then
|
||||
_cutoff=$(date -u -d "$SYNC_DAYS days ago" '+%Y-%m-%dT%H:%M:%SZ' 2>/dev/null || \
|
||||
date -u -v "-${SYNC_DAYS}d" '+%Y-%m-%dT%H:%M:%SZ' 2>/dev/null)
|
||||
[[ -n "$_cutoff" ]] && _date_filter="&MinDateLastSaved=$_cutoff"
|
||||
fi
|
||||
|
||||
for lname in "${!USER_MAP[@]}"; do
|
||||
read -ra _pairs <<< "${USER_MAP[$lname]}"
|
||||
|
||||
# Skip users only on one server
|
||||
[[ "${#_pairs[@]}" -lt 2 ]] && log " $lname — only on 1 server, skipping" && continue
|
||||
|
||||
echo ""
|
||||
echo "── User: $lname (${#_pairs[@]} server(s)) ──"
|
||||
|
||||
# Build per-server user context
|
||||
declare -A U_IDX U_UID
|
||||
for _pair in "${_pairs[@]}"; do
|
||||
IFS=':' read -r _si _ui <<< "$_pair"
|
||||
U_IDX["$_si"]="$_si"
|
||||
U_UID["$_si"]="$_ui"
|
||||
done
|
||||
|
||||
# ── Fetch played items from each server for this user ────────────────────
|
||||
# Key: provider_id_string → sorted list of (epoch, srv_idx, item_id, play_count, ticks, played)
|
||||
declare -A ITEM_MAP # provider_key → JSON per-server data
|
||||
|
||||
for _si in "${!U_IDX[@]}"; do
|
||||
_uid="${U_UID[$_si]}"
|
||||
_endpoint="Users/${_uid}/Items?Recursive=true&Fields=ProviderIds,UserData,Type,ParentIndexNumber,IndexNumber,SeriesName&IncludeItemTypes=${SYNC_TYPES}&Filters=IsPlayed${_date_filter}&Limit=5000"
|
||||
_resp=$(_api_get "${SRV_URL[$_si]}" "${SRV_KEY[$_si]}" "$_endpoint")
|
||||
if [[ -z "$_resp" ]]; then
|
||||
warn " ${SRV_NAME[$_si]} — failed to fetch items for $lname"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Also fetch items with resume position (not yet marked played)
|
||||
_endpoint2="Users/${_uid}/Items?Recursive=true&Fields=ProviderIds,UserData,Type,ParentIndexNumber,IndexNumber,SeriesName&IncludeItemTypes=${SYNC_TYPES}&SortBy=DatePlayed&SortOrder=Descending&Filters=IsResumable${_date_filter}&Limit=500"
|
||||
_resp2=$(_api_get "${SRV_URL[$_si]}" "${SRV_KEY[$_si]}" "$_endpoint2")
|
||||
|
||||
# Combine and deduplicate by Id
|
||||
if [[ -n "$_resp2" ]]; then
|
||||
_combined=$(printf '%s\n%s' "$_resp" "$_resp2" | jq -s \
|
||||
'[.[0].Items // [], .[1].Items // []] | add // [] | unique_by(.Id)' 2>/dev/null)
|
||||
else
|
||||
_combined=$(echo "$_resp" | jq '.Items // []' 2>/dev/null)
|
||||
fi
|
||||
|
||||
_count=$(echo "$_combined" | jq 'length' 2>/dev/null || echo 0)
|
||||
log " ${SRV_NAME[$_si]} — $_count item(s) with state for $lname"
|
||||
|
||||
# Build item lookup by provider key
|
||||
while IFS=$'\t' read -r iid itype season ep imdb tmdb tvdb mbtrack played ticks lplayed pcount; do
|
||||
# Build canonical provider key
|
||||
_pkey=""
|
||||
case "$itype" in
|
||||
Movie)
|
||||
[[ "$imdb" != "null" && -n "$imdb" ]] && _pkey="imdb:${imdb}"
|
||||
[[ -z "$_pkey" && "$tmdb" != "null" && -n "$tmdb" ]] && _pkey="tmdb:movie:${tmdb}"
|
||||
;;
|
||||
Episode)
|
||||
[[ "$tvdb" != "null" && -n "$tvdb" && "$season" != "null" && "$ep" != "null" ]] && \
|
||||
_pkey="tvdb:ep:${tvdb}:s${season}e${ep}"
|
||||
;;
|
||||
Audio)
|
||||
[[ "$mbtrack" != "null" && -n "$mbtrack" ]] && _pkey="mb:track:${mbtrack}"
|
||||
;;
|
||||
esac
|
||||
[[ -z "$_pkey" ]] && continue
|
||||
|
||||
_epoch=$(_iso_to_epoch "$lplayed")
|
||||
_entry="${_si}|${iid}|${played}|${pcount}|${ticks}|${_epoch}|${lplayed}"
|
||||
|
||||
if [[ -n "${ITEM_MAP[$_pkey]}" ]]; then
|
||||
ITEM_MAP[$_pkey]+=$'\n'"$_entry"
|
||||
else
|
||||
ITEM_MAP[$_pkey]="$_entry"
|
||||
fi
|
||||
|
||||
done < <(echo "$_combined" | jq -r '.[] | [
|
||||
.Id,
|
||||
.Type,
|
||||
(.ParentIndexNumber // "null" | tostring),
|
||||
(.IndexNumber // "null" | tostring),
|
||||
(.ProviderIds.Imdb // "null"),
|
||||
(.ProviderIds.Tmdb // "null"),
|
||||
(.ProviderIds.Tvdb // "null"),
|
||||
(.ProviderIds.MusicBrainzTrackId // "null"),
|
||||
(.UserData.Played // false | tostring),
|
||||
(.UserData.PlaybackPositionTicks // 0 | tostring),
|
||||
(.UserData.LastPlayedDate // "null"),
|
||||
(.UserData.PlayCount // 0 | tostring)
|
||||
] | @tsv' 2>/dev/null)
|
||||
done
|
||||
|
||||
# ── Compare and sync ─────────────────────────────────────────────────────
|
||||
for _pkey in "${!ITEM_MAP[@]}"; do
|
||||
# Collect all server entries for this item
|
||||
declare -A E_EPOCH E_PLAYED E_PCOUNT E_TICKS E_IID E_LPLAYED E_SIDX
|
||||
_has_entries=false
|
||||
|
||||
while IFS='|' read -r _si _iid _played _pcount _ticks _epoch _lplayed; do
|
||||
[[ -z "$_si" ]] && continue
|
||||
E_SIDX[$_si]="$_si"
|
||||
E_IID[$_si]="$_iid"
|
||||
E_PLAYED[$_si]="$_played"
|
||||
E_PCOUNT[$_si]="$_pcount"
|
||||
E_TICKS[$_si]="$_ticks"
|
||||
E_EPOCH[$_si]="$_epoch"
|
||||
E_LPLAYED[$_si]="$_lplayed"
|
||||
_has_entries=true
|
||||
done <<< "${ITEM_MAP[$_pkey]}"
|
||||
|
||||
[[ "$_has_entries" == false ]] && continue
|
||||
|
||||
# Find the authoritative server: newest LastPlayedDate epoch
|
||||
# Tie-break: higher PlayCount, then higher Ticks
|
||||
_auth_si=""
|
||||
_auth_epoch=0
|
||||
_auth_pcount=0
|
||||
_auth_ticks=0
|
||||
|
||||
for _si in "${!E_SIDX[@]}"; do
|
||||
_e="${E_EPOCH[$_si]:-0}"
|
||||
_pc="${E_PCOUNT[$_si]:-0}"
|
||||
_tk="${E_TICKS[$_si]:-0}"
|
||||
if [[ "$_e" -gt "$_auth_epoch" ]] || \
|
||||
[[ "$_e" -eq "$_auth_epoch" && "$_pc" -gt "$_auth_pcount" ]] || \
|
||||
[[ "$_e" -eq "$_auth_epoch" && "$_pc" -eq "$_auth_pcount" && "$_tk" -gt "$_auth_ticks" ]]; then
|
||||
_auth_si="$_si"
|
||||
_auth_epoch="$_e"
|
||||
_auth_pcount="$_pc"
|
||||
_auth_ticks="$_tk"
|
||||
fi
|
||||
done
|
||||
|
||||
[[ -z "$_auth_si" ]] && continue
|
||||
|
||||
_auth_played="${E_PLAYED[$_auth_si]}"
|
||||
_auth_lplayed="${E_LPLAYED[$_auth_si]}"
|
||||
_auth_pcount="${E_PCOUNT[$_auth_si]}"
|
||||
_auth_ticks="${E_TICKS[$_auth_si]}"
|
||||
|
||||
# Push to servers with older state OR no state at all
|
||||
for _si in "${!U_IDX[@]}"; do
|
||||
[[ "$_si" == "$_auth_si" ]] && continue
|
||||
_their_epoch="${E_EPOCH[$_si]:-0}"
|
||||
_their_played="${E_PLAYED[$_si]:-false}"
|
||||
|
||||
# Skip if they already have the same/newer state
|
||||
if [[ "$_their_epoch" -ge "$_auth_epoch" ]] && \
|
||||
[[ "$_their_played" == "$_auth_played" ]]; then
|
||||
log " SKIP $_pkey → ${SRV_NAME[$_si]} already up to date"
|
||||
(( TOTAL_SKIPPED++ ))
|
||||
continue
|
||||
fi
|
||||
|
||||
_uid="${U_UID[$_si]}"
|
||||
_iid="${E_IID[$_si]:-}" # might not exist on this server yet
|
||||
|
||||
# Find item ID on target server by provider key if not in our map
|
||||
if [[ -z "$_iid" ]]; then
|
||||
_ptype="${_pkey%%:*}"
|
||||
_pval="${_pkey##*:}"
|
||||
case "$_ptype" in
|
||||
imdb) _search_field="imdb.${_pval}" ;;
|
||||
tmdb) _search_field="tmdb.${_pval##movie:}" ;;
|
||||
tvdb)
|
||||
# _pkey format: tvdb:ep:{tvdb_id}:s{season}e{ep}
|
||||
# ##*: gives "s7e2" (wrong); strip prefix then first :
|
||||
_tvdb_num="${_pkey#tvdb:ep:}"; _tvdb_num="${_tvdb_num%%:*}"
|
||||
_search_field="tvdb.${_tvdb_num}"
|
||||
;;
|
||||
mb) _search_field="" ;; # skip music if not found
|
||||
esac
|
||||
|
||||
if [[ -n "$_search_field" ]]; then
|
||||
_iid=$(_api_get "${SRV_URL[$_si]}" "${SRV_KEY[$_si]}" \
|
||||
"Items?AnyProviderIdEquals=${_search_field}&Recursive=true&Fields=ProviderIds&ExcludeLocationTypes=Virtual&Limit=1" 2>/dev/null \
|
||||
| jq -r '.Items[0].Id // empty' 2>/dev/null)
|
||||
fi
|
||||
[[ -z "$_iid" ]] && log " SKIP $_pkey → ${SRV_NAME[$_si]} item not found on server" && continue
|
||||
fi
|
||||
|
||||
log " SYNC $_pkey → ${SRV_NAME[$_si]} (auth: ${SRV_NAME[$_auth_si]}, epoch: $_auth_epoch)"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo " DRY RUN: would sync $_pkey → ${SRV_NAME[$_si]} user=$lname played=$_auth_played date=$_auth_lplayed"
|
||||
(( TOTAL_SYNCED++ ))
|
||||
continue
|
||||
fi
|
||||
|
||||
# Write state to target server
|
||||
if [[ "$_auth_played" == "true" ]]; then
|
||||
# Mark as played with date
|
||||
_date_param=""
|
||||
if [[ "$_auth_lplayed" != "null" && -n "$_auth_lplayed" ]]; then
|
||||
# Emby's PlayedItems endpoint rejects 7-digit fractional seconds (.0000000)
|
||||
# with HTTP 500; strip to whole seconds before encoding
|
||||
if [[ "$_auth_lplayed" == *.* ]]; then
|
||||
_lp="${_auth_lplayed%.*}"
|
||||
[[ "$_auth_lplayed" == *Z ]] && _lp+="Z"
|
||||
else
|
||||
_lp="$_auth_lplayed"
|
||||
fi
|
||||
_date_param="?DatePlayed=${_lp//[: ]/%3A}"
|
||||
fi
|
||||
_http=$(_api_post "${SRV_URL[$_si]}" "${SRV_KEY[$_si]}" \
|
||||
"Users/${_uid}/PlayedItems/${_iid}${_date_param}")
|
||||
if [[ "$_http" == "200" || "$_http" == "201" ]]; then
|
||||
success " ✓ $_pkey → ${SRV_NAME[$_si]} marked played"
|
||||
(( TOTAL_SYNCED++ ))
|
||||
else
|
||||
warn " ✗ $_pkey → ${SRV_NAME[$_si]} failed (HTTP ${_http:-err})"
|
||||
(( TOTAL_ERRORS++ ))
|
||||
fi
|
||||
else
|
||||
# Sync resume position only
|
||||
_ticks_int=$(( ${_auth_ticks:-0} ))
|
||||
if [[ "$_ticks_int" -gt 0 ]]; then
|
||||
_payload="{\"PlaybackPositionTicks\":${_ticks_int}}"
|
||||
_http=$(_api_post "${SRV_URL[$_si]}" "${SRV_KEY[$_si]}" \
|
||||
"Users/${_uid}/Items/${_iid}/UserData" "$_payload")
|
||||
if [[ "$_http" == "200" || "$_http" == "204" ]]; then
|
||||
success " ✓ $_pkey → ${SRV_NAME[$_si]} resume synced ($(_ticks_to_sec "$_ticks_int")s)"
|
||||
(( TOTAL_SYNCED++ ))
|
||||
else
|
||||
warn " ✗ $_pkey → ${SRV_NAME[$_si]} resume sync failed (HTTP ${_http:-err})"
|
||||
(( TOTAL_ERRORS++ ))
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
unset E_EPOCH E_PLAYED E_PCOUNT E_TICKS E_IID E_LPLAYED E_SIDX
|
||||
done
|
||||
|
||||
unset ITEM_MAP U_IDX U_UID
|
||||
done
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
END=$(date +%s)
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY PLAY STATE SYNC SUMMARY ━━━━━"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo "$ICON_DONE Synced: $TOTAL_SYNCED"
|
||||
[[ "$TOTAL_SKIPPED" -gt 0 ]] && echo "$ICON_RUNNING Skipped: $TOTAL_SKIPPED (already current)"
|
||||
[[ "$TOTAL_ERRORS" -gt 0 ]] && echo "$ICON_ERROR Errors: $TOTAL_ERRORS"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no changes written"
|
||||
elif [[ "$TOTAL_ERRORS" -eq 0 ]]; then
|
||||
success "Done ✅"
|
||||
else
|
||||
warn "Done with $TOTAL_ERRORS error(s)"
|
||||
fi
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,401 @@
|
||||
<?php
|
||||
// Config file parser and writer.
|
||||
// Reads master.conf and the appropriate host*.conf based on running host.
|
||||
|
||||
define('PLUGIN_CFG', '/boot/config/plugins/varaverk/varaverk.cfg');
|
||||
|
||||
$_vv_cfg = @parse_ini_file(PLUGIN_CFG) ?: [];
|
||||
define('SCRIPTS_DIR', $_vv_cfg['SCRIPTS_DIR'] ?? '/boot/config/plugins/varaverk');
|
||||
define('CONF_DIR', SCRIPTS_DIR . '/Configurations');
|
||||
define('DATA_DIR', SCRIPTS_DIR . '/data');
|
||||
define('STATE_DIR', SCRIPTS_DIR . '/State_Files');
|
||||
define('LOG_DIR', '/var/log/varaverk');
|
||||
unset($_vv_cfg);
|
||||
|
||||
define('VV_SETUP_STATE_FILE', STATE_DIR . '/varaverk_setup.db');
|
||||
define('VV_CACHE_DIR', '/tmp/vv_cache');
|
||||
|
||||
// Read the setup state file into a key=>value array.
|
||||
function vv_setup_state_read(): array {
|
||||
$out = [];
|
||||
foreach (file(VV_SETUP_STATE_FILE) ?: [] as $line) {
|
||||
[$k, $v] = explode('=', trim($line), 2) + ['', ''];
|
||||
if ($k !== '') $out[$k] = $v;
|
||||
}
|
||||
return $out;
|
||||
}
|
||||
|
||||
// Write the setup state file (creates or overwrites).
|
||||
function vv_setup_state_write(array $data): void {
|
||||
$content = '';
|
||||
foreach ($data as $k => $v) $content .= "$k=$v\n";
|
||||
file_put_contents(VV_SETUP_STATE_FILE, $content);
|
||||
}
|
||||
|
||||
// Push the setup state file to all remote hosts via scp.
|
||||
// Unlike master.conf push, this does NOT require the plugin to be installed on the remote —
|
||||
// it only needs SSH to be reachable, and pushes to /boot/config/ (always available).
|
||||
function vv_push_setup_state(): void {
|
||||
if (!file_exists(VV_SETUP_STATE_FILE)) return;
|
||||
$myHostId = vv_detect_host();
|
||||
$vars = vv_conf_vars();
|
||||
$sshKey = $vars[strtoupper($myHostId) . '_SSH_KEY'] ?? '';
|
||||
if (!$sshKey || !file_exists($sshKey)) return;
|
||||
|
||||
$master = vv_read_conf_raw('master.conf');
|
||||
preg_match_all('/^\s*(HOST\d+)(?:_NAME)?\s*=\s*["\']?(\S+?)["\']?\s*$/m', $master, $m);
|
||||
$seen = [];
|
||||
foreach ($m[1] as $i => $hostKey) {
|
||||
$hostId = strtolower($hostKey);
|
||||
if ($hostId === $myHostId || isset($seen[$hostId])) continue;
|
||||
$seen[$hostId] = true;
|
||||
$hostname = trim($m[2][$i]);
|
||||
if (!$hostname) continue;
|
||||
$ip = vv_resolve_tailscale_ip($hostname);
|
||||
if (!$ip) continue;
|
||||
$sshBase = 'ssh -i ' . escapeshellarg($sshKey)
|
||||
. ' -o ConnectTimeout=10 -o StrictHostKeyChecking=no root@' . $ip;
|
||||
// Ensure the target dir exists (it always should on Unraid, but be safe)
|
||||
shell_exec($sshBase . ' "mkdir -p /boot/config" 2>/dev/null');
|
||||
$dest = escapeshellarg('root@' . $ip . ':/boot/config/varaverk_setup.db');
|
||||
exec('scp -i ' . escapeshellarg($sshKey)
|
||||
. ' -o ConnectTimeout=10 -o StrictHostKeyChecking=no'
|
||||
. ' ' . escapeshellarg(VV_SETUP_STATE_FILE) . ' ' . $dest . ' 2>&1');
|
||||
}
|
||||
}
|
||||
|
||||
// Push master.conf to all remote hosts via scp after a local save.
|
||||
// Returns one result entry per remote found in master.conf.
|
||||
// Silently returns [] on non-owner hosts (no SSH key, no remote access).
|
||||
function vv_push_master_conf(): array {
|
||||
$myHostId = vv_detect_host();
|
||||
$vars = vv_conf_vars();
|
||||
$sshKey = $vars[strtoupper($myHostId) . '_SSH_KEY'] ?? '';
|
||||
if (!$sshKey || !file_exists($sshKey)) return [];
|
||||
|
||||
$localPath = CONF_DIR . '/master.conf';
|
||||
$master = vv_read_conf_raw('master.conf');
|
||||
preg_match_all('/^\s*(HOST\d+)(?:_NAME)?\s*=\s*["\']?(\S+?)["\']?\s*$/m', $master, $m);
|
||||
|
||||
$results = [];
|
||||
$seen = [];
|
||||
foreach ($m[1] as $i => $hostKey) {
|
||||
$hostId = strtolower($hostKey);
|
||||
if ($hostId === $myHostId || isset($seen[$hostId])) continue;
|
||||
$seen[$hostId] = true;
|
||||
|
||||
$hostname = trim($m[2][$i]);
|
||||
$ip = vv_resolve_tailscale_ip($hostname);
|
||||
if (!$ip) {
|
||||
$results[] = ['host' => $hostKey, 'ok' => false, 'ready' => false, 'error' => 'Tailscale IP not found'];
|
||||
continue;
|
||||
}
|
||||
|
||||
// Single SSH call: get remote SCRIPTS_DIR and verify plugin is installed,
|
||||
// Configurations/ exists, and master.conf is already present.
|
||||
// Any missing piece means the remote isn't ready — skip rather than push blind.
|
||||
$sshBase = 'ssh -i ' . escapeshellarg($sshKey)
|
||||
. ' -o ConnectTimeout=10 -o StrictHostKeyChecking=no root@' . $ip;
|
||||
$probe = trim(shell_exec(
|
||||
$sshBase . ' "cfg=$(grep SCRIPTS_DIR /boot/config/plugins/varaverk/varaverk.cfg 2>/dev/null)'
|
||||
. ' && sd=$(echo \"$cfg\" | grep -oP \'(?<=SCRIPTS_DIR=\")[^\"]+\')'
|
||||
. ' && test -d \"${sd}/Configurations\"'
|
||||
. ' && test -f \"${sd}/Configurations/master.conf\"'
|
||||
. ' && echo \"$sd\""'
|
||||
) ?: '');
|
||||
|
||||
if ($probe === '') {
|
||||
$results[] = ['host' => $hostKey, 'ok' => false, 'ready' => false,
|
||||
'error' => 'plugin not installed, dir missing, or master.conf absent — skipped'];
|
||||
continue;
|
||||
}
|
||||
|
||||
$remoteConf = rtrim($probe, '/') . '/Configurations';
|
||||
$dest = escapeshellarg('root@' . $ip . ':' . $remoteConf . '/master.conf');
|
||||
$cmd = 'scp -i ' . escapeshellarg($sshKey)
|
||||
. ' -o ConnectTimeout=10 -o StrictHostKeyChecking=no'
|
||||
. ' ' . escapeshellarg($localPath) . ' ' . $dest . ' 2>&1';
|
||||
exec($cmd, $out, $rc);
|
||||
$results[] = [
|
||||
'host' => $hostKey,
|
||||
'ok' => $rc === 0,
|
||||
'error' => $rc !== 0 ? implode('; ', $out) : '',
|
||||
];
|
||||
}
|
||||
return $results;
|
||||
}
|
||||
|
||||
function vv_get_hostname(): string {
|
||||
return trim(shell_exec('hostname -s') ?: '');
|
||||
}
|
||||
|
||||
// Mirror of common.sh resolve_tailscale_ip(): tries `tailscale ip -4` first (Tailscale manages
|
||||
// the mapping so this survives IP changes), falls back to parsing `tailscale status` text.
|
||||
function vv_resolve_tailscale_ip(string $hostname): string {
|
||||
$h = strtolower($hostname);
|
||||
$ip = trim(shell_exec('tailscale ip -4 ' . escapeshellarg($h) . ' 2>/dev/null') ?: '');
|
||||
if ($ip) return $ip;
|
||||
$out = shell_exec('tailscale status 2>/dev/null') ?: '';
|
||||
foreach (explode("\n", $out) as $line) {
|
||||
$cols = preg_split('/\s+/', trim($line));
|
||||
if (isset($cols[1]) && stripos($cols[1], $h . '.') === 0) return $cols[0];
|
||||
}
|
||||
return '';
|
||||
}
|
||||
|
||||
function vv_detect_host(): string {
|
||||
// Reads master.conf for HOST1="name" (or HOST1_NAME="name") and matches running hostname.
|
||||
// Returns 'host1', 'host2', 'host3', ... or 'unknown'. Works for any number of hosts.
|
||||
$master = vv_read_conf_raw('master.conf');
|
||||
preg_match_all('/^\s*(HOST\d+)(?:_NAME)?\s*=\s*["\']?(\S+?)["\']?\s*$/m', $master, $m);
|
||||
$hostname = vv_get_hostname();
|
||||
foreach ($m[1] as $i => $key) {
|
||||
if (strcasecmp($hostname, trim($m[2][$i])) === 0) return strtolower($key);
|
||||
}
|
||||
return 'unknown';
|
||||
}
|
||||
|
||||
function vv_is_owner(): bool {
|
||||
return vv_detect_host() === 'host1';
|
||||
}
|
||||
|
||||
function vv_read_conf_raw(string $filename): string {
|
||||
$path = CONF_DIR . '/' . $filename;
|
||||
return file_exists($path) ? file_get_contents($path) : '';
|
||||
}
|
||||
|
||||
function vv_write_conf_raw(string $filename, string $content): bool {
|
||||
$path = CONF_DIR . '/' . $filename;
|
||||
$tmp = $path . '.vv.tmp';
|
||||
if (file_put_contents($tmp, $content) === false) return false;
|
||||
return rename($tmp, $path);
|
||||
}
|
||||
|
||||
function vv_get_conf_files(): array {
|
||||
// Returns conf files this host is allowed to view/edit
|
||||
$host = vv_detect_host();
|
||||
$files = [];
|
||||
if ($host === 'host1') {
|
||||
// Owner sees master.conf + their own host conf
|
||||
$files[] = 'master.conf';
|
||||
$files[] = 'host1.conf';
|
||||
} elseif (preg_match('/^host(\d+)$/', $host)) {
|
||||
// Any other numbered host sees only their own conf
|
||||
$files[] = $host . '.conf';
|
||||
} else {
|
||||
// Unknown host — show all for dev/debug
|
||||
foreach (glob(CONF_DIR . '/*.conf') as $f) {
|
||||
$files[] = basename($f);
|
||||
}
|
||||
}
|
||||
return $files;
|
||||
}
|
||||
|
||||
// Parse conf into key=>value map for $VAR substitution in docs
|
||||
function vv_conf_vars(): array {
|
||||
$vars = [];
|
||||
$files = ['master.conf'];
|
||||
$host = vv_detect_host();
|
||||
if (preg_match('/^host\d+$/', $host)) $files[] = $host . '.conf';
|
||||
|
||||
foreach ($files as $f) {
|
||||
$raw = vv_read_conf_raw($f);
|
||||
// Match: VAR_NAME="value" or VAR_NAME=value (no quotes)
|
||||
preg_match_all('/^\s*([A-Z0-9_]+)\s*=\s*["\']?([^"\'#\n]*?)["\']?\s*(?:#.*)?$/m', $raw, $m);
|
||||
foreach ($m[1] as $i => $key) {
|
||||
$vars[$key] = str_replace('\\$', '$', trim($m[2][$i]));
|
||||
}
|
||||
}
|
||||
return $vars;
|
||||
}
|
||||
|
||||
// Query the Unraid GraphQL API for a given host.
|
||||
// For the local host queries http://localhost/graphql; for remote hosts uses the Tailscale IP.
|
||||
// $apiKey may be passed explicitly (needed when querying a remote host from the local host,
|
||||
// since vv_conf_vars() only loads the current host's conf file).
|
||||
// Returns the decoded 'data' object on success, null on any failure.
|
||||
// Debug log written to /tmp/vv_api_debug.json on failure.
|
||||
function vv_unraid_api_query(string $hostId, string $gql, int $timeoutSec = 5, string $apiKey = ''): ?array {
|
||||
$vars = vv_conf_vars();
|
||||
$key = $apiKey ?: ($vars[strtoupper($hostId) . '_UNRAID_API_KEY'] ?? '');
|
||||
if (!$key) return null;
|
||||
|
||||
$myHostId = vv_detect_host();
|
||||
if (strtolower($hostId) === strtolower($myHostId)) {
|
||||
$url = 'http://localhost/graphql';
|
||||
} else {
|
||||
$hostname = $vars[strtoupper($hostId)] ?? '';
|
||||
if (!$hostname) return null;
|
||||
$ip = vv_resolve_tailscale_ip($hostname);
|
||||
if (!$ip) return null;
|
||||
$url = "http://{$ip}/graphql";
|
||||
}
|
||||
|
||||
$body = json_encode(['query' => $gql]);
|
||||
|
||||
// Use curl (preferred — doesn't require allow_url_fopen, better error handling).
|
||||
if (function_exists('curl_init')) {
|
||||
$ch = curl_init($url);
|
||||
curl_setopt_array($ch, [
|
||||
CURLOPT_POST => true,
|
||||
CURLOPT_HTTPHEADER => ['Content-Type: application/json', "x-api-key: {$key}"],
|
||||
CURLOPT_POSTFIELDS => $body,
|
||||
CURLOPT_RETURNTRANSFER => true,
|
||||
CURLOPT_TIMEOUT => $timeoutSec,
|
||||
CURLOPT_CONNECTTIMEOUT => 3,
|
||||
CURLOPT_FOLLOWLOCATION => false,
|
||||
]);
|
||||
$resp = curl_exec($ch);
|
||||
$httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
|
||||
$curlErr = curl_error($ch);
|
||||
curl_close($ch);
|
||||
} else {
|
||||
// Fallback to file_get_contents if curl is unavailable.
|
||||
$ctx = stream_context_create(['http' => [
|
||||
'method' => 'POST',
|
||||
'header' => "Content-Type: application/json\r\nx-api-key: {$key}",
|
||||
'content' => $body,
|
||||
'timeout' => $timeoutSec,
|
||||
'ignore_errors' => true,
|
||||
]]);
|
||||
$resp = @file_get_contents($url, false, $ctx);
|
||||
$httpCode = $resp !== false ? 200 : 0;
|
||||
$curlErr = '';
|
||||
}
|
||||
|
||||
if ($resp === false || $resp === '' || ($httpCode !== 0 && $httpCode !== 200)) {
|
||||
@file_put_contents(VV_CACHE_DIR . '/vv_api_debug.json', json_encode([
|
||||
'ts' => time(),
|
||||
'host' => $hostId,
|
||||
'url' => $url,
|
||||
'http_code' => $httpCode,
|
||||
'curl_err' => $curlErr,
|
||||
'response' => substr((string)$resp, 0, 800),
|
||||
], JSON_PRETTY_PRINT));
|
||||
return null;
|
||||
}
|
||||
|
||||
$decoded = json_decode((string)$resp, true);
|
||||
|
||||
// If the API returned GraphQL errors, log them for diagnosis.
|
||||
if (!empty($decoded['errors'])) {
|
||||
@file_put_contents(VV_CACHE_DIR . '/vv_api_debug.json', json_encode([
|
||||
'ts' => time(),
|
||||
'host' => $hostId,
|
||||
'url' => $url,
|
||||
'http_code' => $httpCode,
|
||||
'errors' => $decoded['errors'],
|
||||
'data' => $decoded['data'] ?? null,
|
||||
], JSON_PRETTY_PRINT));
|
||||
}
|
||||
|
||||
// data key present (even if null means query ran but returned nothing useful).
|
||||
return array_key_exists('data', $decoded ?? []) ? $decoded['data'] : null;
|
||||
}
|
||||
|
||||
// ── File-based API cache (/tmp/vv_cache — tmpfs, cleared on reboot) ───────────
|
||||
|
||||
// Read a cached payload. Returns null if missing or older than $maxAge seconds.
|
||||
function vv_cache_read(string $key, int $maxAge = 90): ?array {
|
||||
$f = VV_CACHE_DIR . '/' . $key . '.json';
|
||||
if (!file_exists($f) || (time() - filemtime($f)) > $maxAge) return null;
|
||||
$raw = file_get_contents($f);
|
||||
return $raw ? (json_decode($raw, true) ?: null) : null;
|
||||
}
|
||||
|
||||
// Write a payload atomically (tmp + rename) so readers never see a partial file.
|
||||
function vv_cache_write(string $key, array $data): void {
|
||||
if (!is_dir(VV_CACHE_DIR)) @mkdir(VV_CACHE_DIR, 0755, true);
|
||||
$f = VV_CACHE_DIR . '/' . $key . '.json';
|
||||
$tmp = $f . '.tmp';
|
||||
file_put_contents($tmp, json_encode($data));
|
||||
rename($tmp, $f);
|
||||
}
|
||||
|
||||
// ── Shared utility functions (used across include/ and api/ files) ────────────
|
||||
|
||||
// Format seconds into "2d 3h 15m".
|
||||
function vv_format_uptime(int $seconds): string {
|
||||
$d = intdiv($seconds, 86400);
|
||||
$h = intdiv($seconds % 86400, 3600);
|
||||
$m = intdiv($seconds % 3600, 60);
|
||||
return ($d ? "{$d}d " : '') . ($h ? "{$h}h " : '') . "{$m}m";
|
||||
}
|
||||
|
||||
// Parse a scalar value from raw conf text. Matches KEY="value" or KEY=value.
|
||||
// Identical logic was previously duplicated as vv_arr_scalar / vv_wd_scalar /
|
||||
// vv_fb_scalar / vv_media_conf_scalar — all reduce to this one regex.
|
||||
function vv_parse_conf_scalar(string $raw, string $key): string {
|
||||
return preg_match('/^\s*' . preg_quote($key, '/') . '\s*=\s*"?([^"\n]*)"?/m', $raw, $m)
|
||||
? trim($m[1]) : '';
|
||||
}
|
||||
|
||||
// Parse a key=value state file (e.g. fallback_state.db, partnership_state.db).
|
||||
// Returns ['key' => 'value', ...]. Lines without '=' are ignored.
|
||||
function vv_parse_kv_db(string $text): array {
|
||||
$out = [];
|
||||
foreach (explode("\n", $text) as $line) {
|
||||
$line = trim($line);
|
||||
if ($line === '' || $line[0] === '#') continue;
|
||||
[$k, $v] = array_pad(explode('=', $line, 2), 2, '');
|
||||
if ($k !== '') $out[trim($k)] = trim($v);
|
||||
}
|
||||
return $out;
|
||||
}
|
||||
|
||||
// All configured hosts from master.conf as ['host1' => 'hostname', ...].
|
||||
// Canonical version — previously duplicated as vv_arr_known_hosts / vv_fb_known_hosts.
|
||||
function vv_known_hosts(): array {
|
||||
$vars = vv_conf_vars();
|
||||
$hosts = [];
|
||||
foreach ($vars as $k => $v) {
|
||||
if (preg_match('/^HOST(\d+)$/', $k, $m) && $v !== '') {
|
||||
$hosts['host' . $m[1]] = $v;
|
||||
}
|
||||
}
|
||||
ksort($hosts);
|
||||
return $hosts ?: ['host1' => 'HOST1'];
|
||||
}
|
||||
|
||||
// Create (or overwrite) the Varaverk Unraid API key and write it into host conf.
|
||||
// Returns ['ok'=>true,'key_preview'=>'...'] or ['ok'=>false,'error'=>'...'].
|
||||
function vv_auto_create_api_key(string $hostId, string $confFile): array {
|
||||
$varName = strtoupper($hostId) . '_UNRAID_API_KEY';
|
||||
$hostname = trim((string)shell_exec("hostname -s 2>/dev/null | sed 's/^[Uu][Nn][Rr][Aa][Ii][Dd]-//'")) ?: 'Varaverk';
|
||||
$keyName = 'Varaverk ' . $hostname;
|
||||
$output = shell_exec('timeout 10 /usr/local/sbin/unraid-api apikey --name ' . escapeshellarg($keyName) . ' --create --overwrite --description "Varaverk plugin" --roles ADMIN --json </dev/null 2>&1');
|
||||
if (!$output) {
|
||||
return ['ok' => false, 'error' => 'unraid-api returned no output'];
|
||||
}
|
||||
$data = json_decode(trim($output), true);
|
||||
$key = $data['key'] ?? null;
|
||||
if (!$key) {
|
||||
return ['ok' => false, 'error' => 'No key in response'];
|
||||
}
|
||||
$raw = vv_read_conf_raw($confFile);
|
||||
if ($raw === '') {
|
||||
return ['ok' => false, 'error' => 'Cannot read ' . $confFile];
|
||||
}
|
||||
if (!str_contains($raw, $varName)) {
|
||||
foreach ([strtoupper($hostId) . '_OWNER_EMAIL', strtoupper($hostId) . '_SSH_KEY'] as $anchor) {
|
||||
if (str_contains($raw, $anchor)) {
|
||||
$raw = preg_replace('/^(\s*' . preg_quote($anchor, '/') . '\s*=.*$)/m',
|
||||
'$1' . "\n " . $varName . '=""', $raw, 1);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
$raw = preg_replace('/^(\s*' . preg_quote($varName, '/') . '\s*=\s*)"[^"]*"/m',
|
||||
'${1}"' . $key . '"', $raw);
|
||||
vv_write_conf_raw($confFile, $raw);
|
||||
return ['ok' => true, 'key_preview' => substr($key, 0, 8) . '...' . substr($key, -4)];
|
||||
}
|
||||
|
||||
// Local LAN IP via routing table — static-cached per request.
|
||||
// Previously duplicated in include/docker_folders.php and inline in include/docker.php.
|
||||
function vv_local_ip(): string {
|
||||
static $ip = null;
|
||||
if ($ip !== null) return $ip;
|
||||
$ip = trim(shell_exec("ip route get 8.8.8.8 2>/dev/null | awk '/src/{for(i=1;i<=NF;i++)if(\$i==\"src\")print \$(i+1)}'") ?? '');
|
||||
return $ip;
|
||||
}
|
||||
@@ -0,0 +1,386 @@
|
||||
<?php
|
||||
// Config file parser and writer.
|
||||
// Reads master.conf and the appropriate host*.conf based on running host.
|
||||
|
||||
define('PLUGIN_CFG', '/boot/config/plugins/varaverk/varaverk.cfg');
|
||||
|
||||
$_vv_cfg = @parse_ini_file(PLUGIN_CFG) ?: [];
|
||||
define('SCRIPTS_DIR', $_vv_cfg['SCRIPTS_DIR'] ?? '/boot/config/plugins/varaverk');
|
||||
define('CONF_DIR', SCRIPTS_DIR . '/Configurations');
|
||||
define('DATA_DIR', SCRIPTS_DIR . '/data');
|
||||
define('STATE_DIR', SCRIPTS_DIR . '/State_Files');
|
||||
define('LOG_DIR', '/var/log/varaverk');
|
||||
unset($_vv_cfg);
|
||||
|
||||
define('VV_SETUP_STATE_FILE', STATE_DIR . '/varaverk_setup.db');
|
||||
define('VV_CACHE_DIR', '/tmp/vv_cache');
|
||||
|
||||
// Read the setup state file into a key=>value array.
|
||||
function vv_setup_state_read(): array {
|
||||
$out = [];
|
||||
foreach (file(VV_SETUP_STATE_FILE) ?: [] as $line) {
|
||||
[$k, $v] = explode('=', trim($line), 2) + ['', ''];
|
||||
if ($k !== '') $out[$k] = $v;
|
||||
}
|
||||
return $out;
|
||||
}
|
||||
|
||||
// Write the setup state file (creates or overwrites).
|
||||
function vv_setup_state_write(array $data): void {
|
||||
$content = '';
|
||||
foreach ($data as $k => $v) $content .= "$k=$v\n";
|
||||
file_put_contents(VV_SETUP_STATE_FILE, $content);
|
||||
}
|
||||
|
||||
// Push the setup state file to all remote hosts via scp.
|
||||
// Unlike master.conf push, this does NOT require the plugin to be installed on the remote —
|
||||
// it only needs SSH to be reachable, and pushes to /boot/config/ (always available).
|
||||
function vv_push_setup_state(): void {
|
||||
if (!file_exists(VV_SETUP_STATE_FILE)) return;
|
||||
$myHostId = vv_detect_host();
|
||||
$vars = vv_conf_vars();
|
||||
$sshKey = $vars[strtoupper($myHostId) . '_SSH_KEY'] ?? '';
|
||||
if (!$sshKey || !file_exists($sshKey)) return;
|
||||
|
||||
$master = vv_read_conf_raw('master.conf');
|
||||
preg_match_all('/^\s*(HOST\d+)(?:_NAME)?\s*=\s*["\']?(\S+?)["\']?\s*$/m', $master, $m);
|
||||
$seen = [];
|
||||
foreach ($m[1] as $i => $hostKey) {
|
||||
$hostId = strtolower($hostKey);
|
||||
if ($hostId === $myHostId || isset($seen[$hostId])) continue;
|
||||
$seen[$hostId] = true;
|
||||
$hostname = trim($m[2][$i]);
|
||||
if (!$hostname) continue;
|
||||
$ip = vv_resolve_tailscale_ip($hostname);
|
||||
if (!$ip) continue;
|
||||
$sshBase = 'ssh -i ' . escapeshellarg($sshKey)
|
||||
. ' -o ConnectTimeout=10 -o StrictHostKeyChecking=no root@' . $ip;
|
||||
// Ensure the target dir exists (it always should on Unraid, but be safe)
|
||||
shell_exec($sshBase . ' "mkdir -p /boot/config" 2>/dev/null');
|
||||
$dest = escapeshellarg('root@' . $ip . ':/boot/config/varaverk_setup.db');
|
||||
exec('scp -i ' . escapeshellarg($sshKey)
|
||||
. ' -o ConnectTimeout=10 -o StrictHostKeyChecking=no'
|
||||
. ' ' . escapeshellarg(VV_SETUP_STATE_FILE) . ' ' . $dest . ' 2>&1');
|
||||
}
|
||||
}
|
||||
|
||||
// Push master.conf to all remote hosts via scp after a local save.
|
||||
// Returns one result entry per remote found in master.conf.
|
||||
// Silently returns [] on non-owner hosts (no SSH key, no remote access).
|
||||
function vv_push_master_conf(): array {
|
||||
$myHostId = vv_detect_host();
|
||||
$vars = vv_conf_vars();
|
||||
$sshKey = $vars[strtoupper($myHostId) . '_SSH_KEY'] ?? '';
|
||||
if (!$sshKey || !file_exists($sshKey)) return [];
|
||||
|
||||
$localPath = CONF_DIR . '/master.conf';
|
||||
$master = vv_read_conf_raw('master.conf');
|
||||
preg_match_all('/^\s*(HOST\d+)(?:_NAME)?\s*=\s*["\']?(\S+?)["\']?\s*$/m', $master, $m);
|
||||
|
||||
$results = [];
|
||||
$seen = [];
|
||||
foreach ($m[1] as $i => $hostKey) {
|
||||
$hostId = strtolower($hostKey);
|
||||
if ($hostId === $myHostId || isset($seen[$hostId])) continue;
|
||||
$seen[$hostId] = true;
|
||||
|
||||
$hostname = trim($m[2][$i]);
|
||||
$ip = vv_resolve_tailscale_ip($hostname);
|
||||
if (!$ip) {
|
||||
$results[] = ['host' => $hostKey, 'ok' => false, 'ready' => false, 'error' => 'Tailscale IP not found'];
|
||||
continue;
|
||||
}
|
||||
|
||||
// Single SSH call: get remote SCRIPTS_DIR and verify plugin is installed,
|
||||
// Configurations/ exists, and master.conf is already present.
|
||||
// Any missing piece means the remote isn't ready — skip rather than push blind.
|
||||
$sshBase = 'ssh -i ' . escapeshellarg($sshKey)
|
||||
. ' -o ConnectTimeout=10 -o StrictHostKeyChecking=no root@' . $ip;
|
||||
$probe = trim(shell_exec(
|
||||
$sshBase . ' "cfg=$(grep SCRIPTS_DIR /boot/config/plugins/varaverk/varaverk.cfg 2>/dev/null)'
|
||||
. ' && sd=$(echo \"$cfg\" | grep -oP \'(?<=SCRIPTS_DIR=\")[^\"]+\')'
|
||||
. ' && test -d \"${sd}/Configurations\"'
|
||||
. ' && test -f \"${sd}/Configurations/master.conf\"'
|
||||
. ' && echo \"$sd\""'
|
||||
) ?: '');
|
||||
|
||||
if ($probe === '') {
|
||||
$results[] = ['host' => $hostKey, 'ok' => false, 'ready' => false,
|
||||
'error' => 'plugin not installed, dir missing, or master.conf absent — skipped'];
|
||||
continue;
|
||||
}
|
||||
|
||||
$remoteConf = rtrim($probe, '/') . '/Configurations';
|
||||
$dest = escapeshellarg('root@' . $ip . ':' . $remoteConf . '/master.conf');
|
||||
$cmd = 'scp -i ' . escapeshellarg($sshKey)
|
||||
. ' -o ConnectTimeout=10 -o StrictHostKeyChecking=no'
|
||||
. ' ' . escapeshellarg($localPath) . ' ' . $dest . ' 2>&1';
|
||||
exec($cmd, $out, $rc);
|
||||
$results[] = [
|
||||
'host' => $hostKey,
|
||||
'ok' => $rc === 0,
|
||||
'error' => $rc !== 0 ? implode('; ', $out) : '',
|
||||
];
|
||||
}
|
||||
return $results;
|
||||
}
|
||||
|
||||
function vv_get_hostname(): string {
|
||||
return trim(shell_exec('hostname -s') ?: '');
|
||||
}
|
||||
|
||||
// Mirror of common.sh resolve_tailscale_ip(): tries `tailscale ip -4` first (Tailscale manages
|
||||
// the mapping so this survives IP changes), falls back to parsing `tailscale status` text.
|
||||
function vv_resolve_tailscale_ip(string $hostname): string {
|
||||
$h = strtolower($hostname);
|
||||
$ip = trim(shell_exec('tailscale ip -4 ' . escapeshellarg($h) . ' 2>/dev/null') ?: '');
|
||||
if ($ip) return $ip;
|
||||
$out = shell_exec('tailscale status 2>/dev/null') ?: '';
|
||||
foreach (explode("\n", $out) as $line) {
|
||||
$cols = preg_split('/\s+/', trim($line));
|
||||
if (isset($cols[1]) && stripos($cols[1], $h . '.') === 0) return $cols[0];
|
||||
}
|
||||
return '';
|
||||
}
|
||||
|
||||
function vv_detect_host(): string {
|
||||
// Reads master.conf for HOST1="name" (or HOST1_NAME="name") and matches running hostname.
|
||||
// Returns 'host1', 'host2', 'host3', ... or 'unknown'. Works for any number of hosts.
|
||||
$master = vv_read_conf_raw('master.conf');
|
||||
preg_match_all('/^\s*(HOST\d+)(?:_NAME)?\s*=\s*["\']?(\S+?)["\']?\s*$/m', $master, $m);
|
||||
$hostname = vv_get_hostname();
|
||||
foreach ($m[1] as $i => $key) {
|
||||
if (strcasecmp($hostname, trim($m[2][$i])) === 0) return strtolower($key);
|
||||
}
|
||||
return 'unknown';
|
||||
}
|
||||
|
||||
function vv_is_owner(): bool {
|
||||
return vv_detect_host() === 'host1';
|
||||
}
|
||||
|
||||
function vv_read_conf_raw(string $filename): string {
|
||||
$path = CONF_DIR . '/' . $filename;
|
||||
return file_exists($path) ? file_get_contents($path) : '';
|
||||
}
|
||||
|
||||
function vv_write_conf_raw(string $filename, string $content): bool {
|
||||
$path = CONF_DIR . '/' . $filename;
|
||||
$tmp = $path . '.vv.tmp';
|
||||
if (file_put_contents($tmp, $content) === false) return false;
|
||||
return rename($tmp, $path);
|
||||
}
|
||||
|
||||
function vv_get_conf_files(): array {
|
||||
// Returns conf files this host is allowed to view/edit
|
||||
$host = vv_detect_host();
|
||||
$files = [];
|
||||
if ($host === 'host1') {
|
||||
// Owner sees master.conf + their own host conf
|
||||
$files[] = 'master.conf';
|
||||
$files[] = 'host1.conf';
|
||||
} elseif (preg_match('/^host(\d+)$/', $host)) {
|
||||
// Any other numbered host sees only their own conf
|
||||
$files[] = $host . '.conf';
|
||||
} else {
|
||||
// Unknown host — show all for dev/debug
|
||||
foreach (glob(CONF_DIR . '/*.conf') as $f) {
|
||||
$files[] = basename($f);
|
||||
}
|
||||
}
|
||||
return $files;
|
||||
}
|
||||
|
||||
// Parse conf into key=>value map for $VAR substitution in docs
|
||||
function vv_conf_vars(): array {
|
||||
$vars = [];
|
||||
$files = ['master.conf'];
|
||||
$host = vv_detect_host();
|
||||
if (preg_match('/^host\d+$/', $host)) $files[] = $host . '.conf';
|
||||
|
||||
foreach ($files as $f) {
|
||||
$raw = vv_read_conf_raw($f);
|
||||
// Match: VAR_NAME="value" or VAR_NAME=value (no quotes)
|
||||
preg_match_all('/^\s*([A-Z0-9_]+)\s*=\s*["\']?([^"\'#\n]*?)["\']?\s*(?:#.*)?$/m', $raw, $m);
|
||||
foreach ($m[1] as $i => $key) {
|
||||
$vars[$key] = str_replace('\\$', '$', trim($m[2][$i]));
|
||||
}
|
||||
}
|
||||
return $vars;
|
||||
}
|
||||
|
||||
// Query the Unraid GraphQL API for a given host.
|
||||
// For the local host queries http://localhost/graphql; for remote hosts uses the Tailscale IP.
|
||||
// $apiKey may be passed explicitly (needed when querying a remote host from the local host,
|
||||
// since vv_conf_vars() only loads the current host's conf file).
|
||||
// Returns the decoded 'data' object on success, null on any failure.
|
||||
// Debug log written to /tmp/vv_api_debug.json on failure.
|
||||
function vv_unraid_api_query(string $hostId, string $gql, int $timeoutSec = 5, string $apiKey = ''): ?array {
|
||||
$vars = vv_conf_vars();
|
||||
$key = $apiKey ?: ($vars[strtoupper($hostId) . '_UNRAID_API_KEY'] ?? '');
|
||||
if (!$key) return null;
|
||||
|
||||
$myHostId = vv_detect_host();
|
||||
if (strtolower($hostId) === strtolower($myHostId)) {
|
||||
$url = 'http://localhost/graphql';
|
||||
} else {
|
||||
$hostname = $vars[strtoupper($hostId)] ?? '';
|
||||
if (!$hostname) return null;
|
||||
$ip = vv_resolve_tailscale_ip($hostname);
|
||||
if (!$ip) return null;
|
||||
$url = "http://{$ip}/graphql";
|
||||
}
|
||||
|
||||
$body = json_encode(['query' => $gql]);
|
||||
|
||||
// Use curl (preferred — doesn't require allow_url_fopen, better error handling).
|
||||
if (function_exists('curl_init')) {
|
||||
$ch = curl_init($url);
|
||||
curl_setopt_array($ch, [
|
||||
CURLOPT_POST => true,
|
||||
CURLOPT_HTTPHEADER => ['Content-Type: application/json', "x-api-key: {$key}"],
|
||||
CURLOPT_POSTFIELDS => $body,
|
||||
CURLOPT_RETURNTRANSFER => true,
|
||||
CURLOPT_TIMEOUT => $timeoutSec,
|
||||
CURLOPT_CONNECTTIMEOUT => 3,
|
||||
CURLOPT_FOLLOWLOCATION => false,
|
||||
]);
|
||||
$resp = curl_exec($ch);
|
||||
$httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
|
||||
$curlErr = curl_error($ch);
|
||||
curl_close($ch);
|
||||
} else {
|
||||
// Fallback to file_get_contents if curl is unavailable.
|
||||
$ctx = stream_context_create(['http' => [
|
||||
'method' => 'POST',
|
||||
'header' => "Content-Type: application/json\r\nx-api-key: {$key}",
|
||||
'content' => $body,
|
||||
'timeout' => $timeoutSec,
|
||||
'ignore_errors' => true,
|
||||
]]);
|
||||
$resp = @file_get_contents($url, false, $ctx);
|
||||
$httpCode = $resp !== false ? 200 : 0;
|
||||
$curlErr = '';
|
||||
}
|
||||
|
||||
if ($resp === false || $resp === '' || ($httpCode !== 0 && $httpCode !== 200)) {
|
||||
@file_put_contents(VV_CACHE_DIR . '/vv_api_debug.json', json_encode([
|
||||
'ts' => time(),
|
||||
'host' => $hostId,
|
||||
'url' => $url,
|
||||
'http_code' => $httpCode,
|
||||
'curl_err' => $curlErr,
|
||||
'response' => substr((string)$resp, 0, 800),
|
||||
], JSON_PRETTY_PRINT));
|
||||
return null;
|
||||
}
|
||||
|
||||
$decoded = json_decode((string)$resp, true);
|
||||
|
||||
// If the API returned GraphQL errors, log them for diagnosis.
|
||||
if (!empty($decoded['errors'])) {
|
||||
@file_put_contents(VV_CACHE_DIR . '/vv_api_debug.json', json_encode([
|
||||
'ts' => time(),
|
||||
'host' => $hostId,
|
||||
'url' => $url,
|
||||
'http_code' => $httpCode,
|
||||
'errors' => $decoded['errors'],
|
||||
'data' => $decoded['data'] ?? null,
|
||||
], JSON_PRETTY_PRINT));
|
||||
}
|
||||
|
||||
// data key present (even if null means query ran but returned nothing useful).
|
||||
return array_key_exists('data', $decoded ?? []) ? $decoded['data'] : null;
|
||||
}
|
||||
|
||||
// ── File-based API cache (/tmp/vv_cache — tmpfs, cleared on reboot) ───────────
|
||||
|
||||
// Read a cached payload. Returns null if missing or older than $maxAge seconds.
|
||||
function vv_cache_read(string $key, int $maxAge = 90): ?array {
|
||||
$f = VV_CACHE_DIR . '/' . $key . '.json';
|
||||
if (!file_exists($f) || (time() - filemtime($f)) > $maxAge) return null;
|
||||
$raw = file_get_contents($f);
|
||||
return $raw ? (json_decode($raw, true) ?: null) : null;
|
||||
}
|
||||
|
||||
// Write a payload atomically (tmp + rename) so readers never see a partial file.
|
||||
function vv_cache_write(string $key, array $data): void {
|
||||
if (!is_dir(VV_CACHE_DIR)) @mkdir(VV_CACHE_DIR, 0755, true);
|
||||
$f = VV_CACHE_DIR . '/' . $key . '.json';
|
||||
$tmp = $f . '.tmp';
|
||||
file_put_contents($tmp, json_encode($data));
|
||||
rename($tmp, $f);
|
||||
}
|
||||
|
||||
// ── Shared utility functions (used across include/ and api/ files) ────────────
|
||||
|
||||
// Format seconds into "2d 3h 15m".
|
||||
function vv_format_uptime(int $seconds): string {
|
||||
$d = intdiv($seconds, 86400);
|
||||
$h = intdiv($seconds % 86400, 3600);
|
||||
$m = intdiv($seconds % 3600, 60);
|
||||
return ($d ? "{$d}d " : '') . ($h ? "{$h}h " : '') . "{$m}m";
|
||||
}
|
||||
|
||||
// Parse a scalar value from raw conf text. Matches KEY="value" or KEY=value.
|
||||
// Identical logic was previously duplicated as vv_arr_scalar / vv_wd_scalar /
|
||||
// vv_fb_scalar / vv_media_conf_scalar — all reduce to this one regex.
|
||||
function vv_parse_conf_scalar(string $raw, string $key): string {
|
||||
return preg_match('/^\s*' . preg_quote($key, '/') . '\s*=\s*"?([^"\n]*)"?/m', $raw, $m)
|
||||
? trim($m[1]) : '';
|
||||
}
|
||||
|
||||
// Parse a key=value state file (e.g. fallback_state.db, partnership_state.db).
|
||||
// Returns ['key' => 'value', ...]. Lines without '=' are ignored.
|
||||
function vv_parse_kv_db(string $text): array {
|
||||
$out = [];
|
||||
foreach (explode("\n", $text) as $line) {
|
||||
$line = trim($line);
|
||||
if ($line === '' || $line[0] === '#') continue;
|
||||
[$k, $v] = array_pad(explode('=', $line, 2), 2, '');
|
||||
if ($k !== '') $out[trim($k)] = trim($v);
|
||||
}
|
||||
return $out;
|
||||
}
|
||||
|
||||
// All configured hosts from master.conf as ['host1' => 'hostname', ...].
|
||||
// Canonical version — previously duplicated as vv_arr_known_hosts / vv_fb_known_hosts.
|
||||
function vv_known_hosts(): array {
|
||||
$vars = vv_conf_vars();
|
||||
$hosts = [];
|
||||
foreach ($vars as $k => $v) {
|
||||
if (preg_match('/^HOST(\d+)$/', $k, $m) && $v !== '') {
|
||||
$hosts['host' . $m[1]] = $v;
|
||||
}
|
||||
}
|
||||
ksort($hosts);
|
||||
return $hosts ?: ['host1' => 'HOST1'];
|
||||
}
|
||||
|
||||
// Create (or overwrite) the Varaverk Unraid API key and write it into host conf.
|
||||
// Returns ['ok'=>true,'key_preview'=>'...'] or ['ok'=>false,'error'=>'...'].
|
||||
function vv_auto_create_api_key(string $hostId, string $confFile): array {
|
||||
$script = SCRIPTS_DIR . '/Plugin/unraid/System_Essentials/unraid_api_key_renew.sh';
|
||||
if (!file_exists($script)) {
|
||||
return ['ok' => false, 'error' => 'unraid_api_key_renew.sh not found'];
|
||||
}
|
||||
exec('bash ' . escapeshellarg($script) . ' 2>&1', $out, $rc);
|
||||
if ($rc !== 0) {
|
||||
$msg = implode(' ', array_filter(array_map('trim', $out)));
|
||||
return ['ok' => false, 'error' => $msg ?: 'Script failed'];
|
||||
}
|
||||
$varName = strtoupper($hostId) . '_UNRAID_API_KEY';
|
||||
$raw = vv_read_conf_raw($confFile);
|
||||
preg_match('/^\s*' . preg_quote($varName, '/') . '\s*=\s*"([^"]+)"/m', $raw, $m);
|
||||
$key = $m[1] ?? '';
|
||||
return ['ok' => true, 'key_preview' => $key ? substr($key, 0, 8) . '...' . substr($key, -4) : 'registered'];
|
||||
}
|
||||
|
||||
// Local LAN IP via routing table — static-cached per request.
|
||||
// Previously duplicated in include/docker_folders.php and inline in include/docker.php.
|
||||
function vv_local_ip(): string {
|
||||
static $ip = null;
|
||||
if ($ip !== null) return $ip;
|
||||
$ip = trim(shell_exec("ip route get 8.8.8.8 2>/dev/null | awk '/src/{for(i=1;i<=NF;i++)if(\$i==\"src\")print \$(i+1)}'") ?? '');
|
||||
return $ip;
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
<?php
|
||||
header('Content-Type: application/json');
|
||||
require_once dirname(__DIR__) . '/include/config.php';
|
||||
|
||||
$action = $_GET['action'] ?? $_POST['action'] ?? '';
|
||||
|
||||
// ── Boot device detection ─────────────────────────────────────────────────────
|
||||
function vv_storage_detect_transport(): string {
|
||||
$part = trim(shell_exec("findmnt -n -o SOURCE /boot 2>/dev/null") ?: '');
|
||||
if (!$part) return 'unknown';
|
||||
$disk = trim(shell_exec("lsblk -no pkname " . escapeshellarg($part) . " 2>/dev/null") ?: '');
|
||||
if (!$disk) return 'unknown';
|
||||
return strtolower(trim(shell_exec("lsblk -dno TRAN /dev/" . escapeshellarg($disk) . " 2>/dev/null") ?: 'unknown'));
|
||||
}
|
||||
|
||||
// ── Current mode status ───────────────────────────────────────────────────────
|
||||
if ($action === 'status') {
|
||||
$transport = vv_storage_detect_transport();
|
||||
$detected = ($transport === 'usb') ? 'flash' : 'internal';
|
||||
$currentDir = SCRIPTS_DIR;
|
||||
$internalDir = '/boot/config/plugins/varaverk';
|
||||
$flashDir = '/mnt/user/appdata/Varaverk';
|
||||
$currentMode = ($currentDir === $internalDir) ? 'internal'
|
||||
: ($currentDir === $flashDir ? 'flash' : 'custom');
|
||||
|
||||
$myHost = vv_detect_host();
|
||||
$vars = vv_conf_vars();
|
||||
$confKey = strtoupper($myHost) . '_STORAGE_MODE_INTERNAL';
|
||||
$confVal = $vars[$confKey] ?? null;
|
||||
|
||||
// Boot device name for display
|
||||
$bootPart = trim(shell_exec("findmnt -n -o SOURCE /boot 2>/dev/null") ?: '');
|
||||
$bootDisk = $bootPart ? trim(shell_exec("lsblk -no pkname " . escapeshellarg($bootPart) . " 2>/dev/null") ?: '') : '';
|
||||
|
||||
echo json_encode([
|
||||
'ok' => true,
|
||||
'current_mode' => $currentMode,
|
||||
'current_dir' => $currentDir,
|
||||
'internal_dir' => $internalDir,
|
||||
'flash_dir' => $flashDir,
|
||||
'transport' => $transport,
|
||||
'detected' => $detected,
|
||||
'boot_disk' => $bootDisk ? '/dev/' . $bootDisk : 'unknown',
|
||||
'conf_key' => $confKey,
|
||||
'conf_val' => $confVal,
|
||||
'array_started'=> is_dir('/mnt/user') && count(scandir('/mnt/user')) > 2,
|
||||
]);
|
||||
exit;
|
||||
}
|
||||
|
||||
// ── Run migration ─────────────────────────────────────────────────────────────
|
||||
if ($action === 'migrate' && $_SERVER['REQUEST_METHOD'] === 'POST') {
|
||||
$to = trim($_POST['to'] ?? '');
|
||||
if (!in_array($to, ['internal', 'flash'], true)) {
|
||||
echo json_encode(['ok' => false, 'error' => 'Invalid target: must be internal or flash']);
|
||||
exit;
|
||||
}
|
||||
|
||||
$script = dirname(__DIR__) . '/Tools/storage_migrate.sh';
|
||||
if (!file_exists($script)) {
|
||||
echo json_encode(['ok' => false, 'error' => 'storage_migrate.sh not found']);
|
||||
exit;
|
||||
}
|
||||
|
||||
set_time_limit(300);
|
||||
$output = [];
|
||||
$exit = 0;
|
||||
exec('bash ' . escapeshellarg($script) . ' --to=' . escapeshellarg($to) . ' 2>&1', $output, $exit);
|
||||
|
||||
echo json_encode([
|
||||
'ok' => $exit === 0,
|
||||
'exit' => $exit,
|
||||
'output' => implode("\n", $output),
|
||||
]);
|
||||
exit;
|
||||
}
|
||||
|
||||
// ── Auto-detect and write to conf ─────────────────────────────────────────────
|
||||
if ($action === 'detect' && $_SERVER['REQUEST_METHOD'] === 'POST') {
|
||||
$transport = vv_storage_detect_transport();
|
||||
$detected = ($transport === 'usb') ? 'false' : 'true';
|
||||
$myHost = vv_detect_host();
|
||||
$confKey = strtoupper($myHost) . '_STORAGE_MODE_INTERNAL';
|
||||
$confFile = $myHost . '.conf';
|
||||
|
||||
$results = vv_conf_write_changes([[
|
||||
'file' => $confFile,
|
||||
'key' => $confKey,
|
||||
'value' => $detected,
|
||||
'type' => 'scalar',
|
||||
]]);
|
||||
|
||||
$ok = !in_array(false, $results, true);
|
||||
echo json_encode(['ok' => $ok, 'detected' => $detected, 'transport' => $transport]);
|
||||
exit;
|
||||
}
|
||||
|
||||
// ── Unraid API key status ─────────────────────────────────────────────────────
|
||||
// Keys live in host*.conf (private, not master.conf):
|
||||
// host1.conf: HOST1_UNRAID_API_KEY (own) + HOST2_UNRAID_API_KEY (HOST1's access to HOST2)
|
||||
// host2.conf: HOST2_UNRAID_API_KEY (own) + HOST1_UNRAID_API_KEY (HOST2's access to HOST1)
|
||||
if ($action === 'api_status') {
|
||||
require_once dirname(__DIR__) . '/include/unraid_api.php';
|
||||
$localStatus = vv_api_get_status();
|
||||
$vars = vv_conf_vars();
|
||||
$myHost = vv_detect_host();
|
||||
$myId = strtoupper($myHost);
|
||||
|
||||
$hosts = [];
|
||||
foreach ($vars as $k => $v) {
|
||||
if (!preg_match('/^HOST(\d+)$/', $k, $m) || !$v) continue;
|
||||
$id = 'HOST' . $m[1];
|
||||
$keyVar = $id . '_UNRAID_API_KEY';
|
||||
$key = $vars[$keyVar] ?? '';
|
||||
$isLocal = ($id === $myId);
|
||||
// For local: key is Varaverk_HOST1 registered on own machine
|
||||
// For remote: key is Varaverk_HOST1 registered on HOST2's machine (stored in host1.conf)
|
||||
$hosts[] = [
|
||||
'host_id' => $id,
|
||||
'hostname' => $v,
|
||||
'is_local' => $isLocal,
|
||||
'key_var' => $keyVar,
|
||||
'key_name' => 'Varaverk_' . ($isLocal ? $myId : $myId), // Varaverk_HOST1 on that registry
|
||||
'key_present' => !empty($key),
|
||||
'key_preview' => $key ? substr($key, 0, 8) . '...' . substr($key, -4) : null,
|
||||
'api_ok' => $isLocal
|
||||
? (!$localStatus['key_missing'] && $localStatus['available'])
|
||||
: !empty($key),
|
||||
];
|
||||
}
|
||||
usort($hosts, fn($a, $b) => strcmp($a['host_id'], $b['host_id']));
|
||||
|
||||
echo json_encode([
|
||||
'ok' => true,
|
||||
'my_id' => $myId,
|
||||
'hosts' => $hosts,
|
||||
'fallbacks' => $localStatus['fallbacks'],
|
||||
]);
|
||||
exit;
|
||||
}
|
||||
|
||||
// ── Setup/renew API keys (local + all partners via SSH) ───────────────────────
|
||||
if ($action === 'setup_apikeys' && $_SERVER['REQUEST_METHOD'] === 'POST') {
|
||||
$script = SCRIPTS_DIR . '/Plugin/unraid/System_Essentials/unraid_api_key_renew.sh';
|
||||
if (!file_exists($script)) {
|
||||
echo json_encode(['ok' => false, 'error' => 'unraid_api_key_renew.sh not found']); exit;
|
||||
}
|
||||
$allHosts = ($_POST['all_hosts'] ?? '0') === '1';
|
||||
$flags = $allHosts ? ' --all-hosts' : '';
|
||||
set_time_limit(60);
|
||||
$output = []; $exit = 0;
|
||||
exec('bash ' . escapeshellarg($script) . $flags . ' 2>&1', $output, $exit);
|
||||
echo json_encode(['ok' => $exit === 0, 'exit' => $exit, 'output' => implode("\n", $output)]);
|
||||
exit;
|
||||
}
|
||||
|
||||
echo json_encode(['ok' => false, 'error' => 'Unknown action']);
|
||||
@@ -0,0 +1,152 @@
|
||||
<?php
|
||||
header('Content-Type: application/json');
|
||||
require_once dirname(__DIR__) . '/include/config.php';
|
||||
|
||||
$action = $_GET['action'] ?? $_POST['action'] ?? '';
|
||||
|
||||
// ── Boot device detection ─────────────────────────────────────────────────────
|
||||
function vv_storage_detect_transport(): string {
|
||||
$part = trim(shell_exec("findmnt -n -o SOURCE /boot 2>/dev/null") ?: '');
|
||||
if (!$part) return 'unknown';
|
||||
$disk = trim(shell_exec("lsblk -no pkname " . escapeshellarg($part) . " 2>/dev/null") ?: '');
|
||||
if (!$disk) return 'unknown';
|
||||
return strtolower(trim(shell_exec("lsblk -dno TRAN /dev/" . escapeshellarg($disk) . " 2>/dev/null") ?: 'unknown'));
|
||||
}
|
||||
|
||||
// ── Current mode status ───────────────────────────────────────────────────────
|
||||
if ($action === 'status') {
|
||||
$transport = vv_storage_detect_transport();
|
||||
$detected = ($transport === 'usb') ? 'flash' : 'internal';
|
||||
$currentDir = SCRIPTS_DIR;
|
||||
$internalDir = '/boot/config/plugins/varaverk';
|
||||
$flashDir = '/mnt/user/appdata/Varaverk';
|
||||
$currentMode = ($currentDir === $internalDir) ? 'internal'
|
||||
: ($currentDir === $flashDir ? 'flash' : 'custom');
|
||||
|
||||
$myHost = vv_detect_host();
|
||||
$vars = vv_conf_vars();
|
||||
$confKey = strtoupper($myHost) . '_STORAGE_MODE_INTERNAL';
|
||||
$confVal = $vars[$confKey] ?? null;
|
||||
|
||||
// Boot device name for display
|
||||
$bootPart = trim(shell_exec("findmnt -n -o SOURCE /boot 2>/dev/null") ?: '');
|
||||
$bootDisk = $bootPart ? trim(shell_exec("lsblk -no pkname " . escapeshellarg($bootPart) . " 2>/dev/null") ?: '') : '';
|
||||
|
||||
echo json_encode([
|
||||
'ok' => true,
|
||||
'current_mode' => $currentMode,
|
||||
'current_dir' => $currentDir,
|
||||
'internal_dir' => $internalDir,
|
||||
'flash_dir' => $flashDir,
|
||||
'transport' => $transport,
|
||||
'detected' => $detected,
|
||||
'boot_disk' => $bootDisk ? '/dev/' . $bootDisk : 'unknown',
|
||||
'conf_key' => $confKey,
|
||||
'conf_val' => $confVal,
|
||||
'array_started'=> is_dir('/mnt/user') && count(scandir('/mnt/user')) > 2,
|
||||
]);
|
||||
exit;
|
||||
}
|
||||
|
||||
// ── Run migration ─────────────────────────────────────────────────────────────
|
||||
if ($action === 'migrate' && $_SERVER['REQUEST_METHOD'] === 'POST') {
|
||||
$to = trim($_POST['to'] ?? '');
|
||||
if (!in_array($to, ['internal', 'flash'], true)) {
|
||||
echo json_encode(['ok' => false, 'error' => 'Invalid target: must be internal or flash']);
|
||||
exit;
|
||||
}
|
||||
|
||||
$script = dirname(__DIR__) . '/Tools/storage_migrate.sh';
|
||||
if (!file_exists($script)) {
|
||||
echo json_encode(['ok' => false, 'error' => 'storage_migrate.sh not found']);
|
||||
exit;
|
||||
}
|
||||
|
||||
set_time_limit(300);
|
||||
$output = [];
|
||||
$exit = 0;
|
||||
exec('bash ' . escapeshellarg($script) . ' --to=' . escapeshellarg($to) . ' 2>&1', $output, $exit);
|
||||
|
||||
echo json_encode([
|
||||
'ok' => $exit === 0,
|
||||
'exit' => $exit,
|
||||
'output' => implode("\n", $output),
|
||||
]);
|
||||
exit;
|
||||
}
|
||||
|
||||
// ── Auto-detect and write to conf ─────────────────────────────────────────────
|
||||
if ($action === 'detect' && $_SERVER['REQUEST_METHOD'] === 'POST') {
|
||||
$transport = vv_storage_detect_transport();
|
||||
$detected = ($transport === 'usb') ? 'false' : 'true';
|
||||
$myHost = vv_detect_host();
|
||||
$confKey = strtoupper($myHost) . '_STORAGE_MODE_INTERNAL';
|
||||
$confFile = $myHost . '.conf';
|
||||
|
||||
$results = vv_conf_write_changes([[
|
||||
'file' => $confFile,
|
||||
'key' => $confKey,
|
||||
'value' => $detected,
|
||||
'type' => 'scalar',
|
||||
]]);
|
||||
|
||||
$ok = !in_array(false, $results, true);
|
||||
echo json_encode(['ok' => $ok, 'detected' => $detected, 'transport' => $transport]);
|
||||
exit;
|
||||
}
|
||||
|
||||
// ── Unraid API key status ─────────────────────────────────────────────────────
|
||||
if ($action === 'api_status') {
|
||||
require_once dirname(__DIR__) . '/include/unraid_api.php';
|
||||
$localStatus = vv_api_get_status();
|
||||
$vars = vv_conf_vars();
|
||||
$myHost = vv_detect_host();
|
||||
$myId = strtoupper($myHost);
|
||||
$hn = trim((string)shell_exec("hostname -s 2>/dev/null | sed 's/^[Uu][Nn][Rr][Aa][Ii][Dd]-//'")) ?: 'Varaverk';
|
||||
$localKeyName = 'Varaverk ' . $hn;
|
||||
|
||||
$hosts = [];
|
||||
foreach ($vars as $k => $v) {
|
||||
if (!preg_match('/^HOST(\d+)$/', $k, $m) || !$v) continue;
|
||||
$id = 'HOST' . $m[1];
|
||||
$keyVar = $id . '_UNRAID_API_KEY';
|
||||
$key = $vars[$keyVar] ?? '';
|
||||
$isLocal = ($id === $myId);
|
||||
$hosts[] = [
|
||||
'host_id' => $id,
|
||||
'hostname' => $v,
|
||||
'is_local' => $isLocal,
|
||||
'key_var' => $keyVar,
|
||||
'key_present' => !empty($key),
|
||||
'key_preview' => $key ? substr($key, 0, 8) . '...' . substr($key, -4) : null,
|
||||
'api_ok' => $isLocal
|
||||
? (!$localStatus['key_missing'] && $localStatus['available'])
|
||||
: !empty($key),
|
||||
];
|
||||
}
|
||||
usort($hosts, fn($a, $b) => strcmp($a['host_id'], $b['host_id']));
|
||||
|
||||
echo json_encode([
|
||||
'ok' => true,
|
||||
'my_id' => $myId,
|
||||
'key_name' => $localKeyName,
|
||||
'hosts' => $hosts,
|
||||
'fallbacks' => $localStatus['fallbacks'],
|
||||
]);
|
||||
exit;
|
||||
}
|
||||
|
||||
// ── Setup/renew API keys (local + all partners via SSH) ───────────────────────
|
||||
if ($action === 'setup_apikeys' && $_SERVER['REQUEST_METHOD'] === 'POST') {
|
||||
$script = SCRIPTS_DIR . '/Plugin/unraid/System_Essentials/unraid_api_key_renew.sh';
|
||||
if (!file_exists($script)) {
|
||||
echo json_encode(['ok' => false, 'error' => 'unraid_api_key_renew.sh not found']); exit;
|
||||
}
|
||||
set_time_limit(60);
|
||||
$output = []; $exit = 0;
|
||||
exec('bash ' . escapeshellarg($script) . ' 2>&1', $output, $exit);
|
||||
echo json_encode(['ok' => $exit === 0, 'exit' => $exit, 'output' => implode("\n", $output)]);
|
||||
exit;
|
||||
}
|
||||
|
||||
echo json_encode(['ok' => false, 'error' => 'Unknown action']);
|
||||
@@ -0,0 +1,376 @@
|
||||
<?php
|
||||
// Arr (Sonarr / Radarr / Lidarr) data helpers
|
||||
|
||||
require_once __DIR__ . '/config.php';
|
||||
|
||||
// ── Conf helpers ──────────────────────────────────────────────────────────────
|
||||
|
||||
function vv_arr_scalar(string $raw, string $key): string {
|
||||
return vv_parse_conf_scalar($raw, $key);
|
||||
}
|
||||
|
||||
function vv_arr_known_hosts(): array {
|
||||
return vv_known_hosts();
|
||||
}
|
||||
|
||||
function vv_arr_node_names(): array {
|
||||
return array_map(fn($name) => $name, vv_arr_known_hosts());
|
||||
}
|
||||
|
||||
// ── Discovery ─────────────────────────────────────────────────────────────────
|
||||
|
||||
function vv_discover_arrs(): array {
|
||||
$nodes = [];
|
||||
$defs = [
|
||||
'sonarr' => ['SONARR_URL', 'SONARR_API_KEY', 'SONARR_TV_ROOT', 'v3'],
|
||||
'radarr' => ['RADARR_URL', 'RADARR_API_KEY', 'RADARR_MOVIES_ROOT', 'v3'],
|
||||
'lidarr' => ['LIDARR_URL', 'LIDARR_API_KEY', 'LIDARR_MUSIC_ROOT', 'v1'],
|
||||
];
|
||||
foreach (array_keys(vv_arr_known_hosts()) as $h) {
|
||||
$raw = vv_read_conf_raw($h . '.conf');
|
||||
if (!$raw) continue;
|
||||
$pfx = strtoupper($h) . '_';
|
||||
$get = fn($k) => vv_arr_scalar($raw, $pfx . $k);
|
||||
$arrs = [];
|
||||
foreach ($defs as $type => [$uk, $ak, $rk, $api]) {
|
||||
$url = $get($uk);
|
||||
$key = $get($ak);
|
||||
if ($url && $key && !str_contains($key, 'your-')) {
|
||||
$arrs[] = ['type' => $type, 'url' => $url, 'key' => $key,
|
||||
'root' => $get($rk), 'api' => $api];
|
||||
}
|
||||
}
|
||||
if ($arrs) $nodes[] = ['host' => $h, 'arrs' => $arrs];
|
||||
}
|
||||
return $nodes;
|
||||
}
|
||||
|
||||
// ── HTTP ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
function vv_arr_http(string $url, string $apiKey, string $path, int $timeout = 4): ?array {
|
||||
$ctx = stream_context_create(['http' => [
|
||||
'timeout' => $timeout,
|
||||
'header' => "X-Api-Key: $apiKey\r\nAccept: application/json\r\n",
|
||||
'ignore_errors' => true,
|
||||
]]);
|
||||
$raw = @file_get_contents(rtrim($url, '/') . $path, false, $ctx);
|
||||
return $raw ? (json_decode($raw, true) ?: null) : null;
|
||||
}
|
||||
|
||||
// ── Live arr data ─────────────────────────────────────────────────────────────
|
||||
|
||||
function vv_fetch_arr_live(array $arr): array {
|
||||
$url = $arr['url'];
|
||||
$key = $arr['key'];
|
||||
$base = '/api/' . $arr['api'];
|
||||
$type = $arr['type'];
|
||||
|
||||
$out = ['online' => false, 'version' => null, 'health' => [],
|
||||
'queue' => ['dl' => 0, 'warn' => 0, 'err' => 0], 'disk' => []];
|
||||
|
||||
$sys = vv_arr_http($url, $key, "$base/system/status");
|
||||
if (!$sys) return $out;
|
||||
$out['online'] = true;
|
||||
$out['version'] = $sys['version'] ?? null;
|
||||
|
||||
if ($type === 'sonarr') {
|
||||
$data = vv_arr_http($url, $key, "$base/series");
|
||||
if (is_array($data)) {
|
||||
$out['total'] = count($data);
|
||||
$out['monitored'] = count(array_filter($data, fn($x) => !empty($x['monitored'])));
|
||||
$out['episodes'] = array_sum(array_column($data, 'episodeFileCount'));
|
||||
}
|
||||
} elseif ($type === 'radarr') {
|
||||
$data = vv_arr_http($url, $key, "$base/movie");
|
||||
if (is_array($data)) {
|
||||
$out['total'] = count($data);
|
||||
$out['monitored'] = count(array_filter($data, fn($x) => !empty($x['monitored'])));
|
||||
$out['files'] = count(array_filter($data, fn($x) => !empty($x['hasFile'])));
|
||||
}
|
||||
} elseif ($type === 'lidarr') {
|
||||
$data = vv_arr_http($url, $key, "$base/artist");
|
||||
if (is_array($data)) {
|
||||
$out['total'] = count($data);
|
||||
$out['monitored'] = count(array_filter($data, fn($x) => !empty($x['monitored'])));
|
||||
$out['albums'] = array_sum(array_map(
|
||||
fn($a) => $a['statistics']['albumCount'] ?? $a['albumCount'] ?? 0, $data));
|
||||
}
|
||||
}
|
||||
|
||||
$q = vv_arr_http($url, $key, "$base/queue?page=1&pageSize=500");
|
||||
if (is_array($q)) {
|
||||
foreach (($q['records'] ?? $q) as $r) {
|
||||
if (!is_array($r)) continue;
|
||||
$s = $r['status'] ?? '';
|
||||
$tds = strtolower($r['trackedDownloadStatus'] ?? '');
|
||||
$tst = strtolower($r['trackedDownloadState'] ?? '');
|
||||
if ($s === 'downloading') $out['queue']['dl']++;
|
||||
if ($tds === 'warning' || $tst === 'downloadingstalled') $out['queue']['warn']++;
|
||||
if ($tds === 'error') $out['queue']['err']++;
|
||||
}
|
||||
}
|
||||
|
||||
$h = vv_arr_http($url, $key, "$base/health");
|
||||
if (is_array($h)) $out['health'] = $h;
|
||||
|
||||
$d = vv_arr_http($url, $key, "$base/diskspace");
|
||||
if (is_array($d)) $out['disk'] = $d;
|
||||
|
||||
return $out;
|
||||
}
|
||||
|
||||
// ── Log stats ─────────────────────────────────────────────────────────────────
|
||||
|
||||
function vv_arr_cleanup_stats(string $type): array {
|
||||
$slugs = ['sonarr' => 'Media/sonarr_cleanup',
|
||||
'radarr' => 'Media/radarr_cleanup',
|
||||
'lidarr' => 'Media/lidarr_cleanup'];
|
||||
$base = LOG_DIR . '/' . ($slugs[$type] ?? '');
|
||||
$out = ['last_run' => null, 'end' => null, 'status' => null,
|
||||
'tracked' => null, 'total' => null,
|
||||
'orphans' => 0, 'orphans_sz' => '0B', 'junk' => 0];
|
||||
|
||||
$jf = $base . '.json';
|
||||
if (file_exists($jf)) {
|
||||
$meta = json_decode(file_get_contents($jf), true) ?: [];
|
||||
$out['last_run'] = $meta['start'] ?? null;
|
||||
$out['end'] = $meta['end'] ?? null;
|
||||
$out['status'] = $meta['status'] ?? null;
|
||||
|
||||
$lf = $base . '.log';
|
||||
if (file_exists($lf)) {
|
||||
$log = file_get_contents($lf);
|
||||
$parts = preg_split('/━{3,}[^\n]*SUMMARY[^\n]*/u', $log);
|
||||
$blk = count($parts) > 1 ? end($parts) : $log;
|
||||
|
||||
if (preg_match('/Tracked:\s*([\d,]+)\s*files\s*\(([\d,]+)/u', $blk, $m)) {
|
||||
$out['tracked'] = (int)str_replace(',', '', $m[1]);
|
||||
$out['total'] = (int)str_replace(',', '', $m[2]);
|
||||
}
|
||||
if (preg_match('/Orphans:\s*([\d,]+)\s*files\s*\(([^)]+)\)/u', $blk, $m)) {
|
||||
$out['orphans'] = (int)str_replace(',', '', $m[1]);
|
||||
$out['orphans_sz'] = trim($m[2]);
|
||||
}
|
||||
if (preg_match('/Junk:\s*([\d,]+)\s*files/u', $blk, $m)) {
|
||||
$out['junk'] = (int)str_replace(',', '', $m[1]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback: daily aggregate db — date|arr|orphan_count|orphan_bytes|junk_count|junk_bytes|recent_count|tracked_count
|
||||
if ($out['last_run'] === null) {
|
||||
$dbFile = DATA_DIR . '/arr_cleanup_stats.db';
|
||||
if (file_exists($dbFile)) {
|
||||
$last = null;
|
||||
foreach (file($dbFile, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES) as $line) {
|
||||
$p = explode('|', $line);
|
||||
if (count($p) >= 8 && $p[1] === $type) $last = $p;
|
||||
}
|
||||
if ($last) {
|
||||
$out['last_run'] = strtotime($last[0] . ' 23:59:00') ?: null;
|
||||
$out['status'] = 'ok';
|
||||
$out['orphans'] = (int)$last[2];
|
||||
$out['junk'] = (int)$last[4];
|
||||
$out['tracked'] = (int)$last[7];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return $out;
|
||||
}
|
||||
|
||||
function vv_arr_discovery_stats(string $type): array {
|
||||
$slugs = ['sonarr' => 'Media/playback_aware_sonarr_discovery',
|
||||
'radarr' => 'Media/playback_aware_radarr_discovery',
|
||||
'lidarr' => 'Media/playback_aware_lidarr_discovery'];
|
||||
$base = LOG_DIR . '/' . ($slugs[$type] ?? '');
|
||||
$out = ['last_run' => null, 'status' => null, 'added' => null];
|
||||
|
||||
$jf = $base . '.json';
|
||||
if (file_exists($jf)) {
|
||||
$meta = json_decode(file_get_contents($jf), true) ?: [];
|
||||
$out['last_run'] = $meta['start'] ?? null;
|
||||
$out['status'] = $meta['status'] ?? null;
|
||||
|
||||
$lf = $base . '.log';
|
||||
if (file_exists($lf)) {
|
||||
$log = file_get_contents($lf);
|
||||
if (preg_match('/Added[:\s]+(\d+)/i', $log, $m)) $out['added'] = (int)$m[1];
|
||||
elseif (preg_match('/(\d+)\s+added/i', $log, $m)) $out['added'] = (int)$m[1];
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback: per-title history db — status|id|date[|title]
|
||||
if ($out['last_run'] === null) {
|
||||
$dbFile = DATA_DIR . '/' . $type . '_discovery_history.db';
|
||||
if (file_exists($dbFile)) {
|
||||
$lastDate = null; $added = 0;
|
||||
foreach (file($dbFile, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES) as $line) {
|
||||
$p = explode('|', $line);
|
||||
if (count($p) < 3) continue;
|
||||
$date = $p[2];
|
||||
if ($date !== $lastDate) { $lastDate = $date; $added = 0; }
|
||||
if ($p[0] === 'ACCEPT') $added++;
|
||||
}
|
||||
if ($lastDate) {
|
||||
$out['last_run'] = strtotime($lastDate . ' 23:59:00') ?: null;
|
||||
$out['status'] = 'ok';
|
||||
$out['added'] = $added;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return $out;
|
||||
}
|
||||
|
||||
function vv_arr_sync_stats(): array {
|
||||
$base = LOG_DIR . '/Media/arr_sync';
|
||||
$out = ['last_run' => null, 'status' => null, 'added' => null,
|
||||
'nodes' => null, 'blocklist_count' => null];
|
||||
|
||||
$jf = $base . '.json';
|
||||
if (file_exists($jf)) {
|
||||
$meta = json_decode(file_get_contents($jf), true) ?: [];
|
||||
$out['last_run'] = $meta['start'] ?? null;
|
||||
$out['status'] = $meta['status'] ?? null;
|
||||
}
|
||||
|
||||
$master = vv_read_conf_raw('master.conf');
|
||||
if (preg_match('/ARR_SYNC_BLOCKLIST\s*=\s*"?([^"\n#]+)"?/m', $master, $m)) {
|
||||
$blPath = trim($m[1]);
|
||||
if (file_exists($blPath)) {
|
||||
$out['blocklist_count'] = count(array_filter(
|
||||
file($blPath, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES)));
|
||||
}
|
||||
}
|
||||
|
||||
$lf = $base . '.log';
|
||||
if (file_exists($lf)) {
|
||||
$log = file_get_contents($lf);
|
||||
if (preg_match('/Total added[:\s]+(\d+)/i', $log, $m)) $out['added'] = (int)$m[1];
|
||||
if (preg_match('/Nodes?[:\s]+(\d+)/i', $log, $m)) $out['nodes'] = (int)$m[1];
|
||||
}
|
||||
return $out;
|
||||
}
|
||||
|
||||
function vv_arr_recovery_stats(): array {
|
||||
$base = LOG_DIR . '/Media/arrs_failed_stalled_recovery';
|
||||
$out = ['last_run' => null, 'status' => null, 'fixed' => 0, 'searched' => 0];
|
||||
|
||||
$jf = $base . '.json';
|
||||
if (file_exists($jf)) {
|
||||
$meta = json_decode(file_get_contents($jf), true) ?: [];
|
||||
$out['last_run'] = $meta['start'] ?? null;
|
||||
$out['status'] = $meta['status'] ?? null;
|
||||
|
||||
$lf = $base . '.log';
|
||||
if (file_exists($lf)) {
|
||||
$log = file_get_contents($lf);
|
||||
if (preg_match('/Removed[:\s]+(\d+)/i', $log, $m)) $out['fixed'] = (int)$m[1];
|
||||
if (preg_match('/Re-searched[:\s]+(\d+)/i',$log, $m)) $out['searched'] = (int)$m[1];
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback: daily aggregate db — date|time|count|bytes
|
||||
if ($out['last_run'] === null) {
|
||||
$dbFile = DATA_DIR . '/arr_recovery_stats.db';
|
||||
if (file_exists($dbFile)) {
|
||||
$lines = file($dbFile, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES);
|
||||
$last = $lines ? end($lines) : null;
|
||||
if ($last) {
|
||||
$p = explode('|', $last);
|
||||
if (count($p) >= 3) {
|
||||
$ts = strtotime(($p[0] ?? '') . ' ' . ($p[1] ?? '00:00')) ?: null;
|
||||
if ($ts) {
|
||||
$out['last_run'] = $ts;
|
||||
$out['status'] = 'ok';
|
||||
$out['fixed'] = (int)($p[2] ?? 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return $out;
|
||||
}
|
||||
|
||||
// ── Local node only — called via SSH by remote_arr_cache_writer.sh on remote hosts ───────────
|
||||
|
||||
function vv_arrs_local_node(): array {
|
||||
$host = vv_detect_host();
|
||||
$names = vv_arr_node_names();
|
||||
$arrs = [];
|
||||
|
||||
foreach (vv_discover_arrs() as $node) {
|
||||
if ($node['host'] !== $host) continue;
|
||||
foreach ($node['arrs'] as $arr) {
|
||||
$entry = ['type' => $arr['type'], 'root' => $arr['root']];
|
||||
$entry = array_merge($entry, vv_fetch_arr_live($arr));
|
||||
$entry['cleanup'] = vv_arr_cleanup_stats($arr['type']);
|
||||
$entry['discovery'] = vv_arr_discovery_stats($arr['type']);
|
||||
$arrs[] = $entry;
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
return [
|
||||
'host' => $host,
|
||||
'name' => $names[$host] ?? strtoupper($host),
|
||||
'local' => true,
|
||||
'arrs' => $arrs,
|
||||
];
|
||||
}
|
||||
|
||||
// ── Entry point ───────────────────────────────────────────────────────────────
|
||||
|
||||
function vv_arrs_all(): array {
|
||||
$currentHost = vv_detect_host();
|
||||
$names = vv_arr_node_names();
|
||||
|
||||
// Local node — full live data
|
||||
$result = [vv_arrs_local_node()];
|
||||
|
||||
// Remote nodes — read from file cache written by remote_arr_cache_writer.sh
|
||||
foreach (array_keys($names) as $h) {
|
||||
if ($h === $currentHost) continue;
|
||||
$cacheFile = VV_CACHE_DIR . '/arrs_remote_' . $h . '.json';
|
||||
if (file_exists($cacheFile)) {
|
||||
$node = json_decode(file_get_contents($cacheFile), true) ?: [];
|
||||
$node['cached'] = true;
|
||||
$node['cache_age'] = time() - (int)filemtime($cacheFile);
|
||||
$result[] = $node;
|
||||
} else {
|
||||
$result[] = [
|
||||
'host' => $h,
|
||||
'name' => $names[$h],
|
||||
'local' => false,
|
||||
'cached' => false,
|
||||
'cache_miss' => true,
|
||||
'arrs' => [],
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
$vars = vv_conf_vars();
|
||||
$myId = strtoupper($currentHost);
|
||||
|
||||
return [
|
||||
'nodes' => $result,
|
||||
'sync' => vv_arr_sync_stats(),
|
||||
'recovery' => vv_arr_recovery_stats(),
|
||||
'host' => $currentHost,
|
||||
'settings' => [
|
||||
'arr_sync_enabled' => ($vars['ARR_SYNC_ENABLED'] ?? 'true') !== 'false',
|
||||
'sonarr_recovery' => ($vars[$myId . '_SONARR_RECOVERY'] ?? 'true') !== 'false',
|
||||
'radarr_recovery' => ($vars[$myId . '_RADARR_RECOVERY'] ?? 'true') !== 'false',
|
||||
'lidarr_recovery' => ($vars[$myId . '_LIDARR_RECOVERY'] ?? 'true') !== 'false',
|
||||
'recovery_age_hours' => (int)($vars['ARR_IMPORT_RECOVERY_AGE'] ?? 6),
|
||||
'sonarr_port' => (int)($vars['ARR_SYNC_SONARR_PORT'] ?? 8989),
|
||||
'radarr_port' => (int)($vars['ARR_SYNC_RADARR_PORT'] ?? 7878),
|
||||
'lidarr_port' => (int)($vars['ARR_SYNC_LIDARR_PORT'] ?? 8686),
|
||||
'my_host' => $currentHost,
|
||||
'my_id' => $myId,
|
||||
],
|
||||
'ts' => time(),
|
||||
];
|
||||
}
|
||||
@@ -0,0 +1,377 @@
|
||||
<?php
|
||||
// Arr (Sonarr / Radarr / Lidarr) data helpers
|
||||
|
||||
require_once __DIR__ . '/config.php';
|
||||
|
||||
// ── Conf helpers ──────────────────────────────────────────────────────────────
|
||||
|
||||
function vv_arr_scalar(string $raw, string $key): string {
|
||||
return vv_parse_conf_scalar($raw, $key);
|
||||
}
|
||||
|
||||
function vv_arr_known_hosts(): array {
|
||||
return vv_known_hosts();
|
||||
}
|
||||
|
||||
function vv_arr_node_names(): array {
|
||||
return array_map(fn($name) => $name, vv_arr_known_hosts());
|
||||
}
|
||||
|
||||
// ── Discovery ─────────────────────────────────────────────────────────────────
|
||||
|
||||
function vv_discover_arrs(): array {
|
||||
$nodes = [];
|
||||
$defs = [
|
||||
'sonarr' => ['SONARR_URL', 'SONARR_API_KEY', 'SONARR_TV_ROOT', 'v3'],
|
||||
'radarr' => ['RADARR_URL', 'RADARR_API_KEY', 'RADARR_MOVIES_ROOT', 'v3'],
|
||||
'lidarr' => ['LIDARR_URL', 'LIDARR_API_KEY', 'LIDARR_MUSIC_ROOT', 'v1'],
|
||||
];
|
||||
foreach (array_keys(vv_arr_known_hosts()) as $h) {
|
||||
$raw = vv_read_conf_raw($h . '.conf');
|
||||
if (!$raw) continue;
|
||||
$pfx = strtoupper($h) . '_';
|
||||
$get = fn($k) => vv_arr_scalar($raw, $pfx . $k);
|
||||
$arrs = [];
|
||||
foreach ($defs as $type => [$uk, $ak, $rk, $api]) {
|
||||
$url = $get($uk);
|
||||
$key = $get($ak);
|
||||
if ($url && $key && !str_contains($key, 'your-')) {
|
||||
$arrs[] = ['type' => $type, 'url' => $url, 'key' => $key,
|
||||
'root' => $get($rk), 'api' => $api];
|
||||
}
|
||||
}
|
||||
if ($arrs) $nodes[] = ['host' => $h, 'arrs' => $arrs];
|
||||
}
|
||||
return $nodes;
|
||||
}
|
||||
|
||||
// ── HTTP ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
function vv_arr_http(string $url, string $apiKey, string $path, int $timeout = 4): ?array {
|
||||
$ctx = stream_context_create(['http' => [
|
||||
'timeout' => $timeout,
|
||||
'header' => "X-Api-Key: $apiKey\r\nAccept: application/json\r\n",
|
||||
'ignore_errors' => true,
|
||||
]]);
|
||||
$raw = @file_get_contents(rtrim($url, '/') . $path, false, $ctx);
|
||||
return $raw ? (json_decode($raw, true) ?: null) : null;
|
||||
}
|
||||
|
||||
// ── Live arr data ─────────────────────────────────────────────────────────────
|
||||
|
||||
function vv_fetch_arr_live(array $arr): array {
|
||||
$url = $arr['url'];
|
||||
$key = $arr['key'];
|
||||
$base = '/api/' . $arr['api'];
|
||||
$type = $arr['type'];
|
||||
|
||||
$out = ['online' => false, 'version' => null, 'health' => [],
|
||||
'queue' => ['dl' => 0, 'warn' => 0, 'err' => 0], 'disk' => []];
|
||||
|
||||
$sys = vv_arr_http($url, $key, "$base/system/status");
|
||||
if (!$sys) return $out;
|
||||
$out['online'] = true;
|
||||
$out['version'] = $sys['version'] ?? null;
|
||||
|
||||
if ($type === 'sonarr') {
|
||||
$data = vv_arr_http($url, $key, "$base/series");
|
||||
if (is_array($data)) {
|
||||
$out['total'] = count($data);
|
||||
$out['monitored'] = count(array_filter($data, fn($x) => !empty($x['monitored'])));
|
||||
$out['episodes'] = array_sum(array_map(
|
||||
fn($x) => $x['statistics']['episodeFileCount'] ?? $x['episodeFileCount'] ?? 0, $data));
|
||||
}
|
||||
} elseif ($type === 'radarr') {
|
||||
$data = vv_arr_http($url, $key, "$base/movie");
|
||||
if (is_array($data)) {
|
||||
$out['total'] = count($data);
|
||||
$out['monitored'] = count(array_filter($data, fn($x) => !empty($x['monitored'])));
|
||||
$out['files'] = count(array_filter($data, fn($x) => !empty($x['hasFile'])));
|
||||
}
|
||||
} elseif ($type === 'lidarr') {
|
||||
$data = vv_arr_http($url, $key, "$base/artist");
|
||||
if (is_array($data)) {
|
||||
$out['total'] = count($data);
|
||||
$out['monitored'] = count(array_filter($data, fn($x) => !empty($x['monitored'])));
|
||||
$out['albums'] = array_sum(array_map(
|
||||
fn($a) => $a['statistics']['albumCount'] ?? $a['albumCount'] ?? 0, $data));
|
||||
}
|
||||
}
|
||||
|
||||
$q = vv_arr_http($url, $key, "$base/queue?page=1&pageSize=500");
|
||||
if (is_array($q)) {
|
||||
foreach (($q['records'] ?? $q) as $r) {
|
||||
if (!is_array($r)) continue;
|
||||
$s = $r['status'] ?? '';
|
||||
$tds = strtolower($r['trackedDownloadStatus'] ?? '');
|
||||
$tst = strtolower($r['trackedDownloadState'] ?? '');
|
||||
if ($s === 'downloading') $out['queue']['dl']++;
|
||||
if ($tds === 'warning' || $tst === 'downloadingstalled') $out['queue']['warn']++;
|
||||
if ($tds === 'error') $out['queue']['err']++;
|
||||
}
|
||||
}
|
||||
|
||||
$h = vv_arr_http($url, $key, "$base/health");
|
||||
if (is_array($h)) $out['health'] = $h;
|
||||
|
||||
$d = vv_arr_http($url, $key, "$base/diskspace");
|
||||
if (is_array($d)) $out['disk'] = $d;
|
||||
|
||||
return $out;
|
||||
}
|
||||
|
||||
// ── Log stats ─────────────────────────────────────────────────────────────────
|
||||
|
||||
function vv_arr_cleanup_stats(string $type): array {
|
||||
$slugs = ['sonarr' => 'Media/sonarr_cleanup',
|
||||
'radarr' => 'Media/radarr_cleanup',
|
||||
'lidarr' => 'Media/lidarr_cleanup'];
|
||||
$base = LOG_DIR . '/' . ($slugs[$type] ?? '');
|
||||
$out = ['last_run' => null, 'end' => null, 'status' => null,
|
||||
'tracked' => null, 'total' => null,
|
||||
'orphans' => 0, 'orphans_sz' => '0B', 'junk' => 0];
|
||||
|
||||
$jf = $base . '.json';
|
||||
if (file_exists($jf)) {
|
||||
$meta = json_decode(file_get_contents($jf), true) ?: [];
|
||||
$out['last_run'] = $meta['start'] ?? null;
|
||||
$out['end'] = $meta['end'] ?? null;
|
||||
$out['status'] = $meta['status'] ?? null;
|
||||
|
||||
$lf = $base . '.log';
|
||||
if (file_exists($lf)) {
|
||||
$log = file_get_contents($lf);
|
||||
$parts = preg_split('/━{3,}[^\n]*SUMMARY[^\n]*/u', $log);
|
||||
$blk = count($parts) > 1 ? end($parts) : $log;
|
||||
|
||||
if (preg_match('/Tracked:\s*([\d,]+)\s*files\s*\(([\d,]+)/u', $blk, $m)) {
|
||||
$out['tracked'] = (int)str_replace(',', '', $m[1]);
|
||||
$out['total'] = (int)str_replace(',', '', $m[2]);
|
||||
}
|
||||
if (preg_match('/Orphans:\s*([\d,]+)\s*files\s*\(([^)]+)\)/u', $blk, $m)) {
|
||||
$out['orphans'] = (int)str_replace(',', '', $m[1]);
|
||||
$out['orphans_sz'] = trim($m[2]);
|
||||
}
|
||||
if (preg_match('/Junk:\s*([\d,]+)\s*files/u', $blk, $m)) {
|
||||
$out['junk'] = (int)str_replace(',', '', $m[1]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback: daily aggregate db — date|arr|orphan_count|orphan_bytes|junk_count|junk_bytes|recent_count|tracked_count
|
||||
if ($out['last_run'] === null) {
|
||||
$dbFile = DATA_DIR . '/arr_cleanup_stats.db';
|
||||
if (file_exists($dbFile)) {
|
||||
$last = null;
|
||||
foreach (file($dbFile, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES) as $line) {
|
||||
$p = explode('|', $line);
|
||||
if (count($p) >= 8 && $p[1] === $type) $last = $p;
|
||||
}
|
||||
if ($last) {
|
||||
$out['last_run'] = strtotime($last[0] . ' 23:59:00') ?: null;
|
||||
$out['status'] = 'ok';
|
||||
$out['orphans'] = (int)$last[2];
|
||||
$out['junk'] = (int)$last[4];
|
||||
$out['tracked'] = (int)$last[7];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return $out;
|
||||
}
|
||||
|
||||
function vv_arr_discovery_stats(string $type): array {
|
||||
$slugs = ['sonarr' => 'Media/playback_aware_sonarr_discovery',
|
||||
'radarr' => 'Media/playback_aware_radarr_discovery',
|
||||
'lidarr' => 'Media/playback_aware_lidarr_discovery'];
|
||||
$base = LOG_DIR . '/' . ($slugs[$type] ?? '');
|
||||
$out = ['last_run' => null, 'status' => null, 'added' => null];
|
||||
|
||||
$jf = $base . '.json';
|
||||
if (file_exists($jf)) {
|
||||
$meta = json_decode(file_get_contents($jf), true) ?: [];
|
||||
$out['last_run'] = $meta['start'] ?? null;
|
||||
$out['status'] = $meta['status'] ?? null;
|
||||
|
||||
$lf = $base . '.log';
|
||||
if (file_exists($lf)) {
|
||||
$log = file_get_contents($lf);
|
||||
if (preg_match('/Added[:\s]+(\d+)/i', $log, $m)) $out['added'] = (int)$m[1];
|
||||
elseif (preg_match('/(\d+)\s+added/i', $log, $m)) $out['added'] = (int)$m[1];
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback: per-title history db — status|id|date[|title]
|
||||
if ($out['last_run'] === null) {
|
||||
$dbFile = DATA_DIR . '/' . $type . '_discovery_history.db';
|
||||
if (file_exists($dbFile)) {
|
||||
$lastDate = null; $added = 0;
|
||||
foreach (file($dbFile, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES) as $line) {
|
||||
$p = explode('|', $line);
|
||||
if (count($p) < 3) continue;
|
||||
$date = $p[2];
|
||||
if ($date !== $lastDate) { $lastDate = $date; $added = 0; }
|
||||
if ($p[0] === 'ACCEPT') $added++;
|
||||
}
|
||||
if ($lastDate) {
|
||||
$out['last_run'] = strtotime($lastDate . ' 23:59:00') ?: null;
|
||||
$out['status'] = 'ok';
|
||||
$out['added'] = $added;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return $out;
|
||||
}
|
||||
|
||||
function vv_arr_sync_stats(): array {
|
||||
$base = LOG_DIR . '/Media/arr_sync';
|
||||
$out = ['last_run' => null, 'status' => null, 'added' => null,
|
||||
'nodes' => null, 'blocklist_count' => null];
|
||||
|
||||
$jf = $base . '.json';
|
||||
if (file_exists($jf)) {
|
||||
$meta = json_decode(file_get_contents($jf), true) ?: [];
|
||||
$out['last_run'] = $meta['start'] ?? null;
|
||||
$out['status'] = $meta['status'] ?? null;
|
||||
}
|
||||
|
||||
$master = vv_read_conf_raw('master.conf');
|
||||
if (preg_match('/ARR_SYNC_BLOCKLIST\s*=\s*"?([^"\n#]+)"?/m', $master, $m)) {
|
||||
$blPath = trim($m[1]);
|
||||
if (file_exists($blPath)) {
|
||||
$out['blocklist_count'] = count(array_filter(
|
||||
file($blPath, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES)));
|
||||
}
|
||||
}
|
||||
|
||||
$lf = $base . '.log';
|
||||
if (file_exists($lf)) {
|
||||
$log = file_get_contents($lf);
|
||||
if (preg_match('/Total added[:\s]+(\d+)/i', $log, $m)) $out['added'] = (int)$m[1];
|
||||
if (preg_match('/Nodes?[:\s]+(\d+)/i', $log, $m)) $out['nodes'] = (int)$m[1];
|
||||
}
|
||||
return $out;
|
||||
}
|
||||
|
||||
function vv_arr_recovery_stats(): array {
|
||||
$base = LOG_DIR . '/Media/arrs_failed_stalled_recovery';
|
||||
$out = ['last_run' => null, 'status' => null, 'fixed' => 0, 'searched' => 0];
|
||||
|
||||
$jf = $base . '.json';
|
||||
if (file_exists($jf)) {
|
||||
$meta = json_decode(file_get_contents($jf), true) ?: [];
|
||||
$out['last_run'] = $meta['start'] ?? null;
|
||||
$out['status'] = $meta['status'] ?? null;
|
||||
|
||||
$lf = $base . '.log';
|
||||
if (file_exists($lf)) {
|
||||
$log = file_get_contents($lf);
|
||||
if (preg_match('/Removed[:\s]+(\d+)/i', $log, $m)) $out['fixed'] = (int)$m[1];
|
||||
if (preg_match('/Re-searched[:\s]+(\d+)/i',$log, $m)) $out['searched'] = (int)$m[1];
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback: daily aggregate db — date|time|count|bytes
|
||||
if ($out['last_run'] === null) {
|
||||
$dbFile = DATA_DIR . '/arr_recovery_stats.db';
|
||||
if (file_exists($dbFile)) {
|
||||
$lines = file($dbFile, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES);
|
||||
$last = $lines ? end($lines) : null;
|
||||
if ($last) {
|
||||
$p = explode('|', $last);
|
||||
if (count($p) >= 3) {
|
||||
$ts = strtotime(($p[0] ?? '') . ' ' . ($p[1] ?? '00:00')) ?: null;
|
||||
if ($ts) {
|
||||
$out['last_run'] = $ts;
|
||||
$out['status'] = 'ok';
|
||||
$out['fixed'] = (int)($p[2] ?? 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return $out;
|
||||
}
|
||||
|
||||
// ── Local node only — called via SSH by remote_arr_cache_writer.sh on remote hosts ───────────
|
||||
|
||||
function vv_arrs_local_node(): array {
|
||||
$host = vv_detect_host();
|
||||
$names = vv_arr_node_names();
|
||||
$arrs = [];
|
||||
|
||||
foreach (vv_discover_arrs() as $node) {
|
||||
if ($node['host'] !== $host) continue;
|
||||
foreach ($node['arrs'] as $arr) {
|
||||
$entry = ['type' => $arr['type'], 'root' => $arr['root']];
|
||||
$entry = array_merge($entry, vv_fetch_arr_live($arr));
|
||||
$entry['cleanup'] = vv_arr_cleanup_stats($arr['type']);
|
||||
$entry['discovery'] = vv_arr_discovery_stats($arr['type']);
|
||||
$arrs[] = $entry;
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
return [
|
||||
'host' => $host,
|
||||
'name' => $names[$host] ?? strtoupper($host),
|
||||
'local' => true,
|
||||
'arrs' => $arrs,
|
||||
];
|
||||
}
|
||||
|
||||
// ── Entry point ───────────────────────────────────────────────────────────────
|
||||
|
||||
function vv_arrs_all(): array {
|
||||
$currentHost = vv_detect_host();
|
||||
$names = vv_arr_node_names();
|
||||
|
||||
// Local node — full live data
|
||||
$result = [vv_arrs_local_node()];
|
||||
|
||||
// Remote nodes — read from file cache written by remote_arr_cache_writer.sh
|
||||
foreach (array_keys($names) as $h) {
|
||||
if ($h === $currentHost) continue;
|
||||
$cacheFile = VV_CACHE_DIR . '/arrs_remote_' . $h . '.json';
|
||||
if (file_exists($cacheFile)) {
|
||||
$node = json_decode(file_get_contents($cacheFile), true) ?: [];
|
||||
$node['cached'] = true;
|
||||
$node['cache_age'] = time() - (int)filemtime($cacheFile);
|
||||
$result[] = $node;
|
||||
} else {
|
||||
$result[] = [
|
||||
'host' => $h,
|
||||
'name' => $names[$h],
|
||||
'local' => false,
|
||||
'cached' => false,
|
||||
'cache_miss' => true,
|
||||
'arrs' => [],
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
$vars = vv_conf_vars();
|
||||
$myId = strtoupper($currentHost);
|
||||
|
||||
return [
|
||||
'nodes' => $result,
|
||||
'sync' => vv_arr_sync_stats(),
|
||||
'recovery' => vv_arr_recovery_stats(),
|
||||
'host' => $currentHost,
|
||||
'settings' => [
|
||||
'arr_sync_enabled' => ($vars['ARR_SYNC_ENABLED'] ?? 'true') !== 'false',
|
||||
'sonarr_recovery' => ($vars[$myId . '_SONARR_RECOVERY'] ?? 'true') !== 'false',
|
||||
'radarr_recovery' => ($vars[$myId . '_RADARR_RECOVERY'] ?? 'true') !== 'false',
|
||||
'lidarr_recovery' => ($vars[$myId . '_LIDARR_RECOVERY'] ?? 'true') !== 'false',
|
||||
'recovery_age_hours' => (int)($vars['ARR_IMPORT_RECOVERY_AGE'] ?? 6),
|
||||
'sonarr_port' => (int)($vars['ARR_SYNC_SONARR_PORT'] ?? 8989),
|
||||
'radarr_port' => (int)($vars['ARR_SYNC_RADARR_PORT'] ?? 7878),
|
||||
'lidarr_port' => (int)($vars['ARR_SYNC_LIDARR_PORT'] ?? 8686),
|
||||
'my_host' => $currentHost,
|
||||
'my_id' => $myId,
|
||||
],
|
||||
'ts' => time(),
|
||||
];
|
||||
}
|
||||
+913
@@ -0,0 +1,913 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ================================= ARR Sync ===================================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Full-mesh arr library sync across all nodes — Lidarr, Sonarr, and Radarr.
|
||||
# Every node syncs with every other, union model, no hierarchy. Run before
|
||||
# rsync in the weekly sync window: once arrs agree on what to track, rsync
|
||||
# spreads the actual files.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Full mesh: every node syncs with every other — no primary, no hierarchy.
|
||||
# Union model: if any node tracks an item, all nodes get it (unless blocklisted).
|
||||
# Convergence: any node can add content; after one full cycle all nodes agree.
|
||||
# Upgrade-aware: a file upgrade on one node → arr tracks new path → rsync spreads
|
||||
# it → arr_cleanup removes old path on all nodes (arr no longer tracks it).
|
||||
#
|
||||
# Node discovery: reads HOST* vars from master.conf. Add HOST3= and it joins the
|
||||
# sync automatically — no script changes needed for a new node.
|
||||
#
|
||||
# Graceful skip: arr not configured locally → skip cleanly. Arr not reachable on
|
||||
# a remote → skip that node for that arr type, continue with others.
|
||||
#
|
||||
# What gets synced — library items keyed on stable external IDs:
|
||||
# Lidarr — MusicBrainz artist ID (foreignArtistId)
|
||||
# Sonarr — TVDB series ID (tvdbId)
|
||||
# Radarr — TMDB movie ID (tmdbId)
|
||||
# When adding to a remote, that node's own quality profile, metadata profile,
|
||||
# and root folder path are used — settings are never copied from source.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# DESIGN PRINCIPLES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Remote API Access — Cache-First, SSH Fallback
|
||||
# If conf_sync.sh has populated /tmp/.vv/config/cached/.confs/ and
|
||||
# load_config.sh has sourced it, HOST*_<ARR>_API_KEY vars are available
|
||||
# in the environment. Remote functions use them to call the arr API
|
||||
# directly over Tailscale (no SSH, no remote shell). If the cached key
|
||||
# is absent (first boot, cache not yet populated) the functions fall back
|
||||
# to SSHing in and reading the key from config.xml on the remote node.
|
||||
#
|
||||
# Blocklist TSV
|
||||
# ARR_SYNC_BLOCKLIST in DATA_DIR tombstones IDs that must never be re-added
|
||||
# anywhere. Read from ALL nodes via SSH at run start — immediate effect with
|
||||
# no rsync delay.
|
||||
#
|
||||
# --blocklist-add does three things atomically:
|
||||
# 1. Writes the TSV tombstone entry (prevents future re-adds by arr_sync)
|
||||
# 2. Deletes the item from the local arr API (deleteFiles=false)
|
||||
# 3. SSHes each remote node and deletes from their arr API (deleteFiles=false)
|
||||
# Files become orphans on all nodes — arr_cleanup removes them on next run.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# acquire_lock — prevents two sync instances running simultaneously
|
||||
# ARR_SYNC_ENABLED — global gate, exits cleanly when false
|
||||
# SSH connect timeout — ARR_SYNC_CONNECT_TIMEOUT — does not hang on unreachable node
|
||||
# API call timeout — ARR_SYNC_API_TIMEOUT — does not hang on slow arr
|
||||
# Graceful skip — unreachable node/arr → skip and continue, never abort
|
||||
# Blocklist gate — item in blocklist → never added to any node
|
||||
# Silent by default — only additions produce output, clean runs stay silent
|
||||
#
|
||||
# ==============================================================================================
|
||||
# STATE FILES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# ARR_SYNC_BLOCKLIST — TSV file in DATA_DIR (default: DATA_DIR/arr_sync_blocklist.tsv)
|
||||
# Columns: arr_type, id, reason, date_added
|
||||
# Read from all nodes via SSH at the start of each run.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# host*.conf
|
||||
#
|
||||
# HOST*_LIDARR_URL / HOST*_LIDARR_API_KEY — local Lidarr (aliased by detect_hosts)
|
||||
# HOST*_SONARR_URL / HOST*_SONARR_API_KEY — local Sonarr
|
||||
# HOST*_RADARR_URL / HOST*_RADARR_API_KEY — local Radarr
|
||||
#
|
||||
# master.conf
|
||||
#
|
||||
# ARR_SYNC_ENABLED — global on/off toggle (default: true)
|
||||
# ARR_SYNC_BLOCKLIST — path to TSV blocklist file
|
||||
# ARR_SYNC_CONNECT_TIMEOUT — SSH connect timeout in seconds (default: 10)
|
||||
# ARR_SYNC_API_TIMEOUT — curl API call timeout in seconds (default: 60)
|
||||
# DOCKER_APPDATA_BASE — base path for arr appdata dirs (default: /mnt/user/appdata)
|
||||
# ARR_SYNC_LIDARR_PORT — Lidarr port on all nodes (default: 8686)
|
||||
# ARR_SYNC_SONARR_PORT — Sonarr port on all nodes (default: 8989)
|
||||
# ARR_SYNC_RADARR_PORT — Radarr port on all nodes (default: 7878)
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# arr_sync.sh — sync all arr types, all nodes
|
||||
# arr_sync.sh --dry-run — preview only, no changes
|
||||
# arr_sync.sh --log — verbose output
|
||||
# arr_sync.sh --status — show config and exit
|
||||
#
|
||||
# Blocklist management:
|
||||
# arr_sync.sh --blocklist-add lidarr <mbid> "reason" — remove from all arrs + tombstone
|
||||
# arr_sync.sh --blocklist-add sonarr <tvdbId> "reason" — remove from all arrs + tombstone
|
||||
# arr_sync.sh --blocklist-add radarr <tmdbId> "reason" — remove from all arrs + tombstone
|
||||
# arr_sync.sh --blocklist-remove lidarr <id> — un-tombstone (does NOT re-add)
|
||||
# arr_sync.sh --blocklist-list — show all blocklisted IDs
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
# ── Blocklist flag pre-processing ─────────────────────────────────────────────────────────────
|
||||
# Handle before parse_args — these are action flags, not standard options
|
||||
BLOCKLIST_ACTION=""
|
||||
BLOCKLIST_ARR=""
|
||||
BLOCKLIST_ID=""
|
||||
BLOCKLIST_REASON=""
|
||||
FILTERED_ARGS=()
|
||||
_skip_next=false
|
||||
for _arg in "$@"; do
|
||||
if [[ "$_skip_next" == true ]]; then _skip_next=false; continue; fi
|
||||
case "$_arg" in
|
||||
--blocklist-add) BLOCKLIST_ACTION="add" ;;
|
||||
--blocklist-remove) BLOCKLIST_ACTION="remove" ;;
|
||||
--blocklist-list) BLOCKLIST_ACTION="list" ;;
|
||||
*) FILTERED_ARGS+=("$_arg") ;;
|
||||
esac
|
||||
done
|
||||
unset _arg _skip_next
|
||||
|
||||
parse_args "${FILTERED_ARGS[@]}"
|
||||
|
||||
# Consume blocklist positional args from remaining FILTERED_ARGS
|
||||
# --blocklist-add lidarr <id> "reason"
|
||||
# --blocklist-remove lidarr <id>
|
||||
if [[ -n "$BLOCKLIST_ACTION" ]] && [[ "$BLOCKLIST_ACTION" != "list" ]]; then
|
||||
for _a in "${FILTERED_ARGS[@]}"; do
|
||||
[[ "$_a" == --* ]] && continue
|
||||
if [[ -z "$BLOCKLIST_ARR" ]]; then BLOCKLIST_ARR="$_a"
|
||||
elif [[ -z "$BLOCKLIST_ID" ]]; then BLOCKLIST_ID="$_a"
|
||||
elif [[ -z "$BLOCKLIST_REASON" ]]; then BLOCKLIST_REASON="$_a"
|
||||
fi
|
||||
done
|
||||
unset _a
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
acquire_lock
|
||||
|
||||
for _tool in curl jq; do
|
||||
if ! command -v "$_tool" >/dev/null 2>&1; then
|
||||
error "$_tool not found — required for arr API calls"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
unset _tool
|
||||
|
||||
detect_hosts
|
||||
|
||||
# ── Runtime config with defaults ──────────────────────────────────────────────────────────────
|
||||
ARR_SYNC_ENABLED="${ARR_SYNC_ENABLED:-true}"
|
||||
ARR_SYNC_BLOCKLIST="${ARR_SYNC_BLOCKLIST:-${DATA_DIR}/arr_sync_blocklist.tsv}"
|
||||
ARR_SYNC_CONNECT_TIMEOUT="${ARR_SYNC_CONNECT_TIMEOUT:-10}"
|
||||
ARR_SYNC_API_TIMEOUT="${ARR_SYNC_API_TIMEOUT:-60}"
|
||||
DOCKER_APPDATA_BASE="${DOCKER_APPDATA_BASE:-/mnt/user/appdata}"
|
||||
ARR_SYNC_LIDARR_PORT="${ARR_SYNC_LIDARR_PORT:-8686}"
|
||||
ARR_SYNC_SONARR_PORT="${ARR_SYNC_SONARR_PORT:-8989}"
|
||||
ARR_SYNC_RADARR_PORT="${ARR_SYNC_RADARR_PORT:-7878}"
|
||||
|
||||
if [[ "$ARR_SYNC_ENABLED" != "true" ]]; then
|
||||
log "ARR_SYNC_ENABLED=false — exiting"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
log "$ICON_GEAR Config: api-timeout=${ARR_SYNC_API_TIMEOUT}s connect-timeout=${ARR_SYNC_CONNECT_TIMEOUT}s blocklist=${ARR_SYNC_BLOCKLIST}"
|
||||
log "$ICON_GEAR Ports: lidarr=${ARR_SYNC_LIDARR_PORT} sonarr=${ARR_SYNC_SONARR_PORT} radarr=${ARR_SYNC_RADARR_PORT}"
|
||||
|
||||
# ── Arr type definitions ───────────────────────────────────────────────────────────────────────
|
||||
# Each arr type maps to its port, API version, endpoint, stable ID field, and display name field
|
||||
declare -A _PORT=([lidarr]="$ARR_SYNC_LIDARR_PORT" [sonarr]="$ARR_SYNC_SONARR_PORT" [radarr]="$ARR_SYNC_RADARR_PORT")
|
||||
declare -A _VER=( [lidarr]="v1" [sonarr]="v3" [radarr]="v3")
|
||||
declare -A _EP=( [lidarr]="artist" [sonarr]="series" [radarr]="movie")
|
||||
declare -A _ID=( [lidarr]="foreignArtistId" [sonarr]="tvdbId" [radarr]="tmdbId")
|
||||
declare -A _NAME=([lidarr]="artistName" [sonarr]="title" [radarr]="title")
|
||||
# ID type: "string" for MusicBrainz UUID, "int" for TVDB/TMDB numeric IDs
|
||||
declare -A _ID_TYPE=([lidarr]="string" [sonarr]="int" [radarr]="int")
|
||||
ARR_TYPES=(lidarr sonarr radarr)
|
||||
|
||||
# ── Remote node discovery ──────────────────────────────────────────────────────────────────────
|
||||
REMOTE_NODES=()
|
||||
for _hv in HOST1 HOST2 HOST3 HOST4 HOST5 HOST6 HOST7 HOST8; do
|
||||
[[ "$_hv" == "$MY_ID" ]] && continue
|
||||
[[ -z "${!_hv:-}" ]] && continue
|
||||
REMOTE_NODES+=("$_hv")
|
||||
done
|
||||
unset _hv
|
||||
|
||||
if [[ "${#REMOTE_NODES[@]}" -eq 0 ]]; then
|
||||
warn "No remote nodes defined in master.conf — nothing to sync"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo " Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo " Remote nodes: ${REMOTE_NODES[*]}"
|
||||
echo " Blocklist: $ARR_SYNC_BLOCKLIST"
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
|
||||
# ==============================================================================================
|
||||
# ── BLOCKLIST ─────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
declare -A BLOCKLIST_MAP # key: "arr_type:stable_id" → display name
|
||||
|
||||
_load_blocklist() {
|
||||
BLOCKLIST_MAP=()
|
||||
local count=0
|
||||
|
||||
_parse_blocklist_lines() {
|
||||
while IFS=$'\t' read -r arr_type stable_id display_name rest; do
|
||||
[[ -z "$arr_type" ]] || [[ "$arr_type" == \#* ]] && continue
|
||||
BLOCKLIST_MAP["${arr_type}:${stable_id}"]="$display_name"
|
||||
(( count++ ))
|
||||
done
|
||||
}
|
||||
|
||||
# Local blocklist
|
||||
[[ -f "$ARR_SYNC_BLOCKLIST" ]] && _parse_blocklist_lines < "$ARR_SYNC_BLOCKLIST"
|
||||
|
||||
# Remote blocklists — read via SSH so tombstones are effective immediately
|
||||
for _node_id in "${REMOTE_NODES[@]}"; do
|
||||
local _node_ip
|
||||
_node_ip=$(_resolve_node_ip "$_node_id") || continue
|
||||
local _remote_lines
|
||||
_remote_lines=$(ssh -i "$SSH_KEY" -o ConnectTimeout="$ARR_SYNC_CONNECT_TIMEOUT" \
|
||||
root@"$_node_ip" "cat '$ARR_SYNC_BLOCKLIST' 2>/dev/null" 2>/dev/null) || continue
|
||||
_parse_blocklist_lines <<< "$_remote_lines"
|
||||
done
|
||||
unset _node_id _node_ip _remote_lines
|
||||
|
||||
log "Loaded blocklist: $count entries from $(( ${#REMOTE_NODES[@]} + 1 )) nodes"
|
||||
unset -f _parse_blocklist_lines
|
||||
}
|
||||
|
||||
_is_blocklisted() {
|
||||
[[ -n "${BLOCKLIST_MAP["${1}:${2}"]:-}" ]]
|
||||
}
|
||||
|
||||
_blocklist_add() {
|
||||
local arr_type="$1" stable_id="$2" display_name="$3" reason="${4:-manually excluded}"
|
||||
mkdir -p "$(dirname "$ARR_SYNC_BLOCKLIST")"
|
||||
if ! grep -qP "^${arr_type}\t${stable_id}\t" "$ARR_SYNC_BLOCKLIST" 2>/dev/null; then
|
||||
printf '%s\t%s\t%s\t%s\t%s\t%s\n' \
|
||||
"$arr_type" "$stable_id" "$display_name" \
|
||||
"$MY_ID" "$(date -Iseconds)" "$reason" \
|
||||
>> "$ARR_SYNC_BLOCKLIST"
|
||||
BLOCKLIST_MAP["${arr_type}:${stable_id}"]="$display_name"
|
||||
log "Blocklisted: [$arr_type] $display_name ($stable_id)"
|
||||
else
|
||||
log "Already blocklisted: [$arr_type] $stable_id"
|
||||
fi
|
||||
}
|
||||
|
||||
_blocklist_remove() {
|
||||
local arr_type="$1" stable_id="$2"
|
||||
if [[ -f "$ARR_SYNC_BLOCKLIST" ]]; then
|
||||
local tmp
|
||||
tmp=$(mktemp)
|
||||
grep -vP "^${arr_type}\t${stable_id}\t" "$ARR_SYNC_BLOCKLIST" > "$tmp" && \
|
||||
mv "$tmp" "$ARR_SYNC_BLOCKLIST" || rm -f "$tmp"
|
||||
unset "BLOCKLIST_MAP[${arr_type}:${stable_id}]"
|
||||
log "Removed from blocklist: [$arr_type] $stable_id"
|
||||
fi
|
||||
}
|
||||
|
||||
# Look up item in local arr by stable_id — returns "internal_id\tdisplay_name" or empty
|
||||
_lookup_local_item() {
|
||||
local url="$1" api_key="$2" api_ver="$3" endpoint="$4"
|
||||
local id_field="$5" id_type="$6" name_field="$7" stable_id="$8"
|
||||
local raw select_expr
|
||||
raw=$(curl -sf --max-time "$ARR_SYNC_API_TIMEOUT" \
|
||||
-H "X-Api-Key: $api_key" \
|
||||
"${url}/api/${api_ver}/${endpoint}" 2>/dev/null)
|
||||
[[ -z "$raw" ]] && return 1
|
||||
if [[ "$id_type" == "string" ]]; then
|
||||
select_expr=".[] | select(.${id_field} == \"${stable_id}\")"
|
||||
else
|
||||
select_expr=".[] | select(.${id_field} == ${stable_id})"
|
||||
fi
|
||||
echo "$raw" | jq -r "${select_expr} | [(.id | tostring), .${name_field}] | @tsv" 2>/dev/null | head -1
|
||||
}
|
||||
|
||||
# Delete item from local arr by internal integer id — returns HTTP status code
|
||||
_delete_local_item() {
|
||||
local url="$1" api_key="$2" api_ver="$3" endpoint="$4" internal_id="$5"
|
||||
curl -sf -o /dev/null -w '%{http_code}' -X DELETE \
|
||||
-H "X-Api-Key: $api_key" \
|
||||
"${url}/api/${api_ver}/${endpoint}/${internal_id}?deleteFiles=false" 2>/dev/null
|
||||
}
|
||||
|
||||
# Delete item from remote arr by stable_id.
|
||||
# Outputs: HTTP code on success | "not_found" if item absent | empty on failure.
|
||||
# deleteFiles=false — files become orphans for arr_cleanup to handle with its safety checks.
|
||||
# Args: node_id port api_ver arr_type endpoint id_field id_type stable_id
|
||||
_delete_remote_item() {
|
||||
local node_id="$1" port="$2" api_ver="$3" arr_type="$4" endpoint="$5"
|
||||
local id_field="$6" id_type="$7" stable_id="$8"
|
||||
local node_name="${!node_id}"
|
||||
local node_ip
|
||||
node_ip=$(resolve_tailscale_ip "$node_name") || return 1
|
||||
|
||||
local select_expr
|
||||
if [[ "$id_type" == "string" ]]; then
|
||||
select_expr=".[] | select(.${id_field} == \"${stable_id}\") | .id"
|
||||
else
|
||||
select_expr=".[] | select(.${id_field} == ${stable_id}) | .id"
|
||||
fi
|
||||
|
||||
local _kvar="${node_id}_${arr_type^^}_API_KEY"; local cached_key="${!_kvar:-}"
|
||||
if [[ -n "$cached_key" ]]; then
|
||||
local library internal_id
|
||||
library=$(curl -sf --max-time "$ARR_SYNC_API_TIMEOUT" \
|
||||
-H "X-Api-Key: $cached_key" \
|
||||
"http://${node_ip}:${port}/api/${api_ver}/${endpoint}" 2>/dev/null)
|
||||
[[ -z "$library" ]] && return 1
|
||||
internal_id=$(echo "$library" | jq -r "${select_expr}" 2>/dev/null | head -1)
|
||||
[[ -z "$internal_id" ]] && echo "not_found" && return 0
|
||||
curl -sf -o /dev/null -w '%{http_code}' -X DELETE \
|
||||
-H "X-Api-Key: $cached_key" \
|
||||
"http://${node_ip}:${port}/api/${api_ver}/${endpoint}/${internal_id}?deleteFiles=false" 2>/dev/null
|
||||
return
|
||||
fi
|
||||
local config_xml="${DOCKER_APPDATA_BASE}/${arr_type^}/config.xml"
|
||||
ssh -i "$SSH_KEY" -o ConnectTimeout="$ARR_SYNC_CONNECT_TIMEOUT" \
|
||||
root@"$node_ip" bash <<REMOTE 2>/dev/null
|
||||
KEY=\$(grep -oP '(?<=<ApiKey>)[^<]+' '${config_xml}' 2>/dev/null)
|
||||
[[ -z "\$KEY" ]] && exit 1
|
||||
LIBRARY=\$(curl -sf --max-time ${ARR_SYNC_API_TIMEOUT} \
|
||||
-H "X-Api-Key: \$KEY" \
|
||||
"http://localhost:${port}/api/${api_ver}/${endpoint}" 2>/dev/null)
|
||||
[[ -z "\$LIBRARY" ]] && exit 1
|
||||
INTERNAL_ID=\$(echo "\$LIBRARY" | jq -r '${select_expr}' 2>/dev/null | head -1)
|
||||
[[ -z "\$INTERNAL_ID" ]] && echo "not_found" && exit 0
|
||||
curl -sf -o /dev/null -w '%{http_code}' -X DELETE \
|
||||
-H "X-Api-Key: \$KEY" \
|
||||
"http://localhost:${port}/api/${api_ver}/${endpoint}/\${INTERNAL_ID}?deleteFiles=false"
|
||||
REMOTE
|
||||
}
|
||||
|
||||
# ── Blocklist management mode ──────────────────────────────────────────────────────────────────
|
||||
if [[ -n "$BLOCKLIST_ACTION" ]]; then
|
||||
case "$BLOCKLIST_ACTION" in
|
||||
list)
|
||||
echo ""
|
||||
echo "━━━━━ ARR SYNC BLOCKLIST ━━━━━"
|
||||
if [[ ! -f "$ARR_SYNC_BLOCKLIST" ]] || [[ ! -s "$ARR_SYNC_BLOCKLIST" ]]; then
|
||||
echo " (empty)"
|
||||
else
|
||||
echo ""
|
||||
printf '%-8s %-40s %-30s %s\n' "Arr" "Stable ID" "Name" "Reason"
|
||||
printf '%-8s %-40s %-30s %s\n' "---" "---------" "----" "------"
|
||||
while IFS=$'\t' read -r arr_type stable_id display_name tombstoned_by ts reason; do
|
||||
[[ -z "$arr_type" ]] || [[ "$arr_type" == \#* ]] && continue
|
||||
printf '%-8s %-40s %-30s %s\n' "$arr_type" "$stable_id" "$display_name" "$reason"
|
||||
done < "$ARR_SYNC_BLOCKLIST"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
;;
|
||||
add)
|
||||
if [[ -z "$BLOCKLIST_ARR" ]] || [[ -z "$BLOCKLIST_ID" ]]; then
|
||||
error "Usage: arr_sync.sh --blocklist-add <arr_type> <stable_id> [reason]"
|
||||
error " arr_type: lidarr | sonarr | radarr"
|
||||
exit 1
|
||||
fi
|
||||
if [[ -z "${_PORT[$BLOCKLIST_ARR]:-}" ]]; then
|
||||
error "Unknown arr type: $BLOCKLIST_ARR — use lidarr, sonarr, or radarr"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
_bl_port="${_PORT[$BLOCKLIST_ARR]}"
|
||||
_bl_ver="${_VER[$BLOCKLIST_ARR]}"
|
||||
_bl_ep="${_EP[$BLOCKLIST_ARR]}"
|
||||
_bl_id_field="${_ID[$BLOCKLIST_ARR]}"
|
||||
_bl_id_type="${_ID_TYPE[$BLOCKLIST_ARR]}"
|
||||
_bl_name_field="${_NAME[$BLOCKLIST_ARR]}"
|
||||
_bl_url="" _bl_key=""
|
||||
case "$BLOCKLIST_ARR" in
|
||||
lidarr) _bl_url="${LIDARR_URL:-}"; _bl_key="${LIDARR_API_KEY:-}" ;;
|
||||
sonarr) _bl_url="${SONARR_URL:-}"; _bl_key="${SONARR_API_KEY:-}" ;;
|
||||
radarr) _bl_url="${RADARR_URL:-}"; _bl_key="${RADARR_API_KEY:-}" ;;
|
||||
esac
|
||||
|
||||
# Look up display name and internal id from local arr
|
||||
_bl_display_name="$BLOCKLIST_ID"
|
||||
_bl_internal_id=""
|
||||
if [[ -n "$_bl_url" ]] && [[ -n "$_bl_key" ]]; then
|
||||
_bl_lookup=$(_lookup_local_item "$_bl_url" "$_bl_key" "$_bl_ver" "$_bl_ep" \
|
||||
"$_bl_id_field" "$_bl_id_type" "$_bl_name_field" "$BLOCKLIST_ID")
|
||||
if [[ -n "$_bl_lookup" ]]; then
|
||||
IFS=$'\t' read -r _bl_internal_id _bl_display_name <<< "$_bl_lookup"
|
||||
fi
|
||||
fi
|
||||
|
||||
_blocklist_add "$BLOCKLIST_ARR" "$BLOCKLIST_ID" "$_bl_display_name" "${BLOCKLIST_REASON:-manually excluded}"
|
||||
echo " Blocklisted [$BLOCKLIST_ARR] $_bl_display_name ($BLOCKLIST_ID)"
|
||||
|
||||
# Remove from local arr (deleteFiles=false — arr_cleanup handles file removal)
|
||||
if [[ -n "$_bl_internal_id" ]]; then
|
||||
_bl_http=$(_delete_local_item "$_bl_url" "$_bl_key" "$_bl_ver" "$_bl_ep" "$_bl_internal_id")
|
||||
if [[ "$_bl_http" == "200" ]]; then
|
||||
log "Removed from local ${BLOCKLIST_ARR^}: $_bl_display_name"
|
||||
else
|
||||
warn "Failed to remove from local ${BLOCKLIST_ARR^} (HTTP ${_bl_http:-no response}) — remove manually via UI"
|
||||
fi
|
||||
else
|
||||
log "Not found in local ${BLOCKLIST_ARR^} — already removed or not tracked locally"
|
||||
fi
|
||||
|
||||
# Remove from all remote arrs
|
||||
for _bl_node_id in "${REMOTE_NODES[@]}"; do
|
||||
_bl_node_name="${!_bl_node_id}"
|
||||
_bl_result=$(_delete_remote_item "$_bl_node_id" "$_bl_port" "$_bl_ver" \
|
||||
"$BLOCKLIST_ARR" "$_bl_ep" "$_bl_id_field" "$_bl_id_type" "$BLOCKLIST_ID")
|
||||
case "$_bl_result" in
|
||||
200) log "Removed from $_bl_node_name ${BLOCKLIST_ARR^}: $_bl_display_name" ;;
|
||||
not_found) log "Not found on $_bl_node_name ${BLOCKLIST_ARR^} — already removed or not tracked" ;;
|
||||
*) warn "Failed to remove from $_bl_node_name ${BLOCKLIST_ARR^} (${_bl_result:-SSH error}) — remove manually via UI" ;;
|
||||
esac
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo " Files are now orphans on all nodes — arr_cleanup.sh will remove them on next run"
|
||||
exit 0
|
||||
;;
|
||||
remove)
|
||||
if [[ -z "$BLOCKLIST_ARR" ]] || [[ -z "$BLOCKLIST_ID" ]]; then
|
||||
error "Usage: arr_sync.sh --blocklist-remove <arr_type> <stable_id>"
|
||||
exit 1
|
||||
fi
|
||||
_blocklist_remove "$BLOCKLIST_ARR" "$BLOCKLIST_ID"
|
||||
echo " Removed [$BLOCKLIST_ARR] $BLOCKLIST_ID from blocklist"
|
||||
exit 0
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ── STATUS ────────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY ARR SYNC STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_HOST Remote nodes: ${REMOTE_NODES[*]}"
|
||||
echo "$ICON_GEAR Appdata base: $DOCKER_APPDATA_BASE"
|
||||
echo "$ICON_GEAR Blocklist: $ARR_SYNC_BLOCKLIST"
|
||||
echo "$ICON_GEAR Connect timeout: ${ARR_SYNC_CONNECT_TIMEOUT}s"
|
||||
echo "$ICON_GEAR API timeout: ${ARR_SYNC_API_TIMEOUT}s"
|
||||
echo ""
|
||||
echo " Arr Port URL"
|
||||
for _arr in "${ARR_TYPES[@]}"; do
|
||||
local _url="" _key=""
|
||||
case "$_arr" in
|
||||
lidarr) _url="${LIDARR_URL:-not configured}" ;;
|
||||
sonarr) _url="${SONARR_URL:-not configured}" ;;
|
||||
radarr) _url="${RADARR_URL:-not configured}" ;;
|
||||
esac
|
||||
printf ' %-8s %-6s %s\n' "$_arr" "${_PORT[$_arr]}" "$_url"
|
||||
done
|
||||
unset _arr _url
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ── REMOTE HELPERS ────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
_resolve_node_ip() {
|
||||
local node_id="$1"
|
||||
resolve_tailscale_ip "${!node_id}"
|
||||
}
|
||||
|
||||
# Check if arr is reachable on remote node
|
||||
# Args: node_id node_ip port api_ver arr_type
|
||||
_remote_arr_up() {
|
||||
local node_id="$1" node_ip="$2" port="$3" api_ver="$4" arr_type="$5"
|
||||
local _kvar="${node_id}_${arr_type^^}_API_KEY"; local cached_key="${!_kvar:-}"
|
||||
if [[ -n "$cached_key" ]]; then
|
||||
curl -sf --max-time 5 \
|
||||
-H "X-Api-Key: $cached_key" \
|
||||
"http://${node_ip}:${port}/api/${api_ver}/system/status" >/dev/null 2>/dev/null
|
||||
return
|
||||
fi
|
||||
local config_xml="${DOCKER_APPDATA_BASE}/${arr_type^}/config.xml"
|
||||
ssh -i "$SSH_KEY" -o ConnectTimeout="$ARR_SYNC_CONNECT_TIMEOUT" \
|
||||
root@"$node_ip" \
|
||||
"KEY=\$(grep -oP '(?<=<ApiKey>)[^<]+' '${config_xml}' 2>/dev/null)
|
||||
[[ -z \"\$KEY\" ]] && exit 1
|
||||
curl -sf --max-time 5 -H \"X-Api-Key: \$KEY\" \
|
||||
'http://localhost:${port}/api/${api_ver}/system/status' >/dev/null" 2>/dev/null
|
||||
}
|
||||
|
||||
# Fetch full library from remote arr — returns raw JSON array
|
||||
# Args: node_id node_ip port api_ver arr_type endpoint
|
||||
_remote_library() {
|
||||
local node_id="$1" node_ip="$2" port="$3" api_ver="$4" arr_type="$5" endpoint="$6"
|
||||
local _kvar="${node_id}_${arr_type^^}_API_KEY"; local cached_key="${!_kvar:-}"
|
||||
if [[ -n "$cached_key" ]]; then
|
||||
curl -sf --max-time "$ARR_SYNC_API_TIMEOUT" \
|
||||
-H "X-Api-Key: $cached_key" \
|
||||
"http://${node_ip}:${port}/api/${api_ver}/${endpoint}" 2>/dev/null
|
||||
return
|
||||
fi
|
||||
local config_xml="${DOCKER_APPDATA_BASE}/${arr_type^}/config.xml"
|
||||
ssh -i "$SSH_KEY" -o ConnectTimeout="$ARR_SYNC_CONNECT_TIMEOUT" \
|
||||
root@"$node_ip" \
|
||||
"KEY=\$(grep -oP '(?<=<ApiKey>)[^<]+' '${config_xml}' 2>/dev/null)
|
||||
[[ -z \"\$KEY\" ]] && exit 1
|
||||
curl -sf --max-time ${ARR_SYNC_API_TIMEOUT} \
|
||||
-H \"X-Api-Key: \$KEY\" \
|
||||
'http://localhost:${port}/api/${api_ver}/${endpoint}'" 2>/dev/null
|
||||
}
|
||||
|
||||
# Fetch remote arr defaults: qualityProfileId, rootFolderPath, metadataProfileId (Lidarr)
|
||||
# Args: node_id node_ip port api_ver arr_type
|
||||
_remote_defaults() {
|
||||
local node_id="$1" node_ip="$2" port="$3" api_ver="$4" arr_type="$5"
|
||||
local _kvar="${node_id}_${arr_type^^}_API_KEY"; local cached_key="${!_kvar:-}"
|
||||
if [[ -n "$cached_key" ]]; then
|
||||
local base_url="http://${node_ip}:${port}/api/${api_ver}"
|
||||
local qp rf mp
|
||||
qp=$(curl -sf -H "X-Api-Key: $cached_key" "${base_url}/qualityprofile" 2>/dev/null | jq '.[0].id // 1')
|
||||
rf=$(curl -sf -H "X-Api-Key: $cached_key" "${base_url}/rootfolder" 2>/dev/null | jq -r '.[0].path // ""')
|
||||
[[ -z "$qp" ]] && return 1
|
||||
if [[ "$arr_type" == "lidarr" ]]; then
|
||||
mp=$(curl -sf -H "X-Api-Key: $cached_key" "${base_url}/metadataprofile" 2>/dev/null | \
|
||||
jq 'map(select(.name == "Standard")) | .[0].id // .[0].id // 1')
|
||||
jq -n --argjson qp "$qp" --arg rf "$rf" --argjson mp "$mp" \
|
||||
'{qualityProfileId: $qp, rootFolderPath: $rf, metadataProfileId: $mp}'
|
||||
else
|
||||
jq -n --argjson qp "$qp" --arg rf "$rf" \
|
||||
'{qualityProfileId: $qp, rootFolderPath: $rf}'
|
||||
fi
|
||||
return
|
||||
fi
|
||||
local config_xml="${DOCKER_APPDATA_BASE}/${arr_type^}/config.xml"
|
||||
local meta_field=""
|
||||
[[ "$arr_type" == "lidarr" ]] && \
|
||||
meta_field=', metadataProfileId: ($mp | map(select(.name == "Standard")) | .[0].id // .[0].id // 1)'
|
||||
ssh -i "$SSH_KEY" -o ConnectTimeout="$ARR_SYNC_CONNECT_TIMEOUT" \
|
||||
root@"$node_ip" bash <<REMOTE 2>/dev/null
|
||||
KEY=\$(grep -oP '(?<=<ApiKey>)[^<]+' '${config_xml}' 2>/dev/null)
|
||||
[[ -z "\$KEY" ]] && exit 1
|
||||
QP=\$(curl -sf -H "X-Api-Key: \$KEY" "http://localhost:${port}/api/${api_ver}/qualityprofile")
|
||||
RF=\$(curl -sf -H "X-Api-Key: \$KEY" "http://localhost:${port}/api/${api_ver}/rootfolder")
|
||||
MP=\$(curl -sf -H "X-Api-Key: \$KEY" "http://localhost:${port}/api/${api_ver}/metadataprofile" 2>/dev/null || echo '[]')
|
||||
jq -n --argjson qp "\$QP" --argjson rf "\$RF" --argjson mp "\$MP" \
|
||||
'{qualityProfileId: (\$qp | .[0].id // 1), rootFolderPath: (\$rf | .[0].path // "")${meta_field}}'
|
||||
REMOTE
|
||||
}
|
||||
|
||||
# Add item to remote arr — payload is base64-encoded to avoid quoting issues
|
||||
# Args: node_id node_ip port api_ver arr_type endpoint encoded_payload
|
||||
_remote_add() {
|
||||
local node_id="$1" node_ip="$2" port="$3" api_ver="$4" arr_type="$5" endpoint="$6"
|
||||
local encoded="$7"
|
||||
local _kvar="${node_id}_${arr_type^^}_API_KEY"; local cached_key="${!_kvar:-}"
|
||||
if [[ -n "$cached_key" ]]; then
|
||||
local body
|
||||
body=$(printf '%s' "$encoded" | base64 -d)
|
||||
curl -sf -o /dev/null -w '%{http_code}' -X POST \
|
||||
-H "X-Api-Key: $cached_key" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "$body" \
|
||||
"http://${node_ip}:${port}/api/${api_ver}/${endpoint}" 2>/dev/null
|
||||
return
|
||||
fi
|
||||
local config_xml="${DOCKER_APPDATA_BASE}/${arr_type^}/config.xml"
|
||||
ssh -i "$SSH_KEY" -o ConnectTimeout="$ARR_SYNC_CONNECT_TIMEOUT" \
|
||||
root@"$node_ip" bash <<REMOTE 2>/dev/null
|
||||
KEY=\$(grep -oP '(?<=<ApiKey>)[^<]+' '${config_xml}' 2>/dev/null)
|
||||
[[ -z "\$KEY" ]] && exit 1
|
||||
BODY=\$(printf '%s' '${encoded}' | base64 -d)
|
||||
curl -sf -o /dev/null -w '%{http_code}' -X POST \
|
||||
-H "X-Api-Key: \$KEY" \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d "\$BODY" \
|
||||
"http://localhost:${port}/api/${api_ver}/${endpoint}"
|
||||
REMOTE
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ── LOCAL HELPERS ─────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
_local_library() {
|
||||
local url="$1" api_key="$2" api_ver="$3" endpoint="$4"
|
||||
curl -sf --max-time "$ARR_SYNC_API_TIMEOUT" \
|
||||
-H "X-Api-Key: $api_key" \
|
||||
"${url}/api/${api_ver}/${endpoint}" 2>/dev/null
|
||||
}
|
||||
|
||||
_local_defaults() {
|
||||
local url="$1" api_key="$2" api_ver="$3" arr_type="$4"
|
||||
local qp rf mp meta_field=""
|
||||
qp=$(curl -sf -H "X-Api-Key: $api_key" "${url}/api/${api_ver}/qualityprofile" | jq '.[0].id // 1')
|
||||
rf=$(curl -sf -H "X-Api-Key: $api_key" "${url}/api/${api_ver}/rootfolder" | jq -r '.[0].path // ""')
|
||||
if [[ "$arr_type" == "lidarr" ]]; then
|
||||
mp=$(curl -sf -H "X-Api-Key: $api_key" "${url}/api/${api_ver}/metadataprofile" | \
|
||||
jq 'map(select(.name == "Standard")) | .[0].id // .[0].id // 1')
|
||||
jq -n --argjson qp "$qp" --arg rf "$rf" --argjson mp "$mp" \
|
||||
'{qualityProfileId: $qp, rootFolderPath: $rf, metadataProfileId: $mp}'
|
||||
else
|
||||
jq -n --argjson qp "$qp" --arg rf "$rf" \
|
||||
'{qualityProfileId: $qp, rootFolderPath: $rf}'
|
||||
fi
|
||||
}
|
||||
|
||||
_local_add() {
|
||||
local url="$1" api_key="$2" api_ver="$3" endpoint="$4" payload="$5"
|
||||
curl -sf -o /dev/null -w '%{http_code}' -X POST \
|
||||
-H "X-Api-Key: $api_key" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "$payload" \
|
||||
"${url}/api/${api_ver}/${endpoint}" 2>/dev/null
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ── PAYLOAD BUILDER ───────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Builds the minimal POST body to add an item to an arr.
|
||||
# Uses the TARGET node's own defaults — never copies source node settings.
|
||||
|
||||
_build_payload() {
|
||||
local arr_type="$1" stable_id="$2" display_name="$3" monitored="$4" defaults_json="$5"
|
||||
case "$arr_type" in
|
||||
lidarr)
|
||||
jq -n \
|
||||
--arg id "$stable_id" \
|
||||
--arg nm "$display_name" \
|
||||
--argjson mn "$monitored" \
|
||||
--argjson df "$defaults_json" \
|
||||
'{
|
||||
foreignArtistId: $id,
|
||||
artistName: $nm,
|
||||
monitored: $mn,
|
||||
qualityProfileId: $df.qualityProfileId,
|
||||
metadataProfileId: ($df.metadataProfileId // 1),
|
||||
rootFolderPath: $df.rootFolderPath,
|
||||
addOptions: {monitor: "all", searchForMissingAlbums: false}
|
||||
}'
|
||||
;;
|
||||
sonarr)
|
||||
jq -n \
|
||||
--argjson id "$stable_id" \
|
||||
--arg nm "$display_name" \
|
||||
--argjson mn "$monitored" \
|
||||
--argjson df "$defaults_json" \
|
||||
'{
|
||||
tvdbId: $id,
|
||||
title: $nm,
|
||||
monitored: $mn,
|
||||
qualityProfileId: $df.qualityProfileId,
|
||||
rootFolderPath: $df.rootFolderPath,
|
||||
seasons: [],
|
||||
addOptions: {searchForMissingEpisodes: false, monitor: "all"}
|
||||
}'
|
||||
;;
|
||||
radarr)
|
||||
jq -n \
|
||||
--argjson id "$stable_id" \
|
||||
--arg nm "$display_name" \
|
||||
--argjson mn "$monitored" \
|
||||
--argjson df "$defaults_json" \
|
||||
'{
|
||||
tmdbId: $id,
|
||||
title: $nm,
|
||||
monitored: $mn,
|
||||
qualityProfileId: $df.qualityProfileId,
|
||||
rootFolderPath: $df.rootFolderPath,
|
||||
addOptions: {searchForMovie: false}
|
||||
}'
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ── CORE SYNC — one arr type across all remote nodes ──────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
_sync_arr() {
|
||||
local arr_type="$1"
|
||||
local port="${_PORT[$arr_type]}"
|
||||
local api_ver="${_VER[$arr_type]}"
|
||||
local endpoint="${_EP[$arr_type]}"
|
||||
local id_field="${_ID[$arr_type]}"
|
||||
local name_field="${_NAME[$arr_type]}"
|
||||
local id_type="${_ID_TYPE[$arr_type]}"
|
||||
|
||||
# Resolve local credentials
|
||||
local local_url local_key
|
||||
case "$arr_type" in
|
||||
lidarr) local_url="${LIDARR_URL:-}"; local_key="${LIDARR_API_KEY:-}" ;;
|
||||
sonarr) local_url="${SONARR_URL:-}"; local_key="${SONARR_API_KEY:-}" ;;
|
||||
radarr) local_url="${RADARR_URL:-}"; local_key="${RADARR_API_KEY:-}" ;;
|
||||
esac
|
||||
|
||||
if [[ -z "$local_url" ]] || [[ -z "$local_key" ]]; then
|
||||
log "${arr_type^}: not configured on $MY_ID — skipping"
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "━━━ ${arr_type^} ━━━"
|
||||
|
||||
# ── Fetch local library ────────────────────────────────────────────────────────────────────
|
||||
local local_json
|
||||
local_json=$(_local_library "$local_url" "$local_key" "$api_ver" "$endpoint")
|
||||
if [[ -z "$local_json" ]] || ! echo "$local_json" | jq -e '.' >/dev/null 2>&1; then
|
||||
warn "${arr_type^}: could not fetch local library — skipping"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Build local ID map: stable_id → "display_name|monitored"
|
||||
declare -A local_ids
|
||||
local local_count=0
|
||||
local _jq_id
|
||||
[[ "$id_type" == "string" ]] && _jq_id=".${id_field}" || _jq_id="(.${id_field} | tostring)"
|
||||
while IFS=$'\t' read -r stable_id display_name monitored; do
|
||||
[[ -z "$stable_id" ]] && continue
|
||||
local_ids["$stable_id"]="${display_name}|${monitored}"
|
||||
(( local_count++ ))
|
||||
done < <(echo "$local_json" | jq -r \
|
||||
".[] | [${_jq_id}, .${name_field}, (.monitored | tostring)] | @tsv" 2>/dev/null)
|
||||
unset _jq_id
|
||||
|
||||
log "${arr_type^}: $local_count items in local library"
|
||||
|
||||
local total_added_local=0 total_added_remote=0 total_skipped=0
|
||||
|
||||
# ── Sync with each remote node ─────────────────────────────────────────────────────────────
|
||||
for node_id in "${REMOTE_NODES[@]}"; do
|
||||
local node_name="${!node_id}"
|
||||
log "${arr_type^}: syncing with $node_name..."
|
||||
|
||||
local node_ip
|
||||
node_ip=$(_resolve_node_ip "$node_id") || {
|
||||
warn "${arr_type^}: cannot resolve Tailscale IP for $node_name — skipping"
|
||||
continue
|
||||
}
|
||||
|
||||
if ! _remote_arr_up "$node_id" "$node_ip" "$port" "$api_ver" "$arr_type"; then
|
||||
log "${arr_type^}: not reachable on $node_name — skipping"
|
||||
continue
|
||||
fi
|
||||
|
||||
local remote_json
|
||||
remote_json=$(_remote_library "$node_id" "$node_ip" "$port" "$api_ver" "$arr_type" "$endpoint")
|
||||
if [[ -z "$remote_json" ]] || ! echo "$remote_json" | jq -e '.' >/dev/null 2>&1; then
|
||||
warn "${arr_type^}: could not fetch library from $node_name — skipping"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Build remote ID map
|
||||
declare -A remote_ids
|
||||
local remote_count=0
|
||||
local _jq_id
|
||||
[[ "$id_type" == "string" ]] && _jq_id=".${id_field}" || _jq_id="(.${id_field} | tostring)"
|
||||
while IFS=$'\t' read -r stable_id display_name monitored; do
|
||||
[[ -z "$stable_id" ]] && continue
|
||||
remote_ids["$stable_id"]="${display_name}|${monitored}"
|
||||
(( remote_count++ ))
|
||||
done < <(echo "$remote_json" | jq -r \
|
||||
".[] | [${_jq_id}, .${name_field}, (.monitored | tostring)] | @tsv" 2>/dev/null)
|
||||
unset _jq_id
|
||||
|
||||
log "${arr_type^}: $remote_count items on $node_name"
|
||||
|
||||
# ── Remote → Local: items on remote not in local ───────────────────────────────────────
|
||||
local to_add_local=()
|
||||
for stable_id in "${!remote_ids[@]}"; do
|
||||
[[ -n "${local_ids[$stable_id]:-}" ]] && continue
|
||||
if _is_blocklisted "$arr_type" "$stable_id"; then
|
||||
log "BLOCKLISTED [$arr_type] $stable_id — skipping"
|
||||
(( total_skipped++ ))
|
||||
continue
|
||||
fi
|
||||
to_add_local+=("$stable_id")
|
||||
done
|
||||
|
||||
if [[ "${#to_add_local[@]}" -gt 0 ]]; then
|
||||
local local_defs
|
||||
local_defs=$(_local_defaults "$local_url" "$local_key" "$api_ver" "$arr_type")
|
||||
if [[ -z "$local_defs" ]]; then
|
||||
warn "${arr_type^}: could not fetch local defaults — skipping adds from $node_name"
|
||||
else
|
||||
for stable_id in "${to_add_local[@]}"; do
|
||||
IFS='|' read -r display_name monitored <<< "${remote_ids[$stable_id]}"
|
||||
local payload
|
||||
payload=$(_build_payload "$arr_type" "$stable_id" "$display_name" \
|
||||
"$monitored" "$local_defs")
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
log "DRY RUN: would add to local ${arr_type^}: $display_name ($stable_id)"
|
||||
(( total_added_local++ ))
|
||||
else
|
||||
local http_code
|
||||
http_code=$(_local_add "$local_url" "$local_key" "$api_ver" \
|
||||
"$endpoint" "$payload")
|
||||
if [[ "$http_code" == "201" ]] || [[ "$http_code" == "200" ]]; then
|
||||
log "Added to local ${arr_type^}: $display_name"
|
||||
local_ids["$stable_id"]="${display_name}|${monitored}"
|
||||
(( total_added_local++ ))
|
||||
else
|
||||
warn "Failed to add to local ${arr_type^}: $display_name (HTTP $http_code)"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Local → Remote: items on local not on remote ───────────────────────────────────────
|
||||
local to_add_remote=()
|
||||
for stable_id in "${!local_ids[@]}"; do
|
||||
[[ -n "${remote_ids[$stable_id]:-}" ]] && continue
|
||||
if _is_blocklisted "$arr_type" "$stable_id"; then
|
||||
(( total_skipped++ ))
|
||||
continue
|
||||
fi
|
||||
to_add_remote+=("$stable_id")
|
||||
done
|
||||
|
||||
if [[ "${#to_add_remote[@]}" -gt 0 ]]; then
|
||||
local remote_defs
|
||||
remote_defs=$(_remote_defaults "$node_id" "$node_ip" "$port" "$api_ver" "$arr_type")
|
||||
if [[ -z "$remote_defs" ]]; then
|
||||
warn "${arr_type^}: could not fetch defaults from $node_name — skipping remote adds"
|
||||
else
|
||||
for stable_id in "${to_add_remote[@]}"; do
|
||||
IFS='|' read -r display_name monitored <<< "${local_ids[$stable_id]}"
|
||||
local payload
|
||||
payload=$(_build_payload "$arr_type" "$stable_id" "$display_name" \
|
||||
"$monitored" "$remote_defs")
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
log "DRY RUN: would add to $node_name ${arr_type^}: $display_name ($stable_id)"
|
||||
(( total_added_remote++ ))
|
||||
else
|
||||
local encoded http_code
|
||||
encoded=$(printf '%s' "$payload" | base64 -w0)
|
||||
http_code=$(_remote_add "$node_id" "$node_ip" "$port" "$api_ver" \
|
||||
"$arr_type" "$endpoint" "$encoded")
|
||||
if [[ "$http_code" == "201" ]] || [[ "$http_code" == "200" ]]; then
|
||||
log "Added to $node_name ${arr_type^}: $display_name"
|
||||
(( total_added_remote++ ))
|
||||
else
|
||||
warn "Failed to add to $node_name ${arr_type^}: $display_name (HTTP $http_code)"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
fi
|
||||
fi
|
||||
|
||||
log " $node_name: +${#to_add_local[@]} local | +${#to_add_remote[@]} remote | $total_skipped blocklisted"
|
||||
|
||||
unset remote_ids
|
||||
declare -A remote_ids
|
||||
done
|
||||
|
||||
echo " Total added to local: $total_added_local | to remotes: $total_added_remote | blocklisted: $total_skipped"
|
||||
unset local_ids
|
||||
declare -A local_ids
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Main ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Loading Blocklist ━━━"
|
||||
_load_blocklist
|
||||
|
||||
START=$(date +%s)
|
||||
|
||||
for arr_type in "${ARR_TYPES[@]}"; do
|
||||
_sync_arr "$arr_type"
|
||||
done
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY ARR SYNC SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Node: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_SYNC Peers: ${REMOTE_NODES[*]}"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes were made"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
exit 0
|
||||
+913
@@ -0,0 +1,913 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ================================= ARR Sync ===================================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Full-mesh arr library sync across all nodes — Lidarr, Sonarr, and Radarr.
|
||||
# Every node syncs with every other, union model, no hierarchy. Run before
|
||||
# rsync in the weekly sync window: once arrs agree on what to track, rsync
|
||||
# spreads the actual files.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Full mesh: every node syncs with every other — no primary, no hierarchy.
|
||||
# Union model: if any node tracks an item, all nodes get it (unless blocklisted).
|
||||
# Convergence: any node can add content; after one full cycle all nodes agree.
|
||||
# Upgrade-aware: a file upgrade on one node → arr tracks new path → rsync spreads
|
||||
# it → arr_cleanup removes old path on all nodes (arr no longer tracks it).
|
||||
#
|
||||
# Node discovery: reads HOST* vars from master.conf. Add HOST3= and it joins the
|
||||
# sync automatically — no script changes needed for a new node.
|
||||
#
|
||||
# Graceful skip: arr not configured locally → skip cleanly. Arr not reachable on
|
||||
# a remote → skip that node for that arr type, continue with others.
|
||||
#
|
||||
# What gets synced — library items keyed on stable external IDs:
|
||||
# Lidarr — MusicBrainz artist ID (foreignArtistId)
|
||||
# Sonarr — TVDB series ID (tvdbId)
|
||||
# Radarr — TMDB movie ID (tmdbId)
|
||||
# When adding to a remote, that node's own quality profile, metadata profile,
|
||||
# and root folder path are used — settings are never copied from source.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# DESIGN PRINCIPLES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Remote API Access — Cache-First, SSH Fallback
|
||||
# If conf_sync.sh has populated /tmp/.vv/config/cached/.confs/ and
|
||||
# load_config.sh has sourced it, HOST*_<ARR>_API_KEY vars are available
|
||||
# in the environment. Remote functions use them to call the arr API
|
||||
# directly over Tailscale (no SSH, no remote shell). If the cached key
|
||||
# is absent (first boot, cache not yet populated) the functions fall back
|
||||
# to SSHing in and reading the key from config.xml on the remote node.
|
||||
#
|
||||
# Blocklist TSV
|
||||
# ARR_SYNC_BLOCKLIST in DATA_DIR tombstones IDs that must never be re-added
|
||||
# anywhere. Read from ALL nodes via SSH at run start — immediate effect with
|
||||
# no rsync delay.
|
||||
#
|
||||
# --blocklist-add does three things atomically:
|
||||
# 1. Writes the TSV tombstone entry (prevents future re-adds by arr_sync)
|
||||
# 2. Deletes the item from the local arr API (deleteFiles=false)
|
||||
# 3. SSHes each remote node and deletes from their arr API (deleteFiles=false)
|
||||
# Files become orphans on all nodes — arr_cleanup removes them on next run.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# acquire_lock — prevents two sync instances running simultaneously
|
||||
# ARR_SYNC_ENABLED — global gate, exits cleanly when false
|
||||
# SSH connect timeout — ARR_SYNC_CONNECT_TIMEOUT — does not hang on unreachable node
|
||||
# API call timeout — ARR_SYNC_API_TIMEOUT — does not hang on slow arr
|
||||
# Graceful skip — unreachable node/arr → skip and continue, never abort
|
||||
# Blocklist gate — item in blocklist → never added to any node
|
||||
# Silent by default — only additions produce output, clean runs stay silent
|
||||
#
|
||||
# ==============================================================================================
|
||||
# STATE FILES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# ARR_SYNC_BLOCKLIST — TSV file in DATA_DIR (default: DATA_DIR/arr_sync_blocklist.tsv)
|
||||
# Columns: arr_type, id, reason, date_added
|
||||
# Read from all nodes via SSH at the start of each run.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# host*.conf
|
||||
#
|
||||
# HOST*_LIDARR_URL / HOST*_LIDARR_API_KEY — local Lidarr (aliased by detect_hosts)
|
||||
# HOST*_SONARR_URL / HOST*_SONARR_API_KEY — local Sonarr
|
||||
# HOST*_RADARR_URL / HOST*_RADARR_API_KEY — local Radarr
|
||||
#
|
||||
# master.conf
|
||||
#
|
||||
# ARR_SYNC_ENABLED — global on/off toggle (default: true)
|
||||
# ARR_SYNC_BLOCKLIST — path to TSV blocklist file
|
||||
# ARR_SYNC_CONNECT_TIMEOUT — SSH connect timeout in seconds (default: 10)
|
||||
# ARR_SYNC_API_TIMEOUT — curl API call timeout in seconds (default: 60)
|
||||
# DOCKER_APPDATA_BASE — base path for arr appdata dirs (default: /mnt/user/appdata)
|
||||
# ARR_SYNC_LIDARR_PORT — Lidarr port on all nodes (default: 8686)
|
||||
# ARR_SYNC_SONARR_PORT — Sonarr port on all nodes (default: 8989)
|
||||
# ARR_SYNC_RADARR_PORT — Radarr port on all nodes (default: 7878)
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# arr_sync.sh — sync all arr types, all nodes
|
||||
# arr_sync.sh --dry-run — preview only, no changes
|
||||
# arr_sync.sh --log — verbose output
|
||||
# arr_sync.sh --status — show config and exit
|
||||
#
|
||||
# Blocklist management:
|
||||
# arr_sync.sh --blocklist-add lidarr <mbid> "reason" — remove from all arrs + tombstone
|
||||
# arr_sync.sh --blocklist-add sonarr <tvdbId> "reason" — remove from all arrs + tombstone
|
||||
# arr_sync.sh --blocklist-add radarr <tmdbId> "reason" — remove from all arrs + tombstone
|
||||
# arr_sync.sh --blocklist-remove lidarr <id> — un-tombstone (does NOT re-add)
|
||||
# arr_sync.sh --blocklist-list — show all blocklisted IDs
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
# ── Blocklist flag pre-processing ─────────────────────────────────────────────────────────────
|
||||
# Handle before parse_args — these are action flags, not standard options
|
||||
BLOCKLIST_ACTION=""
|
||||
BLOCKLIST_ARR=""
|
||||
BLOCKLIST_ID=""
|
||||
BLOCKLIST_REASON=""
|
||||
FILTERED_ARGS=()
|
||||
_skip_next=false
|
||||
for _arg in "$@"; do
|
||||
if [[ "$_skip_next" == true ]]; then _skip_next=false; continue; fi
|
||||
case "$_arg" in
|
||||
--blocklist-add) BLOCKLIST_ACTION="add" ;;
|
||||
--blocklist-remove) BLOCKLIST_ACTION="remove" ;;
|
||||
--blocklist-list) BLOCKLIST_ACTION="list" ;;
|
||||
*) FILTERED_ARGS+=("$_arg") ;;
|
||||
esac
|
||||
done
|
||||
unset _arg _skip_next
|
||||
|
||||
parse_args "${FILTERED_ARGS[@]}"
|
||||
|
||||
# Consume blocklist positional args from remaining FILTERED_ARGS
|
||||
# --blocklist-add lidarr <id> "reason"
|
||||
# --blocklist-remove lidarr <id>
|
||||
if [[ -n "$BLOCKLIST_ACTION" ]] && [[ "$BLOCKLIST_ACTION" != "list" ]]; then
|
||||
for _a in "${FILTERED_ARGS[@]}"; do
|
||||
[[ "$_a" == --* ]] && continue
|
||||
if [[ -z "$BLOCKLIST_ARR" ]]; then BLOCKLIST_ARR="$_a"
|
||||
elif [[ -z "$BLOCKLIST_ID" ]]; then BLOCKLIST_ID="$_a"
|
||||
elif [[ -z "$BLOCKLIST_REASON" ]]; then BLOCKLIST_REASON="$_a"
|
||||
fi
|
||||
done
|
||||
unset _a
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
acquire_lock
|
||||
|
||||
for _tool in curl jq; do
|
||||
if ! command -v "$_tool" >/dev/null 2>&1; then
|
||||
error "$_tool not found — required for arr API calls"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
unset _tool
|
||||
|
||||
detect_hosts
|
||||
|
||||
# ── Runtime config with defaults ──────────────────────────────────────────────────────────────
|
||||
ARR_SYNC_ENABLED="${ARR_SYNC_ENABLED:-true}"
|
||||
ARR_SYNC_BLOCKLIST="${ARR_SYNC_BLOCKLIST:-${DATA_DIR}/arr_sync_blocklist.tsv}"
|
||||
ARR_SYNC_CONNECT_TIMEOUT="${ARR_SYNC_CONNECT_TIMEOUT:-10}"
|
||||
ARR_SYNC_API_TIMEOUT="${ARR_SYNC_API_TIMEOUT:-60}"
|
||||
DOCKER_APPDATA_BASE="${DOCKER_APPDATA_BASE:-/mnt/user/appdata}"
|
||||
ARR_SYNC_LIDARR_PORT="${ARR_SYNC_LIDARR_PORT:-8686}"
|
||||
ARR_SYNC_SONARR_PORT="${ARR_SYNC_SONARR_PORT:-8989}"
|
||||
ARR_SYNC_RADARR_PORT="${ARR_SYNC_RADARR_PORT:-7878}"
|
||||
|
||||
if [[ "$ARR_SYNC_ENABLED" != "true" ]]; then
|
||||
log "ARR_SYNC_ENABLED=false — exiting"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
log "$ICON_GEAR Config: api-timeout=${ARR_SYNC_API_TIMEOUT}s connect-timeout=${ARR_SYNC_CONNECT_TIMEOUT}s blocklist=${ARR_SYNC_BLOCKLIST}"
|
||||
log "$ICON_GEAR Ports: lidarr=${ARR_SYNC_LIDARR_PORT} sonarr=${ARR_SYNC_SONARR_PORT} radarr=${ARR_SYNC_RADARR_PORT}"
|
||||
|
||||
# ── Arr type definitions ───────────────────────────────────────────────────────────────────────
|
||||
# Each arr type maps to its port, API version, endpoint, stable ID field, and display name field
|
||||
declare -A _PORT=([lidarr]="$ARR_SYNC_LIDARR_PORT" [sonarr]="$ARR_SYNC_SONARR_PORT" [radarr]="$ARR_SYNC_RADARR_PORT")
|
||||
declare -A _VER=( [lidarr]="v1" [sonarr]="v3" [radarr]="v3")
|
||||
declare -A _EP=( [lidarr]="artist" [sonarr]="series" [radarr]="movie")
|
||||
declare -A _ID=( [lidarr]="foreignArtistId" [sonarr]="tvdbId" [radarr]="tmdbId")
|
||||
declare -A _NAME=([lidarr]="artistName" [sonarr]="title" [radarr]="title")
|
||||
# ID type: "string" for MusicBrainz UUID, "int" for TVDB/TMDB numeric IDs
|
||||
declare -A _ID_TYPE=([lidarr]="string" [sonarr]="int" [radarr]="int")
|
||||
ARR_TYPES=(lidarr sonarr radarr)
|
||||
|
||||
# ── Remote node discovery ──────────────────────────────────────────────────────────────────────
|
||||
REMOTE_NODES=()
|
||||
for _hv in HOST1 HOST2 HOST3 HOST4 HOST5 HOST6 HOST7 HOST8; do
|
||||
[[ "$_hv" == "$MY_ID" ]] && continue
|
||||
[[ -z "${!_hv:-}" ]] && continue
|
||||
REMOTE_NODES+=("$_hv")
|
||||
done
|
||||
unset _hv
|
||||
|
||||
if [[ "${#REMOTE_NODES[@]}" -eq 0 ]]; then
|
||||
warn "No remote nodes defined in master.conf — nothing to sync"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo " Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo " Remote nodes: ${REMOTE_NODES[*]}"
|
||||
echo " Blocklist: $ARR_SYNC_BLOCKLIST"
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
|
||||
# ==============================================================================================
|
||||
# ── BLOCKLIST ─────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
declare -A BLOCKLIST_MAP # key: "arr_type:stable_id" → display name
|
||||
|
||||
_load_blocklist() {
|
||||
BLOCKLIST_MAP=()
|
||||
local count=0
|
||||
|
||||
_parse_blocklist_lines() {
|
||||
while IFS=$'\t' read -r arr_type stable_id display_name rest; do
|
||||
[[ -z "$arr_type" ]] || [[ "$arr_type" == \#* ]] && continue
|
||||
BLOCKLIST_MAP["${arr_type}:${stable_id}"]="$display_name"
|
||||
(( count++ ))
|
||||
done
|
||||
}
|
||||
|
||||
# Local blocklist
|
||||
[[ -f "$ARR_SYNC_BLOCKLIST" ]] && _parse_blocklist_lines < "$ARR_SYNC_BLOCKLIST"
|
||||
|
||||
# Remote blocklists — read via SSH so tombstones are effective immediately
|
||||
for _node_id in "${REMOTE_NODES[@]}"; do
|
||||
local _node_ip
|
||||
_node_ip=$(_resolve_node_ip "$_node_id") || continue
|
||||
local _remote_lines
|
||||
_remote_lines=$(ssh -i "$SSH_KEY" -o ConnectTimeout="$ARR_SYNC_CONNECT_TIMEOUT" \
|
||||
root@"$_node_ip" "cat '$ARR_SYNC_BLOCKLIST' 2>/dev/null" 2>/dev/null) || continue
|
||||
_parse_blocklist_lines <<< "$_remote_lines"
|
||||
done
|
||||
unset _node_id _node_ip _remote_lines
|
||||
|
||||
log "Loaded blocklist: $count entries from $(( ${#REMOTE_NODES[@]} + 1 )) nodes"
|
||||
unset -f _parse_blocklist_lines
|
||||
}
|
||||
|
||||
_is_blocklisted() {
|
||||
[[ -n "${BLOCKLIST_MAP["${1}:${2}"]:-}" ]]
|
||||
}
|
||||
|
||||
_blocklist_add() {
|
||||
local arr_type="$1" stable_id="$2" display_name="$3" reason="${4:-manually excluded}"
|
||||
mkdir -p "$(dirname "$ARR_SYNC_BLOCKLIST")"
|
||||
if ! grep -qP "^${arr_type}\t${stable_id}\t" "$ARR_SYNC_BLOCKLIST" 2>/dev/null; then
|
||||
printf '%s\t%s\t%s\t%s\t%s\t%s\n' \
|
||||
"$arr_type" "$stable_id" "$display_name" \
|
||||
"$MY_ID" "$(date -Iseconds)" "$reason" \
|
||||
>> "$ARR_SYNC_BLOCKLIST"
|
||||
BLOCKLIST_MAP["${arr_type}:${stable_id}"]="$display_name"
|
||||
log "Blocklisted: [$arr_type] $display_name ($stable_id)"
|
||||
else
|
||||
log "Already blocklisted: [$arr_type] $stable_id"
|
||||
fi
|
||||
}
|
||||
|
||||
_blocklist_remove() {
|
||||
local arr_type="$1" stable_id="$2"
|
||||
if [[ -f "$ARR_SYNC_BLOCKLIST" ]]; then
|
||||
local tmp
|
||||
tmp=$(mktemp)
|
||||
grep -vP "^${arr_type}\t${stable_id}\t" "$ARR_SYNC_BLOCKLIST" > "$tmp" && \
|
||||
mv "$tmp" "$ARR_SYNC_BLOCKLIST" || rm -f "$tmp"
|
||||
unset "BLOCKLIST_MAP[${arr_type}:${stable_id}]"
|
||||
log "Removed from blocklist: [$arr_type] $stable_id"
|
||||
fi
|
||||
}
|
||||
|
||||
# Look up item in local arr by stable_id — returns "internal_id\tdisplay_name" or empty
|
||||
_lookup_local_item() {
|
||||
local url="$1" api_key="$2" api_ver="$3" endpoint="$4"
|
||||
local id_field="$5" id_type="$6" name_field="$7" stable_id="$8"
|
||||
local raw select_expr
|
||||
raw=$(curl -sf --max-time "$ARR_SYNC_API_TIMEOUT" \
|
||||
-H "X-Api-Key: $api_key" \
|
||||
"${url}/api/${api_ver}/${endpoint}" 2>/dev/null)
|
||||
[[ -z "$raw" ]] && return 1
|
||||
if [[ "$id_type" == "string" ]]; then
|
||||
select_expr=".[] | select(.${id_field} == \"${stable_id}\")"
|
||||
else
|
||||
select_expr=".[] | select(.${id_field} == ${stable_id})"
|
||||
fi
|
||||
echo "$raw" | jq -r "${select_expr} | [(.id | tostring), .${name_field}] | @tsv" 2>/dev/null | head -1
|
||||
}
|
||||
|
||||
# Delete item from local arr by internal integer id — returns HTTP status code
|
||||
_delete_local_item() {
|
||||
local url="$1" api_key="$2" api_ver="$3" endpoint="$4" internal_id="$5"
|
||||
curl -sf -o /dev/null -w '%{http_code}' -X DELETE \
|
||||
-H "X-Api-Key: $api_key" \
|
||||
"${url}/api/${api_ver}/${endpoint}/${internal_id}?deleteFiles=false" 2>/dev/null
|
||||
}
|
||||
|
||||
# Delete item from remote arr by stable_id.
|
||||
# Outputs: HTTP code on success | "not_found" if item absent | empty on failure.
|
||||
# deleteFiles=false — files become orphans for arr_cleanup to handle with its safety checks.
|
||||
# Args: node_id port api_ver arr_type endpoint id_field id_type stable_id
|
||||
_delete_remote_item() {
|
||||
local node_id="$1" port="$2" api_ver="$3" arr_type="$4" endpoint="$5"
|
||||
local id_field="$6" id_type="$7" stable_id="$8"
|
||||
local node_name="${!node_id}"
|
||||
local node_ip
|
||||
node_ip=$(resolve_tailscale_ip "$node_name") || return 1
|
||||
|
||||
local select_expr
|
||||
if [[ "$id_type" == "string" ]]; then
|
||||
select_expr=".[] | select(.${id_field} == \"${stable_id}\") | .id"
|
||||
else
|
||||
select_expr=".[] | select(.${id_field} == ${stable_id}) | .id"
|
||||
fi
|
||||
|
||||
local _kvar="${node_id}_${arr_type^^}_API_KEY"; local cached_key="${!_kvar:-}"
|
||||
if [[ -n "$cached_key" ]]; then
|
||||
local library internal_id
|
||||
library=$(curl -sf --max-time "$ARR_SYNC_API_TIMEOUT" \
|
||||
-H "X-Api-Key: $cached_key" \
|
||||
"http://${node_ip}:${port}/api/${api_ver}/${endpoint}" 2>/dev/null)
|
||||
[[ -z "$library" ]] && return 1
|
||||
internal_id=$(echo "$library" | jq -r "${select_expr}" 2>/dev/null | head -1)
|
||||
[[ -z "$internal_id" ]] && echo "not_found" && return 0
|
||||
curl -sf -o /dev/null -w '%{http_code}' -X DELETE \
|
||||
-H "X-Api-Key: $cached_key" \
|
||||
"http://${node_ip}:${port}/api/${api_ver}/${endpoint}/${internal_id}?deleteFiles=false" 2>/dev/null
|
||||
return
|
||||
fi
|
||||
local config_xml="${DOCKER_APPDATA_BASE}/${arr_type^}/config.xml"
|
||||
ssh -i "$SSH_KEY" -o ConnectTimeout="$ARR_SYNC_CONNECT_TIMEOUT" \
|
||||
root@"$node_ip" bash <<REMOTE 2>/dev/null
|
||||
KEY=\$(grep -oP '(?<=<ApiKey>)[^<]+' '${config_xml}' 2>/dev/null)
|
||||
[[ -z "\$KEY" ]] && exit 1
|
||||
LIBRARY=\$(curl -sf --max-time ${ARR_SYNC_API_TIMEOUT} \
|
||||
-H "X-Api-Key: \$KEY" \
|
||||
"http://localhost:${port}/api/${api_ver}/${endpoint}" 2>/dev/null)
|
||||
[[ -z "\$LIBRARY" ]] && exit 1
|
||||
INTERNAL_ID=\$(echo "\$LIBRARY" | jq -r '${select_expr}' 2>/dev/null | head -1)
|
||||
[[ -z "\$INTERNAL_ID" ]] && echo "not_found" && exit 0
|
||||
curl -sf -o /dev/null -w '%{http_code}' -X DELETE \
|
||||
-H "X-Api-Key: \$KEY" \
|
||||
"http://localhost:${port}/api/${api_ver}/${endpoint}/\${INTERNAL_ID}?deleteFiles=false"
|
||||
REMOTE
|
||||
}
|
||||
|
||||
# ── Blocklist management mode ──────────────────────────────────────────────────────────────────
|
||||
if [[ -n "$BLOCKLIST_ACTION" ]]; then
|
||||
case "$BLOCKLIST_ACTION" in
|
||||
list)
|
||||
echo ""
|
||||
echo "━━━━━ ARR SYNC BLOCKLIST ━━━━━"
|
||||
if [[ ! -f "$ARR_SYNC_BLOCKLIST" ]] || [[ ! -s "$ARR_SYNC_BLOCKLIST" ]]; then
|
||||
echo " (empty)"
|
||||
else
|
||||
echo ""
|
||||
printf '%-8s %-40s %-30s %s\n' "Arr" "Stable ID" "Name" "Reason"
|
||||
printf '%-8s %-40s %-30s %s\n' "---" "---------" "----" "------"
|
||||
while IFS=$'\t' read -r arr_type stable_id display_name tombstoned_by ts reason; do
|
||||
[[ -z "$arr_type" ]] || [[ "$arr_type" == \#* ]] && continue
|
||||
printf '%-8s %-40s %-30s %s\n' "$arr_type" "$stable_id" "$display_name" "$reason"
|
||||
done < "$ARR_SYNC_BLOCKLIST"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
;;
|
||||
add)
|
||||
if [[ -z "$BLOCKLIST_ARR" ]] || [[ -z "$BLOCKLIST_ID" ]]; then
|
||||
error "Usage: arr_sync.sh --blocklist-add <arr_type> <stable_id> [reason]"
|
||||
error " arr_type: lidarr | sonarr | radarr"
|
||||
exit 1
|
||||
fi
|
||||
if [[ -z "${_PORT[$BLOCKLIST_ARR]:-}" ]]; then
|
||||
error "Unknown arr type: $BLOCKLIST_ARR — use lidarr, sonarr, or radarr"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
_bl_port="${_PORT[$BLOCKLIST_ARR]}"
|
||||
_bl_ver="${_VER[$BLOCKLIST_ARR]}"
|
||||
_bl_ep="${_EP[$BLOCKLIST_ARR]}"
|
||||
_bl_id_field="${_ID[$BLOCKLIST_ARR]}"
|
||||
_bl_id_type="${_ID_TYPE[$BLOCKLIST_ARR]}"
|
||||
_bl_name_field="${_NAME[$BLOCKLIST_ARR]}"
|
||||
_bl_url="" _bl_key=""
|
||||
case "$BLOCKLIST_ARR" in
|
||||
lidarr) _bl_url="${LIDARR_URL:-}"; _bl_key="${LIDARR_API_KEY:-}" ;;
|
||||
sonarr) _bl_url="${SONARR_URL:-}"; _bl_key="${SONARR_API_KEY:-}" ;;
|
||||
radarr) _bl_url="${RADARR_URL:-}"; _bl_key="${RADARR_API_KEY:-}" ;;
|
||||
esac
|
||||
|
||||
# Look up display name and internal id from local arr
|
||||
_bl_display_name="$BLOCKLIST_ID"
|
||||
_bl_internal_id=""
|
||||
if [[ -n "$_bl_url" ]] && [[ -n "$_bl_key" ]]; then
|
||||
_bl_lookup=$(_lookup_local_item "$_bl_url" "$_bl_key" "$_bl_ver" "$_bl_ep" \
|
||||
"$_bl_id_field" "$_bl_id_type" "$_bl_name_field" "$BLOCKLIST_ID")
|
||||
if [[ -n "$_bl_lookup" ]]; then
|
||||
IFS=$'\t' read -r _bl_internal_id _bl_display_name <<< "$_bl_lookup"
|
||||
fi
|
||||
fi
|
||||
|
||||
_blocklist_add "$BLOCKLIST_ARR" "$BLOCKLIST_ID" "$_bl_display_name" "${BLOCKLIST_REASON:-manually excluded}"
|
||||
echo " Blocklisted [$BLOCKLIST_ARR] $_bl_display_name ($BLOCKLIST_ID)"
|
||||
|
||||
# Remove from local arr (deleteFiles=false — arr_cleanup handles file removal)
|
||||
if [[ -n "$_bl_internal_id" ]]; then
|
||||
_bl_http=$(_delete_local_item "$_bl_url" "$_bl_key" "$_bl_ver" "$_bl_ep" "$_bl_internal_id")
|
||||
if [[ "$_bl_http" == "200" ]]; then
|
||||
log "Removed from local ${BLOCKLIST_ARR^}: $_bl_display_name"
|
||||
else
|
||||
warn "Failed to remove from local ${BLOCKLIST_ARR^} (HTTP ${_bl_http:-no response}) — remove manually via UI"
|
||||
fi
|
||||
else
|
||||
log "Not found in local ${BLOCKLIST_ARR^} — already removed or not tracked locally"
|
||||
fi
|
||||
|
||||
# Remove from all remote arrs
|
||||
for _bl_node_id in "${REMOTE_NODES[@]}"; do
|
||||
_bl_node_name="${!_bl_node_id}"
|
||||
_bl_result=$(_delete_remote_item "$_bl_node_id" "$_bl_port" "$_bl_ver" \
|
||||
"$BLOCKLIST_ARR" "$_bl_ep" "$_bl_id_field" "$_bl_id_type" "$BLOCKLIST_ID")
|
||||
case "$_bl_result" in
|
||||
200) log "Removed from $_bl_node_name ${BLOCKLIST_ARR^}: $_bl_display_name" ;;
|
||||
not_found) log "Not found on $_bl_node_name ${BLOCKLIST_ARR^} — already removed or not tracked" ;;
|
||||
*) warn "Failed to remove from $_bl_node_name ${BLOCKLIST_ARR^} (${_bl_result:-SSH error}) — remove manually via UI" ;;
|
||||
esac
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo " Files are now orphans on all nodes — arr_cleanup.sh will remove them on next run"
|
||||
exit 0
|
||||
;;
|
||||
remove)
|
||||
if [[ -z "$BLOCKLIST_ARR" ]] || [[ -z "$BLOCKLIST_ID" ]]; then
|
||||
error "Usage: arr_sync.sh --blocklist-remove <arr_type> <stable_id>"
|
||||
exit 1
|
||||
fi
|
||||
_blocklist_remove "$BLOCKLIST_ARR" "$BLOCKLIST_ID"
|
||||
echo " Removed [$BLOCKLIST_ARR] $BLOCKLIST_ID from blocklist"
|
||||
exit 0
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ── STATUS ────────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY ARR SYNC STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_HOST Remote nodes: ${REMOTE_NODES[*]}"
|
||||
echo "$ICON_GEAR Appdata base: $DOCKER_APPDATA_BASE"
|
||||
echo "$ICON_GEAR Blocklist: $ARR_SYNC_BLOCKLIST"
|
||||
echo "$ICON_GEAR Connect timeout: ${ARR_SYNC_CONNECT_TIMEOUT}s"
|
||||
echo "$ICON_GEAR API timeout: ${ARR_SYNC_API_TIMEOUT}s"
|
||||
echo ""
|
||||
echo " Arr Port URL"
|
||||
for _arr in "${ARR_TYPES[@]}"; do
|
||||
local _url="" _key=""
|
||||
case "$_arr" in
|
||||
lidarr) _url="${LIDARR_URL:-not configured}" ;;
|
||||
sonarr) _url="${SONARR_URL:-not configured}" ;;
|
||||
radarr) _url="${RADARR_URL:-not configured}" ;;
|
||||
esac
|
||||
printf ' %-8s %-6s %s\n' "$_arr" "${_PORT[$_arr]}" "$_url"
|
||||
done
|
||||
unset _arr _url
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ── REMOTE HELPERS ────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
_resolve_node_ip() {
|
||||
local node_id="$1"
|
||||
resolve_tailscale_ip "${!node_id}"
|
||||
}
|
||||
|
||||
# Check if arr is reachable on remote node
|
||||
# Args: node_id node_ip port api_ver arr_type
|
||||
_remote_arr_up() {
|
||||
local node_id="$1" node_ip="$2" port="$3" api_ver="$4" arr_type="$5"
|
||||
local _kvar="${node_id}_${arr_type^^}_API_KEY"; local cached_key="${!_kvar:-}"
|
||||
if [[ -n "$cached_key" ]]; then
|
||||
curl -sf --max-time 5 \
|
||||
-H "X-Api-Key: $cached_key" \
|
||||
"http://${node_ip}:${port}/api/${api_ver}/system/status" >/dev/null 2>/dev/null
|
||||
return
|
||||
fi
|
||||
local config_xml="${DOCKER_APPDATA_BASE}/${arr_type^}/config.xml"
|
||||
ssh -i "$SSH_KEY" -o ConnectTimeout="$ARR_SYNC_CONNECT_TIMEOUT" \
|
||||
root@"$node_ip" \
|
||||
"KEY=\$(grep -oP '(?<=<ApiKey>)[^<]+' '${config_xml}' 2>/dev/null)
|
||||
[[ -z \"\$KEY\" ]] && exit 1
|
||||
curl -sf --max-time 5 -H \"X-Api-Key: \$KEY\" \
|
||||
'http://localhost:${port}/api/${api_ver}/system/status' >/dev/null" 2>/dev/null
|
||||
}
|
||||
|
||||
# Fetch full library from remote arr — returns raw JSON array
|
||||
# Args: node_id node_ip port api_ver arr_type endpoint
|
||||
_remote_library() {
|
||||
local node_id="$1" node_ip="$2" port="$3" api_ver="$4" arr_type="$5" endpoint="$6"
|
||||
local _kvar="${node_id}_${arr_type^^}_API_KEY"; local cached_key="${!_kvar:-}"
|
||||
if [[ -n "$cached_key" ]]; then
|
||||
curl -sf --max-time "$ARR_SYNC_API_TIMEOUT" \
|
||||
-H "X-Api-Key: $cached_key" \
|
||||
"http://${node_ip}:${port}/api/${api_ver}/${endpoint}" 2>/dev/null
|
||||
return
|
||||
fi
|
||||
local config_xml="${DOCKER_APPDATA_BASE}/${arr_type^}/config.xml"
|
||||
ssh -i "$SSH_KEY" -o ConnectTimeout="$ARR_SYNC_CONNECT_TIMEOUT" \
|
||||
root@"$node_ip" \
|
||||
"KEY=\$(grep -oP '(?<=<ApiKey>)[^<]+' '${config_xml}' 2>/dev/null)
|
||||
[[ -z \"\$KEY\" ]] && exit 1
|
||||
curl -sf --max-time ${ARR_SYNC_API_TIMEOUT} \
|
||||
-H \"X-Api-Key: \$KEY\" \
|
||||
'http://localhost:${port}/api/${api_ver}/${endpoint}'" 2>/dev/null
|
||||
}
|
||||
|
||||
# Fetch remote arr defaults: qualityProfileId, rootFolderPath, metadataProfileId (Lidarr)
|
||||
# Args: node_id node_ip port api_ver arr_type
|
||||
_remote_defaults() {
|
||||
local node_id="$1" node_ip="$2" port="$3" api_ver="$4" arr_type="$5"
|
||||
local _kvar="${node_id}_${arr_type^^}_API_KEY"; local cached_key="${!_kvar:-}"
|
||||
if [[ -n "$cached_key" ]]; then
|
||||
local base_url="http://${node_ip}:${port}/api/${api_ver}"
|
||||
local qp rf mp
|
||||
qp=$(curl -sf -H "X-Api-Key: $cached_key" "${base_url}/qualityprofile" 2>/dev/null | jq '.[0].id // 1')
|
||||
rf=$(curl -sf -H "X-Api-Key: $cached_key" "${base_url}/rootfolder" 2>/dev/null | jq -r '.[0].path // ""')
|
||||
[[ -z "$qp" ]] && return 1
|
||||
if [[ "$arr_type" == "lidarr" ]]; then
|
||||
mp=$(curl -sf -H "X-Api-Key: $cached_key" "${base_url}/metadataprofile" 2>/dev/null | \
|
||||
jq 'map(select(.name == "Standard")) | .[0].id // .[0].id // 1')
|
||||
jq -n --argjson qp "$qp" --arg rf "$rf" --argjson mp "$mp" \
|
||||
'{qualityProfileId: $qp, rootFolderPath: $rf, metadataProfileId: $mp}'
|
||||
else
|
||||
jq -n --argjson qp "$qp" --arg rf "$rf" \
|
||||
'{qualityProfileId: $qp, rootFolderPath: $rf}'
|
||||
fi
|
||||
return
|
||||
fi
|
||||
local config_xml="${DOCKER_APPDATA_BASE}/${arr_type^}/config.xml"
|
||||
local meta_field=""
|
||||
[[ "$arr_type" == "lidarr" ]] && \
|
||||
meta_field=', metadataProfileId: ($mp | map(select(.name == "Standard")) | .[0].id // .[0].id // 1)'
|
||||
ssh -i "$SSH_KEY" -o ConnectTimeout="$ARR_SYNC_CONNECT_TIMEOUT" \
|
||||
root@"$node_ip" bash <<REMOTE 2>/dev/null
|
||||
KEY=\$(grep -oP '(?<=<ApiKey>)[^<]+' '${config_xml}' 2>/dev/null)
|
||||
[[ -z "\$KEY" ]] && exit 1
|
||||
QP=\$(curl -sf -H "X-Api-Key: \$KEY" "http://localhost:${port}/api/${api_ver}/qualityprofile")
|
||||
RF=\$(curl -sf -H "X-Api-Key: \$KEY" "http://localhost:${port}/api/${api_ver}/rootfolder")
|
||||
MP=\$(curl -sf -H "X-Api-Key: \$KEY" "http://localhost:${port}/api/${api_ver}/metadataprofile" 2>/dev/null || echo '[]')
|
||||
jq -n --argjson qp "\$QP" --argjson rf "\$RF" --argjson mp "\$MP" \
|
||||
'{qualityProfileId: (\$qp | .[0].id // 1), rootFolderPath: (\$rf | .[0].path // "")${meta_field}}'
|
||||
REMOTE
|
||||
}
|
||||
|
||||
# Add item to remote arr — payload is base64-encoded to avoid quoting issues
|
||||
# Args: node_id node_ip port api_ver arr_type endpoint encoded_payload
|
||||
_remote_add() {
|
||||
local node_id="$1" node_ip="$2" port="$3" api_ver="$4" arr_type="$5" endpoint="$6"
|
||||
local encoded="$7"
|
||||
local _kvar="${node_id}_${arr_type^^}_API_KEY"; local cached_key="${!_kvar:-}"
|
||||
if [[ -n "$cached_key" ]]; then
|
||||
local body
|
||||
body=$(printf '%s' "$encoded" | base64 -d)
|
||||
curl -sf -o /dev/null -w '%{http_code}' -X POST \
|
||||
-H "X-Api-Key: $cached_key" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "$body" \
|
||||
"http://${node_ip}:${port}/api/${api_ver}/${endpoint}" 2>/dev/null
|
||||
return
|
||||
fi
|
||||
local config_xml="${DOCKER_APPDATA_BASE}/${arr_type^}/config.xml"
|
||||
ssh -i "$SSH_KEY" -o ConnectTimeout="$ARR_SYNC_CONNECT_TIMEOUT" \
|
||||
root@"$node_ip" bash <<REMOTE 2>/dev/null
|
||||
KEY=\$(grep -oP '(?<=<ApiKey>)[^<]+' '${config_xml}' 2>/dev/null)
|
||||
[[ -z "\$KEY" ]] && exit 1
|
||||
BODY=\$(printf '%s' '${encoded}' | base64 -d)
|
||||
curl -sf -o /dev/null -w '%{http_code}' -X POST \
|
||||
-H "X-Api-Key: \$KEY" \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d "\$BODY" \
|
||||
"http://localhost:${port}/api/${api_ver}/${endpoint}"
|
||||
REMOTE
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ── LOCAL HELPERS ─────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
_local_library() {
|
||||
local url="$1" api_key="$2" api_ver="$3" endpoint="$4"
|
||||
curl -sf --max-time "$ARR_SYNC_API_TIMEOUT" \
|
||||
-H "X-Api-Key: $api_key" \
|
||||
"${url}/api/${api_ver}/${endpoint}" 2>/dev/null
|
||||
}
|
||||
|
||||
_local_defaults() {
|
||||
local url="$1" api_key="$2" api_ver="$3" arr_type="$4"
|
||||
local qp rf mp meta_field=""
|
||||
qp=$(curl -sf -H "X-Api-Key: $api_key" "${url}/api/${api_ver}/qualityprofile" | jq '.[0].id // 1')
|
||||
rf=$(curl -sf -H "X-Api-Key: $api_key" "${url}/api/${api_ver}/rootfolder" | jq -r '.[0].path // ""')
|
||||
if [[ "$arr_type" == "lidarr" ]]; then
|
||||
mp=$(curl -sf -H "X-Api-Key: $api_key" "${url}/api/${api_ver}/metadataprofile" | \
|
||||
jq 'map(select(.name == "Standard")) | .[0].id // .[0].id // 1')
|
||||
jq -n --argjson qp "$qp" --arg rf "$rf" --argjson mp "$mp" \
|
||||
'{qualityProfileId: $qp, rootFolderPath: $rf, metadataProfileId: $mp}'
|
||||
else
|
||||
jq -n --argjson qp "$qp" --arg rf "$rf" \
|
||||
'{qualityProfileId: $qp, rootFolderPath: $rf}'
|
||||
fi
|
||||
}
|
||||
|
||||
_local_add() {
|
||||
local url="$1" api_key="$2" api_ver="$3" endpoint="$4" payload="$5"
|
||||
curl -sf -o /dev/null -w '%{http_code}' -X POST \
|
||||
-H "X-Api-Key: $api_key" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "$payload" \
|
||||
"${url}/api/${api_ver}/${endpoint}" 2>/dev/null
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ── PAYLOAD BUILDER ───────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Builds the minimal POST body to add an item to an arr.
|
||||
# Uses the TARGET node's own defaults — never copies source node settings.
|
||||
|
||||
_build_payload() {
|
||||
local arr_type="$1" stable_id="$2" display_name="$3" monitored="$4" defaults_json="$5"
|
||||
case "$arr_type" in
|
||||
lidarr)
|
||||
jq -n \
|
||||
--arg id "$stable_id" \
|
||||
--arg nm "$display_name" \
|
||||
--argjson mn "$monitored" \
|
||||
--argjson df "$defaults_json" \
|
||||
'{
|
||||
foreignArtistId: $id,
|
||||
artistName: $nm,
|
||||
monitored: $mn,
|
||||
qualityProfileId: $df.qualityProfileId,
|
||||
metadataProfileId: ($df.metadataProfileId // 1),
|
||||
rootFolderPath: $df.rootFolderPath,
|
||||
addOptions: {monitor: "all", searchForMissingAlbums: false}
|
||||
}'
|
||||
;;
|
||||
sonarr)
|
||||
jq -n \
|
||||
--argjson id "$stable_id" \
|
||||
--arg nm "$display_name" \
|
||||
--argjson mn "$monitored" \
|
||||
--argjson df "$defaults_json" \
|
||||
'{
|
||||
tvdbId: $id,
|
||||
title: $nm,
|
||||
monitored: $mn,
|
||||
qualityProfileId: $df.qualityProfileId,
|
||||
rootFolderPath: $df.rootFolderPath,
|
||||
seasons: [],
|
||||
addOptions: {searchForMissingEpisodes: false, monitor: "all"}
|
||||
}'
|
||||
;;
|
||||
radarr)
|
||||
jq -n \
|
||||
--argjson id "$stable_id" \
|
||||
--arg nm "$display_name" \
|
||||
--argjson mn "$monitored" \
|
||||
--argjson df "$defaults_json" \
|
||||
'{
|
||||
tmdbId: $id,
|
||||
title: $nm,
|
||||
monitored: $mn,
|
||||
qualityProfileId: $df.qualityProfileId,
|
||||
rootFolderPath: $df.rootFolderPath,
|
||||
addOptions: {searchForMovie: false}
|
||||
}'
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ── CORE SYNC — one arr type across all remote nodes ──────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
_sync_arr() {
|
||||
local arr_type="$1"
|
||||
local port="${_PORT[$arr_type]}"
|
||||
local api_ver="${_VER[$arr_type]}"
|
||||
local endpoint="${_EP[$arr_type]}"
|
||||
local id_field="${_ID[$arr_type]}"
|
||||
local name_field="${_NAME[$arr_type]}"
|
||||
local id_type="${_ID_TYPE[$arr_type]}"
|
||||
|
||||
# Resolve local credentials
|
||||
local local_url local_key
|
||||
case "$arr_type" in
|
||||
lidarr) local_url="${LIDARR_URL:-}"; local_key="${LIDARR_API_KEY:-}" ;;
|
||||
sonarr) local_url="${SONARR_URL:-}"; local_key="${SONARR_API_KEY:-}" ;;
|
||||
radarr) local_url="${RADARR_URL:-}"; local_key="${RADARR_API_KEY:-}" ;;
|
||||
esac
|
||||
|
||||
if [[ -z "$local_url" ]] || [[ -z "$local_key" ]]; then
|
||||
log "${arr_type^}: not configured on $MY_ID — skipping"
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "━━━ ${arr_type^} ━━━"
|
||||
|
||||
# ── Fetch local library ────────────────────────────────────────────────────────────────────
|
||||
local local_json
|
||||
local_json=$(_local_library "$local_url" "$local_key" "$api_ver" "$endpoint")
|
||||
if [[ -z "$local_json" ]] || ! echo "$local_json" | jq -e '.' >/dev/null 2>&1; then
|
||||
warn "${arr_type^}: could not fetch local library — skipping"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Build local ID map: stable_id → "display_name|monitored"
|
||||
declare -A local_ids
|
||||
local local_count=0
|
||||
local _jq_id
|
||||
[[ "$id_type" == "string" ]] && _jq_id=".${id_field}" || _jq_id="(.${id_field} | tostring)"
|
||||
while IFS=$'\t' read -r stable_id display_name monitored; do
|
||||
[[ -z "$stable_id" ]] && continue
|
||||
local_ids["$stable_id"]="${display_name}|${monitored}"
|
||||
(( local_count++ ))
|
||||
done < <(echo "$local_json" | jq -r \
|
||||
".[] | [${_jq_id}, .${name_field}, (.monitored | tostring)] | @tsv" 2>/dev/null)
|
||||
unset _jq_id
|
||||
|
||||
log "${arr_type^}: $local_count items in local library"
|
||||
|
||||
local total_added_local=0 total_added_remote=0 total_skipped=0
|
||||
|
||||
# ── Sync with each remote node ─────────────────────────────────────────────────────────────
|
||||
for node_id in "${REMOTE_NODES[@]}"; do
|
||||
local node_name="${!node_id}"
|
||||
log "${arr_type^}: syncing with $node_name..."
|
||||
|
||||
local node_ip
|
||||
node_ip=$(_resolve_node_ip "$node_id") || {
|
||||
warn "${arr_type^}: cannot resolve Tailscale IP for $node_name — skipping"
|
||||
continue
|
||||
}
|
||||
|
||||
if ! _remote_arr_up "$node_id" "$node_ip" "$port" "$api_ver" "$arr_type"; then
|
||||
log "${arr_type^}: not reachable on $node_name — skipping"
|
||||
continue
|
||||
fi
|
||||
|
||||
local remote_json
|
||||
remote_json=$(_remote_library "$node_id" "$node_ip" "$port" "$api_ver" "$arr_type" "$endpoint")
|
||||
if [[ -z "$remote_json" ]] || ! echo "$remote_json" | jq -e '.' >/dev/null 2>&1; then
|
||||
warn "${arr_type^}: could not fetch library from $node_name — skipping"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Build remote ID map
|
||||
declare -A remote_ids
|
||||
local remote_count=0
|
||||
local _jq_id
|
||||
[[ "$id_type" == "string" ]] && _jq_id=".${id_field}" || _jq_id="(.${id_field} | tostring)"
|
||||
while IFS=$'\t' read -r stable_id display_name monitored; do
|
||||
[[ -z "$stable_id" ]] && continue
|
||||
remote_ids["$stable_id"]="${display_name}|${monitored}"
|
||||
(( remote_count++ ))
|
||||
done < <(echo "$remote_json" | jq -r \
|
||||
".[] | [${_jq_id}, .${name_field}, (.monitored | tostring)] | @tsv" 2>/dev/null)
|
||||
unset _jq_id
|
||||
|
||||
log "${arr_type^}: $remote_count items on $node_name"
|
||||
|
||||
# ── Remote → Local: items on remote not in local ───────────────────────────────────────
|
||||
local to_add_local=()
|
||||
for stable_id in "${!remote_ids[@]}"; do
|
||||
[[ -n "${local_ids[$stable_id]:-}" ]] && continue
|
||||
if _is_blocklisted "$arr_type" "$stable_id"; then
|
||||
log "BLOCKLISTED [$arr_type] $stable_id — skipping"
|
||||
(( total_skipped++ ))
|
||||
continue
|
||||
fi
|
||||
to_add_local+=("$stable_id")
|
||||
done
|
||||
|
||||
if [[ "${#to_add_local[@]}" -gt 0 ]]; then
|
||||
local local_defs
|
||||
local_defs=$(_local_defaults "$local_url" "$local_key" "$api_ver" "$arr_type")
|
||||
if [[ -z "$local_defs" ]]; then
|
||||
warn "${arr_type^}: could not fetch local defaults — skipping adds from $node_name"
|
||||
else
|
||||
for stable_id in "${to_add_local[@]}"; do
|
||||
IFS='|' read -r display_name _ <<< "${remote_ids[$stable_id]}"
|
||||
local payload
|
||||
payload=$(_build_payload "$arr_type" "$stable_id" "$display_name" \
|
||||
"true" "$local_defs")
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
log "DRY RUN: would add to local ${arr_type^}: $display_name ($stable_id)"
|
||||
(( total_added_local++ ))
|
||||
else
|
||||
local http_code
|
||||
http_code=$(_local_add "$local_url" "$local_key" "$api_ver" \
|
||||
"$endpoint" "$payload")
|
||||
if [[ "$http_code" == "201" ]] || [[ "$http_code" == "200" ]]; then
|
||||
log "Added to local ${arr_type^}: $display_name"
|
||||
local_ids["$stable_id"]="${display_name}|true"
|
||||
(( total_added_local++ ))
|
||||
else
|
||||
warn "Failed to add to local ${arr_type^}: $display_name (HTTP $http_code)"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Local → Remote: items on local not on remote ───────────────────────────────────────
|
||||
local to_add_remote=()
|
||||
for stable_id in "${!local_ids[@]}"; do
|
||||
[[ -n "${remote_ids[$stable_id]:-}" ]] && continue
|
||||
if _is_blocklisted "$arr_type" "$stable_id"; then
|
||||
(( total_skipped++ ))
|
||||
continue
|
||||
fi
|
||||
to_add_remote+=("$stable_id")
|
||||
done
|
||||
|
||||
if [[ "${#to_add_remote[@]}" -gt 0 ]]; then
|
||||
local remote_defs
|
||||
remote_defs=$(_remote_defaults "$node_id" "$node_ip" "$port" "$api_ver" "$arr_type")
|
||||
if [[ -z "$remote_defs" ]]; then
|
||||
warn "${arr_type^}: could not fetch defaults from $node_name — skipping remote adds"
|
||||
else
|
||||
for stable_id in "${to_add_remote[@]}"; do
|
||||
IFS='|' read -r display_name _ <<< "${local_ids[$stable_id]}"
|
||||
local payload
|
||||
payload=$(_build_payload "$arr_type" "$stable_id" "$display_name" \
|
||||
"true" "$remote_defs")
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
log "DRY RUN: would add to $node_name ${arr_type^}: $display_name ($stable_id)"
|
||||
(( total_added_remote++ ))
|
||||
else
|
||||
local encoded http_code
|
||||
encoded=$(printf '%s' "$payload" | base64 -w0)
|
||||
http_code=$(_remote_add "$node_id" "$node_ip" "$port" "$api_ver" \
|
||||
"$arr_type" "$endpoint" "$encoded")
|
||||
if [[ "$http_code" == "201" ]] || [[ "$http_code" == "200" ]]; then
|
||||
log "Added to $node_name ${arr_type^}: $display_name"
|
||||
(( total_added_remote++ ))
|
||||
else
|
||||
warn "Failed to add to $node_name ${arr_type^}: $display_name (HTTP $http_code)"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
fi
|
||||
fi
|
||||
|
||||
log " $node_name: +${#to_add_local[@]} local | +${#to_add_remote[@]} remote | $total_skipped blocklisted"
|
||||
|
||||
unset remote_ids
|
||||
declare -A remote_ids
|
||||
done
|
||||
|
||||
echo " Total added to local: $total_added_local | to remotes: $total_added_remote | blocklisted: $total_skipped"
|
||||
unset local_ids
|
||||
declare -A local_ids
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Main ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Loading Blocklist ━━━"
|
||||
_load_blocklist
|
||||
|
||||
START=$(date +%s)
|
||||
|
||||
for arr_type in "${ARR_TYPES[@]}"; do
|
||||
_sync_arr "$arr_type"
|
||||
done
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY ARR SYNC SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Node: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_SYNC Peers: ${REMOTE_NODES[*]}"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes were made"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
exit 0
|
||||
+960
@@ -0,0 +1,960 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ================================= ARR Sync ===================================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Full-mesh arr library sync across all nodes — Lidarr, Sonarr, and Radarr.
|
||||
# Every node syncs with every other, union model, no hierarchy. Run before
|
||||
# rsync in the weekly sync window: once arrs agree on what to track, rsync
|
||||
# spreads the actual files.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Full mesh: every node syncs with every other — no primary, no hierarchy.
|
||||
# Union model: if any node tracks an item, all nodes get it (unless blocklisted).
|
||||
# Convergence: any node can add content; after one full cycle all nodes agree.
|
||||
# Upgrade-aware: a file upgrade on one node → arr tracks new path → rsync spreads
|
||||
# it → arr_cleanup removes old path on all nodes (arr no longer tracks it).
|
||||
#
|
||||
# Node discovery: reads HOST* vars from master.conf. Add HOST3= and it joins the
|
||||
# sync automatically — no script changes needed for a new node.
|
||||
#
|
||||
# Graceful skip: arr not configured locally → skip cleanly. Arr not reachable on
|
||||
# a remote → skip that node for that arr type, continue with others.
|
||||
#
|
||||
# What gets synced — library items keyed on stable external IDs:
|
||||
# Lidarr — MusicBrainz artist ID (foreignArtistId)
|
||||
# Sonarr — TVDB series ID (tvdbId)
|
||||
# Radarr — TMDB movie ID (tmdbId)
|
||||
# When adding to a remote, that node's own quality profile, metadata profile,
|
||||
# and root folder path are used — settings are never copied from source.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# DESIGN PRINCIPLES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Remote API Access — Cache-First, SSH Fallback
|
||||
# If conf_sync.sh has populated /tmp/.vv/config/cached/.confs/ and
|
||||
# load_config.sh has sourced it, HOST*_<ARR>_API_KEY vars are available
|
||||
# in the environment. Remote functions use them to call the arr API
|
||||
# directly over Tailscale (no SSH, no remote shell). If the cached key
|
||||
# is absent (first boot, cache not yet populated) the functions fall back
|
||||
# to SSHing in and reading the key from config.xml on the remote node.
|
||||
#
|
||||
# Blocklist TSV
|
||||
# ARR_SYNC_BLOCKLIST in DATA_DIR tombstones IDs that must never be re-added
|
||||
# anywhere. Read from ALL nodes via SSH at run start — immediate effect with
|
||||
# no rsync delay.
|
||||
#
|
||||
# --blocklist-add does three things atomically:
|
||||
# 1. Writes the TSV tombstone entry (prevents future re-adds by arr_sync)
|
||||
# 2. Deletes the item from the local arr API (deleteFiles=false)
|
||||
# 3. SSHes each remote node and deletes from their arr API (deleteFiles=false)
|
||||
# Files become orphans on all nodes — arr_cleanup removes them on next run.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# acquire_lock — prevents two sync instances running simultaneously
|
||||
# ARR_SYNC_ENABLED — global gate, exits cleanly when false
|
||||
# SSH connect timeout — ARR_SYNC_CONNECT_TIMEOUT — does not hang on unreachable node
|
||||
# API call timeout — ARR_SYNC_API_TIMEOUT — does not hang on slow arr
|
||||
# Graceful skip — unreachable node/arr → skip and continue, never abort
|
||||
# Blocklist gate — item in blocklist → never added to any node
|
||||
# Silent by default — only additions produce output, clean runs stay silent
|
||||
#
|
||||
# ==============================================================================================
|
||||
# STATE FILES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# ARR_SYNC_BLOCKLIST — TSV file in DATA_DIR (default: DATA_DIR/arr_sync_blocklist.tsv)
|
||||
# Columns: arr_type, id, reason, date_added
|
||||
# Read from all nodes via SSH at the start of each run.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# host*.conf
|
||||
#
|
||||
# HOST*_LIDARR_URL / HOST*_LIDARR_API_KEY — local Lidarr (aliased by detect_hosts)
|
||||
# HOST*_SONARR_URL / HOST*_SONARR_API_KEY — local Sonarr
|
||||
# HOST*_RADARR_URL / HOST*_RADARR_API_KEY — local Radarr
|
||||
#
|
||||
# master.conf
|
||||
#
|
||||
# ARR_SYNC_ENABLED — global on/off toggle (default: true)
|
||||
# ARR_SYNC_BLOCKLIST — path to TSV blocklist file
|
||||
# ARR_SYNC_CONNECT_TIMEOUT — SSH connect timeout in seconds (default: 10)
|
||||
# ARR_SYNC_API_TIMEOUT — curl API call timeout in seconds (default: 60)
|
||||
# DOCKER_APPDATA_BASE — base path for arr appdata dirs (default: /mnt/user/appdata)
|
||||
# ARR_SYNC_LIDARR_PORT — Lidarr port on all nodes (default: 8686)
|
||||
# ARR_SYNC_SONARR_PORT — Sonarr port on all nodes (default: 8989)
|
||||
# ARR_SYNC_RADARR_PORT — Radarr port on all nodes (default: 7878)
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# arr_sync.sh — sync all arr types, all nodes
|
||||
# arr_sync.sh --dry-run — preview only, no changes
|
||||
# arr_sync.sh --log — verbose output
|
||||
# arr_sync.sh --status — show config and exit
|
||||
#
|
||||
# Blocklist management:
|
||||
# arr_sync.sh --blocklist-add lidarr <mbid> "reason" — remove from all arrs + tombstone
|
||||
# arr_sync.sh --blocklist-add sonarr <tvdbId> "reason" — remove from all arrs + tombstone
|
||||
# arr_sync.sh --blocklist-add radarr <tmdbId> "reason" — remove from all arrs + tombstone
|
||||
# arr_sync.sh --blocklist-remove lidarr <id> — un-tombstone (does NOT re-add)
|
||||
# arr_sync.sh --blocklist-list — show all blocklisted IDs
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
# ── Blocklist flag pre-processing ─────────────────────────────────────────────────────────────
|
||||
# Handle before parse_args — these are action flags, not standard options
|
||||
BLOCKLIST_ACTION=""
|
||||
BLOCKLIST_ARR=""
|
||||
BLOCKLIST_ID=""
|
||||
BLOCKLIST_REASON=""
|
||||
FILTERED_ARGS=()
|
||||
_skip_next=false
|
||||
for _arg in "$@"; do
|
||||
if [[ "$_skip_next" == true ]]; then _skip_next=false; continue; fi
|
||||
case "$_arg" in
|
||||
--blocklist-add) BLOCKLIST_ACTION="add" ;;
|
||||
--blocklist-remove) BLOCKLIST_ACTION="remove" ;;
|
||||
--blocklist-list) BLOCKLIST_ACTION="list" ;;
|
||||
*) FILTERED_ARGS+=("$_arg") ;;
|
||||
esac
|
||||
done
|
||||
unset _arg _skip_next
|
||||
|
||||
parse_args "${FILTERED_ARGS[@]}"
|
||||
|
||||
# Consume blocklist positional args from remaining FILTERED_ARGS
|
||||
# --blocklist-add lidarr <id> "reason"
|
||||
# --blocklist-remove lidarr <id>
|
||||
if [[ -n "$BLOCKLIST_ACTION" ]] && [[ "$BLOCKLIST_ACTION" != "list" ]]; then
|
||||
for _a in "${FILTERED_ARGS[@]}"; do
|
||||
[[ "$_a" == --* ]] && continue
|
||||
if [[ -z "$BLOCKLIST_ARR" ]]; then BLOCKLIST_ARR="$_a"
|
||||
elif [[ -z "$BLOCKLIST_ID" ]]; then BLOCKLIST_ID="$_a"
|
||||
elif [[ -z "$BLOCKLIST_REASON" ]]; then BLOCKLIST_REASON="$_a"
|
||||
fi
|
||||
done
|
||||
unset _a
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
acquire_lock
|
||||
|
||||
for _tool in curl jq; do
|
||||
if ! command -v "$_tool" >/dev/null 2>&1; then
|
||||
error "$_tool not found — required for arr API calls"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
unset _tool
|
||||
|
||||
detect_hosts
|
||||
|
||||
# ── Runtime config with defaults ──────────────────────────────────────────────────────────────
|
||||
ARR_SYNC_ENABLED="${ARR_SYNC_ENABLED:-true}"
|
||||
ARR_SYNC_BLOCKLIST="${ARR_SYNC_BLOCKLIST:-${DATA_DIR}/arr_sync_blocklist.tsv}"
|
||||
ARR_SYNC_CONNECT_TIMEOUT="${ARR_SYNC_CONNECT_TIMEOUT:-10}"
|
||||
ARR_SYNC_API_TIMEOUT="${ARR_SYNC_API_TIMEOUT:-60}"
|
||||
DOCKER_APPDATA_BASE="${DOCKER_APPDATA_BASE:-/mnt/user/appdata}"
|
||||
ARR_SYNC_LIDARR_PORT="${ARR_SYNC_LIDARR_PORT:-8686}"
|
||||
ARR_SYNC_SONARR_PORT="${ARR_SYNC_SONARR_PORT:-8989}"
|
||||
ARR_SYNC_RADARR_PORT="${ARR_SYNC_RADARR_PORT:-7878}"
|
||||
|
||||
if [[ "$ARR_SYNC_ENABLED" != "true" ]]; then
|
||||
log "ARR_SYNC_ENABLED=false — exiting"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
log "$ICON_GEAR Config: api-timeout=${ARR_SYNC_API_TIMEOUT}s connect-timeout=${ARR_SYNC_CONNECT_TIMEOUT}s blocklist=${ARR_SYNC_BLOCKLIST}"
|
||||
log "$ICON_GEAR Ports: lidarr=${ARR_SYNC_LIDARR_PORT} sonarr=${ARR_SYNC_SONARR_PORT} radarr=${ARR_SYNC_RADARR_PORT}"
|
||||
|
||||
# ── Arr type definitions ───────────────────────────────────────────────────────────────────────
|
||||
# Each arr type maps to its port, API version, endpoint, stable ID field, and display name field
|
||||
declare -A _PORT=([lidarr]="$ARR_SYNC_LIDARR_PORT" [sonarr]="$ARR_SYNC_SONARR_PORT" [radarr]="$ARR_SYNC_RADARR_PORT")
|
||||
declare -A _VER=( [lidarr]="v1" [sonarr]="v3" [radarr]="v3")
|
||||
declare -A _EP=( [lidarr]="artist" [sonarr]="series" [radarr]="movie")
|
||||
declare -A _ID=( [lidarr]="foreignArtistId" [sonarr]="tvdbId" [radarr]="tmdbId")
|
||||
declare -A _NAME=([lidarr]="artistName" [sonarr]="title" [radarr]="title")
|
||||
# ID type: "string" for MusicBrainz UUID, "int" for TVDB/TMDB numeric IDs
|
||||
declare -A _ID_TYPE=([lidarr]="string" [sonarr]="int" [radarr]="int")
|
||||
ARR_TYPES=(lidarr sonarr radarr)
|
||||
|
||||
# ── Remote node discovery ──────────────────────────────────────────────────────────────────────
|
||||
REMOTE_NODES=()
|
||||
for _hv in HOST1 HOST2 HOST3 HOST4 HOST5 HOST6 HOST7 HOST8; do
|
||||
[[ "$_hv" == "$MY_ID" ]] && continue
|
||||
[[ -z "${!_hv:-}" ]] && continue
|
||||
REMOTE_NODES+=("$_hv")
|
||||
done
|
||||
unset _hv
|
||||
|
||||
if [[ "${#REMOTE_NODES[@]}" -eq 0 ]]; then
|
||||
warn "No remote nodes defined in master.conf — nothing to sync"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo " Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo " Remote nodes: ${REMOTE_NODES[*]}"
|
||||
echo " Blocklist: $ARR_SYNC_BLOCKLIST"
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
|
||||
# ==============================================================================================
|
||||
# ── BLOCKLIST ─────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
declare -A BLOCKLIST_MAP # key: "arr_type:stable_id" → display name
|
||||
|
||||
_load_blocklist() {
|
||||
BLOCKLIST_MAP=()
|
||||
local count=0
|
||||
|
||||
_parse_blocklist_lines() {
|
||||
while IFS=$'\t' read -r arr_type stable_id display_name rest; do
|
||||
[[ -z "$arr_type" ]] || [[ "$arr_type" == \#* ]] && continue
|
||||
BLOCKLIST_MAP["${arr_type}:${stable_id}"]="$display_name"
|
||||
(( count++ ))
|
||||
done
|
||||
}
|
||||
|
||||
# Local blocklist
|
||||
[[ -f "$ARR_SYNC_BLOCKLIST" ]] && _parse_blocklist_lines < "$ARR_SYNC_BLOCKLIST"
|
||||
|
||||
# Remote blocklists — read via SSH so tombstones are effective immediately
|
||||
for _node_id in "${REMOTE_NODES[@]}"; do
|
||||
local _node_ip
|
||||
_node_ip=$(_resolve_node_ip "$_node_id") || continue
|
||||
local _remote_lines
|
||||
_remote_lines=$(ssh -i "$SSH_KEY" -o ConnectTimeout="$ARR_SYNC_CONNECT_TIMEOUT" \
|
||||
root@"$_node_ip" "cat '$ARR_SYNC_BLOCKLIST' 2>/dev/null" 2>/dev/null) || continue
|
||||
_parse_blocklist_lines <<< "$_remote_lines"
|
||||
done
|
||||
unset _node_id _node_ip _remote_lines
|
||||
|
||||
log "Loaded blocklist: $count entries from $(( ${#REMOTE_NODES[@]} + 1 )) nodes"
|
||||
unset -f _parse_blocklist_lines
|
||||
}
|
||||
|
||||
_is_blocklisted() {
|
||||
[[ -n "${BLOCKLIST_MAP["${1}:${2}"]:-}" ]]
|
||||
}
|
||||
|
||||
_blocklist_add() {
|
||||
local arr_type="$1" stable_id="$2" display_name="$3" reason="${4:-manually excluded}"
|
||||
mkdir -p "$(dirname "$ARR_SYNC_BLOCKLIST")"
|
||||
if ! grep -qP "^${arr_type}\t${stable_id}\t" "$ARR_SYNC_BLOCKLIST" 2>/dev/null; then
|
||||
printf '%s\t%s\t%s\t%s\t%s\t%s\n' \
|
||||
"$arr_type" "$stable_id" "$display_name" \
|
||||
"$MY_ID" "$(date -Iseconds)" "$reason" \
|
||||
>> "$ARR_SYNC_BLOCKLIST"
|
||||
BLOCKLIST_MAP["${arr_type}:${stable_id}"]="$display_name"
|
||||
log "Blocklisted: [$arr_type] $display_name ($stable_id)"
|
||||
else
|
||||
log "Already blocklisted: [$arr_type] $stable_id"
|
||||
fi
|
||||
}
|
||||
|
||||
_blocklist_remove() {
|
||||
local arr_type="$1" stable_id="$2"
|
||||
if [[ -f "$ARR_SYNC_BLOCKLIST" ]]; then
|
||||
local tmp
|
||||
tmp=$(mktemp)
|
||||
grep -vP "^${arr_type}\t${stable_id}\t" "$ARR_SYNC_BLOCKLIST" > "$tmp" && \
|
||||
mv "$tmp" "$ARR_SYNC_BLOCKLIST" || rm -f "$tmp"
|
||||
unset "BLOCKLIST_MAP[${arr_type}:${stable_id}]"
|
||||
log "Removed from blocklist: [$arr_type] $stable_id"
|
||||
fi
|
||||
}
|
||||
|
||||
# Look up item in local arr by stable_id — returns "internal_id\tdisplay_name" or empty
|
||||
_lookup_local_item() {
|
||||
local url="$1" api_key="$2" api_ver="$3" endpoint="$4"
|
||||
local id_field="$5" id_type="$6" name_field="$7" stable_id="$8"
|
||||
local raw select_expr
|
||||
raw=$(curl -sf --max-time "$ARR_SYNC_API_TIMEOUT" \
|
||||
-H "X-Api-Key: $api_key" \
|
||||
"${url}/api/${api_ver}/${endpoint}" 2>/dev/null)
|
||||
[[ -z "$raw" ]] && return 1
|
||||
if [[ "$id_type" == "string" ]]; then
|
||||
select_expr=".[] | select(.${id_field} == \"${stable_id}\")"
|
||||
else
|
||||
select_expr=".[] | select(.${id_field} == ${stable_id})"
|
||||
fi
|
||||
echo "$raw" | jq -r "${select_expr} | [(.id | tostring), .${name_field}] | @tsv" 2>/dev/null | head -1
|
||||
}
|
||||
|
||||
# Delete item from local arr by internal integer id — returns HTTP status code
|
||||
_delete_local_item() {
|
||||
local url="$1" api_key="$2" api_ver="$3" endpoint="$4" internal_id="$5"
|
||||
curl -sf -o /dev/null -w '%{http_code}' -X DELETE \
|
||||
-H "X-Api-Key: $api_key" \
|
||||
"${url}/api/${api_ver}/${endpoint}/${internal_id}?deleteFiles=false" 2>/dev/null
|
||||
}
|
||||
|
||||
# Delete item from remote arr by stable_id.
|
||||
# Outputs: HTTP code on success | "not_found" if item absent | empty on failure.
|
||||
# deleteFiles=false — files become orphans for arr_cleanup to handle with its safety checks.
|
||||
# Args: node_id port api_ver arr_type endpoint id_field id_type stable_id
|
||||
_delete_remote_item() {
|
||||
local node_id="$1" port="$2" api_ver="$3" arr_type="$4" endpoint="$5"
|
||||
local id_field="$6" id_type="$7" stable_id="$8"
|
||||
local node_name="${!node_id}"
|
||||
local node_ip
|
||||
node_ip=$(resolve_tailscale_ip "$node_name") || return 1
|
||||
|
||||
local select_expr
|
||||
if [[ "$id_type" == "string" ]]; then
|
||||
select_expr=".[] | select(.${id_field} == \"${stable_id}\") | .id"
|
||||
else
|
||||
select_expr=".[] | select(.${id_field} == ${stable_id}) | .id"
|
||||
fi
|
||||
|
||||
local _kvar="${node_id}_${arr_type^^}_API_KEY"; local cached_key="${!_kvar:-}"
|
||||
if [[ -n "$cached_key" ]]; then
|
||||
local library internal_id
|
||||
library=$(curl -sf --max-time "$ARR_SYNC_API_TIMEOUT" \
|
||||
-H "X-Api-Key: $cached_key" \
|
||||
"http://${node_ip}:${port}/api/${api_ver}/${endpoint}" 2>/dev/null)
|
||||
[[ -z "$library" ]] && return 1
|
||||
internal_id=$(echo "$library" | jq -r "${select_expr}" 2>/dev/null | head -1)
|
||||
[[ -z "$internal_id" ]] && echo "not_found" && return 0
|
||||
curl -sf -o /dev/null -w '%{http_code}' -X DELETE \
|
||||
-H "X-Api-Key: $cached_key" \
|
||||
"http://${node_ip}:${port}/api/${api_ver}/${endpoint}/${internal_id}?deleteFiles=false" 2>/dev/null
|
||||
return
|
||||
fi
|
||||
local config_xml="${DOCKER_APPDATA_BASE}/${arr_type^}/config.xml"
|
||||
ssh -i "$SSH_KEY" -o ConnectTimeout="$ARR_SYNC_CONNECT_TIMEOUT" \
|
||||
root@"$node_ip" bash <<REMOTE 2>/dev/null
|
||||
KEY=\$(grep -oP '(?<=<ApiKey>)[^<]+' '${config_xml}' 2>/dev/null)
|
||||
[[ -z "\$KEY" ]] && exit 1
|
||||
LIBRARY=\$(curl -sf --max-time ${ARR_SYNC_API_TIMEOUT} \
|
||||
-H "X-Api-Key: \$KEY" \
|
||||
"http://localhost:${port}/api/${api_ver}/${endpoint}" 2>/dev/null)
|
||||
[[ -z "\$LIBRARY" ]] && exit 1
|
||||
INTERNAL_ID=\$(echo "\$LIBRARY" | jq -r '${select_expr}' 2>/dev/null | head -1)
|
||||
[[ -z "\$INTERNAL_ID" ]] && echo "not_found" && exit 0
|
||||
curl -sf -o /dev/null -w '%{http_code}' -X DELETE \
|
||||
-H "X-Api-Key: \$KEY" \
|
||||
"http://localhost:${port}/api/${api_ver}/${endpoint}/\${INTERNAL_ID}?deleteFiles=false"
|
||||
REMOTE
|
||||
}
|
||||
|
||||
# ── Blocklist management mode ──────────────────────────────────────────────────────────────────
|
||||
if [[ -n "$BLOCKLIST_ACTION" ]]; then
|
||||
case "$BLOCKLIST_ACTION" in
|
||||
list)
|
||||
echo ""
|
||||
echo "━━━━━ ARR SYNC BLOCKLIST ━━━━━"
|
||||
if [[ ! -f "$ARR_SYNC_BLOCKLIST" ]] || [[ ! -s "$ARR_SYNC_BLOCKLIST" ]]; then
|
||||
echo " (empty)"
|
||||
else
|
||||
echo ""
|
||||
printf '%-8s %-40s %-30s %s\n' "Arr" "Stable ID" "Name" "Reason"
|
||||
printf '%-8s %-40s %-30s %s\n' "---" "---------" "----" "------"
|
||||
while IFS=$'\t' read -r arr_type stable_id display_name tombstoned_by ts reason; do
|
||||
[[ -z "$arr_type" ]] || [[ "$arr_type" == \#* ]] && continue
|
||||
printf '%-8s %-40s %-30s %s\n' "$arr_type" "$stable_id" "$display_name" "$reason"
|
||||
done < "$ARR_SYNC_BLOCKLIST"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
;;
|
||||
add)
|
||||
if [[ -z "$BLOCKLIST_ARR" ]] || [[ -z "$BLOCKLIST_ID" ]]; then
|
||||
error "Usage: arr_sync.sh --blocklist-add <arr_type> <stable_id> [reason]"
|
||||
error " arr_type: lidarr | sonarr | radarr"
|
||||
exit 1
|
||||
fi
|
||||
if [[ -z "${_PORT[$BLOCKLIST_ARR]:-}" ]]; then
|
||||
error "Unknown arr type: $BLOCKLIST_ARR — use lidarr, sonarr, or radarr"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
_bl_port="${_PORT[$BLOCKLIST_ARR]}"
|
||||
_bl_ver="${_VER[$BLOCKLIST_ARR]}"
|
||||
_bl_ep="${_EP[$BLOCKLIST_ARR]}"
|
||||
_bl_id_field="${_ID[$BLOCKLIST_ARR]}"
|
||||
_bl_id_type="${_ID_TYPE[$BLOCKLIST_ARR]}"
|
||||
_bl_name_field="${_NAME[$BLOCKLIST_ARR]}"
|
||||
_bl_url="" _bl_key=""
|
||||
case "$BLOCKLIST_ARR" in
|
||||
lidarr) _bl_url="${LIDARR_URL:-}"; _bl_key="${LIDARR_API_KEY:-}" ;;
|
||||
sonarr) _bl_url="${SONARR_URL:-}"; _bl_key="${SONARR_API_KEY:-}" ;;
|
||||
radarr) _bl_url="${RADARR_URL:-}"; _bl_key="${RADARR_API_KEY:-}" ;;
|
||||
esac
|
||||
|
||||
# Look up display name and internal id from local arr
|
||||
_bl_display_name="$BLOCKLIST_ID"
|
||||
_bl_internal_id=""
|
||||
if [[ -n "$_bl_url" ]] && [[ -n "$_bl_key" ]]; then
|
||||
_bl_lookup=$(_lookup_local_item "$_bl_url" "$_bl_key" "$_bl_ver" "$_bl_ep" \
|
||||
"$_bl_id_field" "$_bl_id_type" "$_bl_name_field" "$BLOCKLIST_ID")
|
||||
if [[ -n "$_bl_lookup" ]]; then
|
||||
IFS=$'\t' read -r _bl_internal_id _bl_display_name <<< "$_bl_lookup"
|
||||
fi
|
||||
fi
|
||||
|
||||
_blocklist_add "$BLOCKLIST_ARR" "$BLOCKLIST_ID" "$_bl_display_name" "${BLOCKLIST_REASON:-manually excluded}"
|
||||
echo " Blocklisted [$BLOCKLIST_ARR] $_bl_display_name ($BLOCKLIST_ID)"
|
||||
|
||||
# Remove from local arr (deleteFiles=false — arr_cleanup handles file removal)
|
||||
if [[ -n "$_bl_internal_id" ]]; then
|
||||
_bl_http=$(_delete_local_item "$_bl_url" "$_bl_key" "$_bl_ver" "$_bl_ep" "$_bl_internal_id")
|
||||
if [[ "$_bl_http" == "200" ]]; then
|
||||
log "Removed from local ${BLOCKLIST_ARR^}: $_bl_display_name"
|
||||
else
|
||||
warn "Failed to remove from local ${BLOCKLIST_ARR^} (HTTP ${_bl_http:-no response}) — remove manually via UI"
|
||||
fi
|
||||
else
|
||||
log "Not found in local ${BLOCKLIST_ARR^} — already removed or not tracked locally"
|
||||
fi
|
||||
|
||||
# Remove from all remote arrs
|
||||
for _bl_node_id in "${REMOTE_NODES[@]}"; do
|
||||
_bl_node_name="${!_bl_node_id}"
|
||||
_bl_result=$(_delete_remote_item "$_bl_node_id" "$_bl_port" "$_bl_ver" \
|
||||
"$BLOCKLIST_ARR" "$_bl_ep" "$_bl_id_field" "$_bl_id_type" "$BLOCKLIST_ID")
|
||||
case "$_bl_result" in
|
||||
200) log "Removed from $_bl_node_name ${BLOCKLIST_ARR^}: $_bl_display_name" ;;
|
||||
not_found) log "Not found on $_bl_node_name ${BLOCKLIST_ARR^} — already removed or not tracked" ;;
|
||||
*) warn "Failed to remove from $_bl_node_name ${BLOCKLIST_ARR^} (${_bl_result:-SSH error}) — remove manually via UI" ;;
|
||||
esac
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo " Files are now orphans on all nodes — arr_cleanup.sh will remove them on next run"
|
||||
exit 0
|
||||
;;
|
||||
remove)
|
||||
if [[ -z "$BLOCKLIST_ARR" ]] || [[ -z "$BLOCKLIST_ID" ]]; then
|
||||
error "Usage: arr_sync.sh --blocklist-remove <arr_type> <stable_id>"
|
||||
exit 1
|
||||
fi
|
||||
_blocklist_remove "$BLOCKLIST_ARR" "$BLOCKLIST_ID"
|
||||
echo " Removed [$BLOCKLIST_ARR] $BLOCKLIST_ID from blocklist"
|
||||
exit 0
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ── STATUS ────────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY ARR SYNC STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_HOST Remote nodes: ${REMOTE_NODES[*]}"
|
||||
echo "$ICON_GEAR Appdata base: $DOCKER_APPDATA_BASE"
|
||||
echo "$ICON_GEAR Blocklist: $ARR_SYNC_BLOCKLIST"
|
||||
echo "$ICON_GEAR Connect timeout: ${ARR_SYNC_CONNECT_TIMEOUT}s"
|
||||
echo "$ICON_GEAR API timeout: ${ARR_SYNC_API_TIMEOUT}s"
|
||||
echo ""
|
||||
echo " Arr Port URL"
|
||||
for _arr in "${ARR_TYPES[@]}"; do
|
||||
local _url="" _key=""
|
||||
case "$_arr" in
|
||||
lidarr) _url="${LIDARR_URL:-not configured}" ;;
|
||||
sonarr) _url="${SONARR_URL:-not configured}" ;;
|
||||
radarr) _url="${RADARR_URL:-not configured}" ;;
|
||||
esac
|
||||
printf ' %-8s %-6s %s\n' "$_arr" "${_PORT[$_arr]}" "$_url"
|
||||
done
|
||||
unset _arr _url
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ── REMOTE HELPERS ────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
_resolve_node_ip() {
|
||||
local node_id="$1"
|
||||
resolve_tailscale_ip "${!node_id}"
|
||||
}
|
||||
|
||||
# Check if arr is reachable on remote node
|
||||
# Args: node_id node_ip port api_ver arr_type
|
||||
_remote_arr_up() {
|
||||
local node_id="$1" node_ip="$2" port="$3" api_ver="$4" arr_type="$5"
|
||||
local _kvar="${node_id}_${arr_type^^}_API_KEY"; local cached_key="${!_kvar:-}"
|
||||
if [[ -n "$cached_key" ]]; then
|
||||
curl -sf --max-time 5 \
|
||||
-H "X-Api-Key: $cached_key" \
|
||||
"http://${node_ip}:${port}/api/${api_ver}/system/status" >/dev/null 2>/dev/null
|
||||
return
|
||||
fi
|
||||
local config_xml="${DOCKER_APPDATA_BASE}/${arr_type^}/config.xml"
|
||||
ssh -i "$SSH_KEY" -o ConnectTimeout="$ARR_SYNC_CONNECT_TIMEOUT" \
|
||||
root@"$node_ip" \
|
||||
"KEY=\$(grep -oP '(?<=<ApiKey>)[^<]+' '${config_xml}' 2>/dev/null)
|
||||
[[ -z \"\$KEY\" ]] && exit 1
|
||||
curl -sf --max-time 5 -H \"X-Api-Key: \$KEY\" \
|
||||
'http://localhost:${port}/api/${api_ver}/system/status' >/dev/null" 2>/dev/null
|
||||
}
|
||||
|
||||
# Fetch full library from remote arr — returns raw JSON array
|
||||
# Args: node_id node_ip port api_ver arr_type endpoint
|
||||
_remote_library() {
|
||||
local node_id="$1" node_ip="$2" port="$3" api_ver="$4" arr_type="$5" endpoint="$6"
|
||||
local _kvar="${node_id}_${arr_type^^}_API_KEY"; local cached_key="${!_kvar:-}"
|
||||
if [[ -n "$cached_key" ]]; then
|
||||
curl -sf --max-time "$ARR_SYNC_API_TIMEOUT" \
|
||||
-H "X-Api-Key: $cached_key" \
|
||||
"http://${node_ip}:${port}/api/${api_ver}/${endpoint}" 2>/dev/null
|
||||
return
|
||||
fi
|
||||
local config_xml="${DOCKER_APPDATA_BASE}/${arr_type^}/config.xml"
|
||||
ssh -i "$SSH_KEY" -o ConnectTimeout="$ARR_SYNC_CONNECT_TIMEOUT" \
|
||||
root@"$node_ip" \
|
||||
"KEY=\$(grep -oP '(?<=<ApiKey>)[^<]+' '${config_xml}' 2>/dev/null)
|
||||
[[ -z \"\$KEY\" ]] && exit 1
|
||||
curl -sf --max-time ${ARR_SYNC_API_TIMEOUT} \
|
||||
-H \"X-Api-Key: \$KEY\" \
|
||||
'http://localhost:${port}/api/${api_ver}/${endpoint}'" 2>/dev/null
|
||||
}
|
||||
|
||||
# Fetch remote arr defaults: qualityProfileId, rootFolderPath, metadataProfileId (Lidarr)
|
||||
# Args: node_id node_ip port api_ver arr_type
|
||||
_remote_defaults() {
|
||||
local node_id="$1" node_ip="$2" port="$3" api_ver="$4" arr_type="$5"
|
||||
local _kvar="${node_id}_${arr_type^^}_API_KEY"; local cached_key="${!_kvar:-}"
|
||||
if [[ -n "$cached_key" ]]; then
|
||||
local base_url="http://${node_ip}:${port}/api/${api_ver}"
|
||||
local qp rf mp
|
||||
qp=$(curl -sf -H "X-Api-Key: $cached_key" "${base_url}/qualityprofile" 2>/dev/null | jq '.[0].id // 1')
|
||||
rf=$(curl -sf -H "X-Api-Key: $cached_key" "${base_url}/rootfolder" 2>/dev/null | jq -r '.[0].path // ""')
|
||||
[[ -z "$qp" ]] && return 1
|
||||
if [[ "$arr_type" == "lidarr" ]]; then
|
||||
mp=$(curl -sf -H "X-Api-Key: $cached_key" "${base_url}/metadataprofile" 2>/dev/null | \
|
||||
jq 'map(select(.name == "Standard")) | .[0].id // .[0].id // 1')
|
||||
jq -n --argjson qp "$qp" --arg rf "$rf" --argjson mp "$mp" \
|
||||
'{qualityProfileId: $qp, rootFolderPath: $rf, metadataProfileId: $mp}'
|
||||
else
|
||||
jq -n --argjson qp "$qp" --arg rf "$rf" \
|
||||
'{qualityProfileId: $qp, rootFolderPath: $rf}'
|
||||
fi
|
||||
return
|
||||
fi
|
||||
local config_xml="${DOCKER_APPDATA_BASE}/${arr_type^}/config.xml"
|
||||
local meta_field=""
|
||||
[[ "$arr_type" == "lidarr" ]] && \
|
||||
meta_field=', metadataProfileId: ($mp | map(select(.name == "Standard")) | .[0].id // .[0].id // 1)'
|
||||
ssh -i "$SSH_KEY" -o ConnectTimeout="$ARR_SYNC_CONNECT_TIMEOUT" \
|
||||
root@"$node_ip" bash <<REMOTE 2>/dev/null
|
||||
KEY=\$(grep -oP '(?<=<ApiKey>)[^<]+' '${config_xml}' 2>/dev/null)
|
||||
[[ -z "\$KEY" ]] && exit 1
|
||||
QP=\$(curl -sf -H "X-Api-Key: \$KEY" "http://localhost:${port}/api/${api_ver}/qualityprofile")
|
||||
RF=\$(curl -sf -H "X-Api-Key: \$KEY" "http://localhost:${port}/api/${api_ver}/rootfolder")
|
||||
MP=\$(curl -sf -H "X-Api-Key: \$KEY" "http://localhost:${port}/api/${api_ver}/metadataprofile" 2>/dev/null || echo '[]')
|
||||
jq -n --argjson qp "\$QP" --argjson rf "\$RF" --argjson mp "\$MP" \
|
||||
'{qualityProfileId: (\$qp | .[0].id // 1), rootFolderPath: (\$rf | .[0].path // "")${meta_field}}'
|
||||
REMOTE
|
||||
}
|
||||
|
||||
# Add item to remote arr — payload is base64-encoded to avoid quoting issues
|
||||
# Args: node_id node_ip port api_ver arr_type endpoint encoded_payload
|
||||
_remote_add() {
|
||||
local node_id="$1" node_ip="$2" port="$3" api_ver="$4" arr_type="$5" endpoint="$6"
|
||||
local encoded="$7"
|
||||
local _kvar="${node_id}_${arr_type^^}_API_KEY"; local cached_key="${!_kvar:-}"
|
||||
if [[ -n "$cached_key" ]]; then
|
||||
local body
|
||||
body=$(printf '%s' "$encoded" | base64 -d)
|
||||
curl -sf -o /dev/null -w '%{http_code}' -X POST \
|
||||
-H "X-Api-Key: $cached_key" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "$body" \
|
||||
"http://${node_ip}:${port}/api/${api_ver}/${endpoint}" 2>/dev/null
|
||||
return
|
||||
fi
|
||||
local config_xml="${DOCKER_APPDATA_BASE}/${arr_type^}/config.xml"
|
||||
ssh -i "$SSH_KEY" -o ConnectTimeout="$ARR_SYNC_CONNECT_TIMEOUT" \
|
||||
root@"$node_ip" bash <<REMOTE 2>/dev/null
|
||||
KEY=\$(grep -oP '(?<=<ApiKey>)[^<]+' '${config_xml}' 2>/dev/null)
|
||||
[[ -z "\$KEY" ]] && exit 1
|
||||
BODY=\$(printf '%s' '${encoded}' | base64 -d)
|
||||
curl -sf -o /dev/null -w '%{http_code}' -X POST \
|
||||
-H "X-Api-Key: \$KEY" \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d "\$BODY" \
|
||||
"http://localhost:${port}/api/${api_ver}/${endpoint}"
|
||||
REMOTE
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ── LOCAL HELPERS ─────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
_local_library() {
|
||||
local url="$1" api_key="$2" api_ver="$3" endpoint="$4"
|
||||
curl -sf --max-time "$ARR_SYNC_API_TIMEOUT" \
|
||||
-H "X-Api-Key: $api_key" \
|
||||
"${url}/api/${api_ver}/${endpoint}" 2>/dev/null
|
||||
}
|
||||
|
||||
_local_defaults() {
|
||||
local url="$1" api_key="$2" api_ver="$3" arr_type="$4"
|
||||
local qp rf mp meta_field=""
|
||||
qp=$(curl -sf -H "X-Api-Key: $api_key" "${url}/api/${api_ver}/qualityprofile" | jq '.[0].id // 1')
|
||||
rf=$(curl -sf -H "X-Api-Key: $api_key" "${url}/api/${api_ver}/rootfolder" | jq -r '.[0].path // ""')
|
||||
if [[ "$arr_type" == "lidarr" ]]; then
|
||||
mp=$(curl -sf -H "X-Api-Key: $api_key" "${url}/api/${api_ver}/metadataprofile" | \
|
||||
jq 'map(select(.name == "Standard")) | .[0].id // .[0].id // 1')
|
||||
jq -n --argjson qp "$qp" --arg rf "$rf" --argjson mp "$mp" \
|
||||
'{qualityProfileId: $qp, rootFolderPath: $rf, metadataProfileId: $mp}'
|
||||
else
|
||||
jq -n --argjson qp "$qp" --arg rf "$rf" \
|
||||
'{qualityProfileId: $qp, rootFolderPath: $rf}'
|
||||
fi
|
||||
}
|
||||
|
||||
_local_add() {
|
||||
local url="$1" api_key="$2" api_ver="$3" endpoint="$4" payload="$5"
|
||||
curl -sf -o /dev/null -w '%{http_code}' -X POST \
|
||||
-H "X-Api-Key: $api_key" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "$payload" \
|
||||
"${url}/api/${api_ver}/${endpoint}" 2>/dev/null
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ── PAYLOAD BUILDER ───────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Builds the minimal POST body to add an item to an arr.
|
||||
# Uses the TARGET node's own defaults — never copies source node settings.
|
||||
|
||||
_build_payload() {
|
||||
local arr_type="$1" stable_id="$2" display_name="$3" monitored="$4" defaults_json="$5"
|
||||
case "$arr_type" in
|
||||
lidarr)
|
||||
jq -n \
|
||||
--arg id "$stable_id" \
|
||||
--arg nm "$display_name" \
|
||||
--argjson mn "$monitored" \
|
||||
--argjson df "$defaults_json" \
|
||||
'{
|
||||
foreignArtistId: $id,
|
||||
artistName: $nm,
|
||||
monitored: $mn,
|
||||
qualityProfileId: $df.qualityProfileId,
|
||||
metadataProfileId: ($df.metadataProfileId // 1),
|
||||
rootFolderPath: $df.rootFolderPath,
|
||||
addOptions: {monitor: "all", searchForMissingAlbums: false}
|
||||
}'
|
||||
;;
|
||||
sonarr)
|
||||
jq -n \
|
||||
--argjson id "$stable_id" \
|
||||
--arg nm "$display_name" \
|
||||
--argjson mn "$monitored" \
|
||||
--argjson df "$defaults_json" \
|
||||
'{
|
||||
tvdbId: $id,
|
||||
title: $nm,
|
||||
monitored: $mn,
|
||||
qualityProfileId: $df.qualityProfileId,
|
||||
rootFolderPath: $df.rootFolderPath,
|
||||
seasons: [],
|
||||
addOptions: {searchForMissingEpisodes: false, monitor: "all"}
|
||||
}'
|
||||
;;
|
||||
radarr)
|
||||
jq -n \
|
||||
--argjson id "$stable_id" \
|
||||
--arg nm "$display_name" \
|
||||
--argjson mn "$monitored" \
|
||||
--argjson df "$defaults_json" \
|
||||
'{
|
||||
tmdbId: $id,
|
||||
title: $nm,
|
||||
monitored: $mn,
|
||||
qualityProfileId: $df.qualityProfileId,
|
||||
rootFolderPath: $df.rootFolderPath,
|
||||
addOptions: {searchForMovie: false}
|
||||
}'
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ── MONITORED ENFORCEMENT — re-monitor any unmonitored items via bulk editor ──────────────────
|
||||
# ==============================================================================================
|
||||
# All library members must be monitored. Unmonitored items won't be searched and eventually
|
||||
# lose their files when cleanup runs after the series is removed from the arr.
|
||||
|
||||
_enforce_monitored() {
|
||||
local arr_type="$1" url="$2" api_key="$3" api_ver="$4" local_json="$5"
|
||||
|
||||
local bulk_endpoint ids_key
|
||||
case "$arr_type" in
|
||||
sonarr) bulk_endpoint="series/editor"; ids_key="seriesIds" ;;
|
||||
radarr) bulk_endpoint="movie/editor"; ids_key="movieIds" ;;
|
||||
lidarr) bulk_endpoint="artist/editor"; ids_key="artistIds" ;;
|
||||
*) return 0 ;;
|
||||
esac
|
||||
|
||||
local ids_json count
|
||||
ids_json=$(echo "$local_json" | jq '[.[] | select(.monitored == false) | .id]' 2>/dev/null)
|
||||
count=$(echo "$ids_json" | jq 'length' 2>/dev/null || echo 0)
|
||||
[[ "$count" -eq 0 ]] && return 0
|
||||
|
||||
warn "${arr_type^}: $count unmonitored items — re-monitoring"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN: would re-monitor $count ${arr_type^} items"
|
||||
return 0
|
||||
fi
|
||||
|
||||
local body http_code
|
||||
body=$(jq -n --arg k "$ids_key" --argjson ids "$ids_json" '{($k): $ids, monitored: true}')
|
||||
http_code=$(curl -sf -o /dev/null -w '%{http_code}' -X PUT \
|
||||
-H "X-Api-Key: $api_key" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "$body" \
|
||||
"${url}/api/${api_ver}/${bulk_endpoint}" 2>/dev/null)
|
||||
|
||||
if [[ "$http_code" == "200" || "$http_code" == "202" ]]; then
|
||||
log "${arr_type^}: re-monitored $count items ✅"
|
||||
else
|
||||
warn "${arr_type^}: bulk re-monitor failed (HTTP $http_code)"
|
||||
fi
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ── CORE SYNC — one arr type across all remote nodes ──────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
_sync_arr() {
|
||||
local arr_type="$1"
|
||||
local port="${_PORT[$arr_type]}"
|
||||
local api_ver="${_VER[$arr_type]}"
|
||||
local endpoint="${_EP[$arr_type]}"
|
||||
local id_field="${_ID[$arr_type]}"
|
||||
local name_field="${_NAME[$arr_type]}"
|
||||
local id_type="${_ID_TYPE[$arr_type]}"
|
||||
|
||||
# Resolve local credentials
|
||||
local local_url local_key
|
||||
case "$arr_type" in
|
||||
lidarr) local_url="${LIDARR_URL:-}"; local_key="${LIDARR_API_KEY:-}" ;;
|
||||
sonarr) local_url="${SONARR_URL:-}"; local_key="${SONARR_API_KEY:-}" ;;
|
||||
radarr) local_url="${RADARR_URL:-}"; local_key="${RADARR_API_KEY:-}" ;;
|
||||
esac
|
||||
|
||||
if [[ -z "$local_url" ]] || [[ -z "$local_key" ]]; then
|
||||
log "${arr_type^}: not configured on $MY_ID — skipping"
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "━━━ ${arr_type^} ━━━"
|
||||
|
||||
# ── Fetch local library ────────────────────────────────────────────────────────────────────
|
||||
local local_json
|
||||
local_json=$(_local_library "$local_url" "$local_key" "$api_ver" "$endpoint")
|
||||
if [[ -z "$local_json" ]] || ! echo "$local_json" | jq -e '.' >/dev/null 2>&1; then
|
||||
warn "${arr_type^}: could not fetch local library — skipping"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Build local ID map: stable_id → "display_name|monitored"
|
||||
declare -A local_ids
|
||||
local local_count=0
|
||||
local _jq_id
|
||||
[[ "$id_type" == "string" ]] && _jq_id=".${id_field}" || _jq_id="(.${id_field} | tostring)"
|
||||
while IFS=$'\t' read -r stable_id display_name monitored; do
|
||||
[[ -z "$stable_id" ]] && continue
|
||||
local_ids["$stable_id"]="${display_name}|${monitored}"
|
||||
(( local_count++ ))
|
||||
done < <(echo "$local_json" | jq -r \
|
||||
".[] | [${_jq_id}, .${name_field}, (.monitored | tostring)] | @tsv" 2>/dev/null)
|
||||
unset _jq_id
|
||||
|
||||
log "${arr_type^}: $local_count items in local library"
|
||||
|
||||
# ── Enforce monitored — fix any unmonitored items before sync ─────────────────────────────
|
||||
_enforce_monitored "$arr_type" "$local_url" "$local_key" "$api_ver" "$local_json"
|
||||
|
||||
local total_added_local=0 total_added_remote=0 total_skipped=0
|
||||
|
||||
# ── Sync with each remote node ─────────────────────────────────────────────────────────────
|
||||
for node_id in "${REMOTE_NODES[@]}"; do
|
||||
local node_name="${!node_id}"
|
||||
log "${arr_type^}: syncing with $node_name..."
|
||||
|
||||
local node_ip
|
||||
node_ip=$(_resolve_node_ip "$node_id") || {
|
||||
warn "${arr_type^}: cannot resolve Tailscale IP for $node_name — skipping"
|
||||
continue
|
||||
}
|
||||
|
||||
if ! _remote_arr_up "$node_id" "$node_ip" "$port" "$api_ver" "$arr_type"; then
|
||||
log "${arr_type^}: not reachable on $node_name — skipping"
|
||||
continue
|
||||
fi
|
||||
|
||||
local remote_json
|
||||
remote_json=$(_remote_library "$node_id" "$node_ip" "$port" "$api_ver" "$arr_type" "$endpoint")
|
||||
if [[ -z "$remote_json" ]] || ! echo "$remote_json" | jq -e '.' >/dev/null 2>&1; then
|
||||
warn "${arr_type^}: could not fetch library from $node_name — skipping"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Build remote ID map
|
||||
declare -A remote_ids
|
||||
local remote_count=0
|
||||
local _jq_id
|
||||
[[ "$id_type" == "string" ]] && _jq_id=".${id_field}" || _jq_id="(.${id_field} | tostring)"
|
||||
while IFS=$'\t' read -r stable_id display_name monitored; do
|
||||
[[ -z "$stable_id" ]] && continue
|
||||
remote_ids["$stable_id"]="${display_name}|${monitored}"
|
||||
(( remote_count++ ))
|
||||
done < <(echo "$remote_json" | jq -r \
|
||||
".[] | [${_jq_id}, .${name_field}, (.monitored | tostring)] | @tsv" 2>/dev/null)
|
||||
unset _jq_id
|
||||
|
||||
log "${arr_type^}: $remote_count items on $node_name"
|
||||
|
||||
# ── Remote → Local: items on remote not in local ───────────────────────────────────────
|
||||
local to_add_local=()
|
||||
for stable_id in "${!remote_ids[@]}"; do
|
||||
[[ -n "${local_ids[$stable_id]:-}" ]] && continue
|
||||
if _is_blocklisted "$arr_type" "$stable_id"; then
|
||||
log "BLOCKLISTED [$arr_type] $stable_id — skipping"
|
||||
(( total_skipped++ ))
|
||||
continue
|
||||
fi
|
||||
to_add_local+=("$stable_id")
|
||||
done
|
||||
|
||||
if [[ "${#to_add_local[@]}" -gt 0 ]]; then
|
||||
local local_defs
|
||||
local_defs=$(_local_defaults "$local_url" "$local_key" "$api_ver" "$arr_type")
|
||||
if [[ -z "$local_defs" ]]; then
|
||||
warn "${arr_type^}: could not fetch local defaults — skipping adds from $node_name"
|
||||
else
|
||||
for stable_id in "${to_add_local[@]}"; do
|
||||
IFS='|' read -r display_name _ <<< "${remote_ids[$stable_id]}"
|
||||
local payload
|
||||
payload=$(_build_payload "$arr_type" "$stable_id" "$display_name" \
|
||||
"true" "$local_defs")
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
log "DRY RUN: would add to local ${arr_type^}: $display_name ($stable_id)"
|
||||
(( total_added_local++ ))
|
||||
else
|
||||
local http_code
|
||||
http_code=$(_local_add "$local_url" "$local_key" "$api_ver" \
|
||||
"$endpoint" "$payload")
|
||||
if [[ "$http_code" == "201" ]] || [[ "$http_code" == "200" ]]; then
|
||||
log "Added to local ${arr_type^}: $display_name"
|
||||
local_ids["$stable_id"]="${display_name}|true"
|
||||
(( total_added_local++ ))
|
||||
else
|
||||
warn "Failed to add to local ${arr_type^}: $display_name (HTTP $http_code)"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Local → Remote: items on local not on remote ───────────────────────────────────────
|
||||
local to_add_remote=()
|
||||
for stable_id in "${!local_ids[@]}"; do
|
||||
[[ -n "${remote_ids[$stable_id]:-}" ]] && continue
|
||||
if _is_blocklisted "$arr_type" "$stable_id"; then
|
||||
(( total_skipped++ ))
|
||||
continue
|
||||
fi
|
||||
to_add_remote+=("$stable_id")
|
||||
done
|
||||
|
||||
if [[ "${#to_add_remote[@]}" -gt 0 ]]; then
|
||||
local remote_defs
|
||||
remote_defs=$(_remote_defaults "$node_id" "$node_ip" "$port" "$api_ver" "$arr_type")
|
||||
if [[ -z "$remote_defs" ]]; then
|
||||
warn "${arr_type^}: could not fetch defaults from $node_name — skipping remote adds"
|
||||
else
|
||||
for stable_id in "${to_add_remote[@]}"; do
|
||||
IFS='|' read -r display_name _ <<< "${local_ids[$stable_id]}"
|
||||
local payload
|
||||
payload=$(_build_payload "$arr_type" "$stable_id" "$display_name" \
|
||||
"true" "$remote_defs")
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
log "DRY RUN: would add to $node_name ${arr_type^}: $display_name ($stable_id)"
|
||||
(( total_added_remote++ ))
|
||||
else
|
||||
local encoded http_code
|
||||
encoded=$(printf '%s' "$payload" | base64 -w0)
|
||||
http_code=$(_remote_add "$node_id" "$node_ip" "$port" "$api_ver" \
|
||||
"$arr_type" "$endpoint" "$encoded")
|
||||
if [[ "$http_code" == "201" ]] || [[ "$http_code" == "200" ]]; then
|
||||
log "Added to $node_name ${arr_type^}: $display_name"
|
||||
(( total_added_remote++ ))
|
||||
else
|
||||
warn "Failed to add to $node_name ${arr_type^}: $display_name (HTTP $http_code)"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
fi
|
||||
fi
|
||||
|
||||
log " $node_name: +${#to_add_local[@]} local | +${#to_add_remote[@]} remote | $total_skipped blocklisted"
|
||||
|
||||
unset remote_ids
|
||||
declare -A remote_ids
|
||||
done
|
||||
|
||||
echo " Total added to local: $total_added_local | to remotes: $total_added_remote | blocklisted: $total_skipped"
|
||||
unset local_ids
|
||||
declare -A local_ids
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Main ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Loading Blocklist ━━━"
|
||||
_load_blocklist
|
||||
|
||||
START=$(date +%s)
|
||||
|
||||
for arr_type in "${ARR_TYPES[@]}"; do
|
||||
_sync_arr "$arr_type"
|
||||
done
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY ARR SYNC SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Node: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_SYNC Peers: ${REMOTE_NODES[*]}"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes were made"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
exit 0
|
||||
+112
@@ -0,0 +1,112 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ================================= Claude Code Startup ========================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Restores Claude Code's persistent data after an Unraid reboot and launches Claude.
|
||||
# Unraid's root filesystem lives in RAM — /root/.claude and /root/.local are wiped on
|
||||
# every reboot. This script symlinks both directories back to persistent appdata storage
|
||||
# before launching Claude, so memory, sessions, and settings survive across reboots.
|
||||
#
|
||||
# On first run with no existing persistent data, migrates from the current live locations:
|
||||
# /root/.claude → PERSIST_DIR/.claude (memory, sessions, settings)
|
||||
# /root/.local/share/claude → PERSIST_DIR/local/share/claude (installed binaries)
|
||||
# Subsequent runs skip the migration and only create the symlinks.
|
||||
#
|
||||
# Standalone script — no common.sh dependency. Safe to run directly from terminal
|
||||
# or from array_started.sh.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# claude_startup.sh
|
||||
# Set up persistent symlinks only — default, used by array_started.sh on boot.
|
||||
#
|
||||
# claude_startup.sh --launch
|
||||
# Set up persistent symlinks and launch Claude interactively.
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
PERSIST_DIR="/mnt/user/appdata/claude-code"
|
||||
CLAUDE_DATA="$PERSIST_DIR/.claude"
|
||||
CLAUDE_BIN="$PERSIST_DIR/local/share/claude"
|
||||
|
||||
LAUNCH=false
|
||||
[[ "$1" == "--launch" ]] && LAUNCH=true
|
||||
|
||||
# Standalone — no common.sh dependency
|
||||
_log() { echo " ✅ $*"; }
|
||||
_warn() { echo " ⚠️ $*"; }
|
||||
_err() { echo " ❌ $*" >&2; }
|
||||
|
||||
echo ""
|
||||
echo "━━━ Claude Code Startup ━━━"
|
||||
echo ""
|
||||
|
||||
# ── Array must be mounted ─────────────────────────────────────────────────────────────────────
|
||||
if ! mountpoint -q /mnt/user 2>/dev/null; then
|
||||
_err "Array not mounted — /mnt/user not available"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ── Create persistent dirs ────────────────────────────────────────────────────────────────────
|
||||
mkdir -p "$CLAUDE_DATA" "$CLAUDE_BIN"
|
||||
|
||||
# ── Migrate .claude on first run ──────────────────────────────────────────────────────────────
|
||||
if [[ ! -L /root/.claude && -d /root/.claude ]]; then
|
||||
_warn "First run — migrating /root/.claude → $CLAUDE_DATA"
|
||||
cp -a /root/.claude/. "$CLAUDE_DATA/"
|
||||
rm -rf /root/.claude
|
||||
_log "Migrated .claude (memory, sessions, settings)"
|
||||
elif [[ -z "$(ls -A "$CLAUDE_DATA" 2>/dev/null)" && -d /root/.claude ]]; then
|
||||
_warn "Persistent storage empty — copying current .claude data"
|
||||
cp -a /root/.claude/. "$CLAUDE_DATA/"
|
||||
_log "Copied .claude data to persistent storage"
|
||||
fi
|
||||
|
||||
# ── Migrate Claude binaries on first run ──────────────────────────────────────────────────────
|
||||
if [[ ! -L /root/.local/share/claude && -d /root/.local/share/claude ]]; then
|
||||
_warn "First run — migrating Claude binaries → $CLAUDE_BIN"
|
||||
cp -a /root/.local/share/claude/. "$CLAUDE_BIN/"
|
||||
_log "Migrated Claude binaries"
|
||||
fi
|
||||
|
||||
# ── Create symlinks ───────────────────────────────────────────────────────────────────────────
|
||||
# Remove any real directories first — ln -sfn silently creates inside a dir instead of
|
||||
# replacing it, which produces a circular symlink on subsequent runs after migration.
|
||||
mkdir -p /root/.local/share /root/.local/bin
|
||||
|
||||
[[ -d /root/.claude && ! -L /root/.claude ]] && rm -rf /root/.claude
|
||||
ln -sfn "$CLAUDE_DATA" /root/.claude
|
||||
_log ".claude → $CLAUDE_DATA"
|
||||
|
||||
[[ -d /root/.local/share/claude && ! -L /root/.local/share/claude ]] && rm -rf /root/.local/share/claude
|
||||
ln -sfn "$CLAUDE_BIN" /root/.local/share/claude
|
||||
_log "claude binary → $CLAUDE_BIN"
|
||||
|
||||
# ── Point the claude binary at the latest installed version ───────────────────────────────────
|
||||
LATEST=$(ls "$CLAUDE_BIN/versions/" 2>/dev/null | sort -V | tail -1)
|
||||
if [[ -z "$LATEST" ]]; then
|
||||
_err "No Claude versions found in $CLAUDE_BIN/versions/"
|
||||
_err "Install Claude Code first: npm install -g @anthropic-ai/claude-code"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
ln -sfn "$CLAUDE_BIN/versions/$LATEST" /root/.local/bin/claude
|
||||
_log "claude v$LATEST ready"
|
||||
|
||||
echo ""
|
||||
|
||||
# ── Setup-only mode (used by array_started.sh or other callers) ─────────────────────────────────
|
||||
if [[ "$LAUNCH" == false ]]; then
|
||||
_log "Setup complete — run 'claude' to start"
|
||||
echo ""
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ── Launch ────────────────────────────────────────────────────────────────────────────────────
|
||||
export PATH="$HOME/.local/bin:$PATH"
|
||||
exec claude
|
||||
+123
@@ -0,0 +1,123 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ================================= Claude Code Startup ========================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Restores Claude Code's persistent data after an Unraid reboot and launches Claude.
|
||||
# Unraid's root filesystem lives in RAM — /root/.claude and /root/.local are wiped on
|
||||
# every reboot. This script symlinks both directories back to persistent appdata storage
|
||||
# before launching Claude, so memory, sessions, and settings survive across reboots.
|
||||
#
|
||||
# On first run with no existing persistent data, migrates from the current live locations:
|
||||
# /root/.claude → PERSIST_DIR/.claude (memory, sessions, settings)
|
||||
# /root/.local/share/claude → PERSIST_DIR/local/share/claude (installed binaries)
|
||||
# Subsequent runs skip the migration and only create the symlinks.
|
||||
#
|
||||
# Standalone script — no common.sh dependency. Safe to run directly from terminal
|
||||
# or from array_started.sh.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# claude_startup.sh
|
||||
# Set up persistent symlinks only — default, used by array_started.sh on boot.
|
||||
#
|
||||
# claude_startup.sh --launch
|
||||
# Set up persistent symlinks and launch Claude interactively.
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
PERSIST_DIR="/mnt/user/appdata/claude-code"
|
||||
CLAUDE_DATA="$PERSIST_DIR/.claude"
|
||||
CLAUDE_BIN="$PERSIST_DIR/local/share/claude"
|
||||
|
||||
LAUNCH=false
|
||||
[[ "$1" == "--launch" ]] && LAUNCH=true
|
||||
|
||||
# Standalone — no common.sh dependency
|
||||
_log() { echo " ✅ $*"; }
|
||||
_warn() { echo " ⚠️ $*"; }
|
||||
_err() { echo " ❌ $*" >&2; }
|
||||
|
||||
echo ""
|
||||
echo "━━━ Claude Code Startup ━━━"
|
||||
echo ""
|
||||
|
||||
# ── Array must be mounted ─────────────────────────────────────────────────────────────────────
|
||||
if ! mountpoint -q /mnt/user 2>/dev/null; then
|
||||
_err "Array not mounted — /mnt/user not available"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ── Create persistent dirs ────────────────────────────────────────────────────────────────────
|
||||
mkdir -p "$CLAUDE_DATA" "$CLAUDE_BIN"
|
||||
|
||||
# ── Migrate .claude on first run ──────────────────────────────────────────────────────────────
|
||||
if [[ ! -L /root/.claude && -d /root/.claude ]]; then
|
||||
_warn "First run — migrating /root/.claude → $CLAUDE_DATA"
|
||||
cp -a /root/.claude/. "$CLAUDE_DATA/"
|
||||
rm -rf /root/.claude
|
||||
_log "Migrated .claude (memory, sessions, settings)"
|
||||
elif [[ -z "$(ls -A "$CLAUDE_DATA" 2>/dev/null)" && -d /root/.claude ]]; then
|
||||
_warn "Persistent storage empty — copying current .claude data"
|
||||
cp -a /root/.claude/. "$CLAUDE_DATA/"
|
||||
_log "Copied .claude data to persistent storage"
|
||||
fi
|
||||
|
||||
# ── Migrate Claude binaries on first run ──────────────────────────────────────────────────────
|
||||
if [[ ! -L /root/.local/share/claude && -d /root/.local/share/claude ]]; then
|
||||
_warn "First run — migrating Claude binaries → $CLAUDE_BIN"
|
||||
cp -a /root/.local/share/claude/. "$CLAUDE_BIN/"
|
||||
_log "Migrated Claude binaries"
|
||||
fi
|
||||
|
||||
# ── Create symlinks ───────────────────────────────────────────────────────────────────────────
|
||||
# Remove any real directories first — ln -sfn silently creates inside a dir instead of
|
||||
# replacing it, which produces a circular symlink on subsequent runs after migration.
|
||||
mkdir -p /root/.local/share /root/.local/bin
|
||||
|
||||
[[ -d /root/.claude && ! -L /root/.claude ]] && rm -rf /root/.claude
|
||||
ln -sfn "$CLAUDE_DATA" /root/.claude
|
||||
_log ".claude → $CLAUDE_DATA"
|
||||
|
||||
[[ -d /root/.local/share/claude && ! -L /root/.local/share/claude ]] && rm -rf /root/.local/share/claude
|
||||
ln -sfn "$CLAUDE_BIN" /root/.local/share/claude
|
||||
_log "claude binary → $CLAUDE_BIN"
|
||||
|
||||
# ── Symlink CLAUDE.md ─────────────────────────────────────────────────────────────────────────
|
||||
# Lives on /boot so it survives reboots without appdata. Symlinked into /root so Claude
|
||||
# picks it up automatically from the working directory on every session.
|
||||
CLAUDE_MD="/boot/config/plugins/varaverk/CLAUDE.md"
|
||||
if [[ -f "$CLAUDE_MD" ]]; then
|
||||
ln -sfn "$CLAUDE_MD" /root/CLAUDE.md
|
||||
_log "CLAUDE.md → $CLAUDE_MD"
|
||||
else
|
||||
_warn "CLAUDE.md not found at $CLAUDE_MD — skipping symlink"
|
||||
fi
|
||||
|
||||
# ── Point the claude binary at the latest installed version ───────────────────────────────────
|
||||
LATEST=$(ls "$CLAUDE_BIN/versions/" 2>/dev/null | sort -V | tail -1)
|
||||
if [[ -z "$LATEST" ]]; then
|
||||
_err "No Claude versions found in $CLAUDE_BIN/versions/"
|
||||
_err "Install Claude Code first: npm install -g @anthropic-ai/claude-code"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
ln -sfn "$CLAUDE_BIN/versions/$LATEST" /root/.local/bin/claude
|
||||
_log "claude v$LATEST ready"
|
||||
|
||||
echo ""
|
||||
|
||||
# ── Setup-only mode (used by array_started.sh or other callers) ─────────────────────────────────
|
||||
if [[ "$LAUNCH" == false ]]; then
|
||||
_log "Setup complete — run 'claude' to start"
|
||||
echo ""
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ── Launch ────────────────────────────────────────────────────────────────────────────────────
|
||||
export PATH="$HOME/.local/bin:$PATH"
|
||||
exec claude
|
||||
Reference in New Issue
Block a user